]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
86db1e29 JA |
2 | /* |
3 | * Functions related to mapping data to requests | |
4 | */ | |
5 | #include <linux/kernel.h> | |
68db0cf1 | 6 | #include <linux/sched/task_stack.h> |
86db1e29 JA |
7 | #include <linux/module.h> |
8 | #include <linux/bio.h> | |
9 | #include <linux/blkdev.h> | |
26e49cfc | 10 | #include <linux/uio.h> |
86db1e29 JA |
11 | |
12 | #include "blk.h" | |
13 | ||
130879f1 | 14 | struct bio_map_data { |
f3256075 CH |
15 | bool is_our_pages : 1; |
16 | bool is_null_mapped : 1; | |
130879f1 CH |
17 | struct iov_iter iter; |
18 | struct iovec iov[]; | |
19 | }; | |
20 | ||
21 | static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, | |
22 | gfp_t gfp_mask) | |
23 | { | |
24 | struct bio_map_data *bmd; | |
25 | ||
26 | if (data->nr_segs > UIO_MAXIOV) | |
27 | return NULL; | |
28 | ||
29 | bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask); | |
30 | if (!bmd) | |
31 | return NULL; | |
32 | memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs); | |
33 | bmd->iter = *data; | |
34 | bmd->iter.iov = bmd->iov; | |
35 | return bmd; | |
36 | } | |
37 | ||
38 | /** | |
39 | * bio_copy_from_iter - copy all pages from iov_iter to bio | |
40 | * @bio: The &struct bio which describes the I/O as destination | |
41 | * @iter: iov_iter as source | |
42 | * | |
43 | * Copy all pages from iov_iter to bio. | |
44 | * Returns 0 on success, or error on failure. | |
45 | */ | |
46 | static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter) | |
47 | { | |
48 | struct bio_vec *bvec; | |
49 | struct bvec_iter_all iter_all; | |
50 | ||
51 | bio_for_each_segment_all(bvec, bio, iter_all) { | |
52 | ssize_t ret; | |
53 | ||
54 | ret = copy_page_from_iter(bvec->bv_page, | |
55 | bvec->bv_offset, | |
56 | bvec->bv_len, | |
57 | iter); | |
58 | ||
59 | if (!iov_iter_count(iter)) | |
60 | break; | |
61 | ||
62 | if (ret < bvec->bv_len) | |
63 | return -EFAULT; | |
64 | } | |
65 | ||
66 | return 0; | |
67 | } | |
68 | ||
69 | /** | |
70 | * bio_copy_to_iter - copy all pages from bio to iov_iter | |
71 | * @bio: The &struct bio which describes the I/O as source | |
72 | * @iter: iov_iter as destination | |
73 | * | |
74 | * Copy all pages from bio to iov_iter. | |
75 | * Returns 0 on success, or error on failure. | |
76 | */ | |
77 | static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter) | |
78 | { | |
79 | struct bio_vec *bvec; | |
80 | struct bvec_iter_all iter_all; | |
81 | ||
82 | bio_for_each_segment_all(bvec, bio, iter_all) { | |
83 | ssize_t ret; | |
84 | ||
85 | ret = copy_page_to_iter(bvec->bv_page, | |
86 | bvec->bv_offset, | |
87 | bvec->bv_len, | |
88 | &iter); | |
89 | ||
90 | if (!iov_iter_count(&iter)) | |
91 | break; | |
92 | ||
93 | if (ret < bvec->bv_len) | |
94 | return -EFAULT; | |
95 | } | |
96 | ||
97 | return 0; | |
98 | } | |
99 | ||
100 | /** | |
101 | * bio_uncopy_user - finish previously mapped bio | |
102 | * @bio: bio being terminated | |
103 | * | |
104 | * Free pages allocated from bio_copy_user_iov() and write back data | |
105 | * to user space in case of a read. | |
106 | */ | |
107 | static int bio_uncopy_user(struct bio *bio) | |
108 | { | |
109 | struct bio_map_data *bmd = bio->bi_private; | |
110 | int ret = 0; | |
111 | ||
3310eeba | 112 | if (!bmd->is_null_mapped) { |
130879f1 CH |
113 | /* |
114 | * if we're in a workqueue, the request is orphaned, so | |
115 | * don't copy into a random user address space, just free | |
116 | * and return -EINTR so user space doesn't expect any data. | |
117 | */ | |
118 | if (!current->mm) | |
119 | ret = -EINTR; | |
120 | else if (bio_data_dir(bio) == READ) | |
121 | ret = bio_copy_to_iter(bio, bmd->iter); | |
122 | if (bmd->is_our_pages) | |
123 | bio_free_pages(bio); | |
124 | } | |
125 | kfree(bmd); | |
130879f1 CH |
126 | return ret; |
127 | } | |
128 | ||
7589ad67 CH |
129 | static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, |
130 | struct iov_iter *iter, gfp_t gfp_mask) | |
130879f1 CH |
131 | { |
132 | struct bio_map_data *bmd; | |
133 | struct page *page; | |
393bb12e | 134 | struct bio *bio; |
130879f1 CH |
135 | int i = 0, ret; |
136 | int nr_pages; | |
137 | unsigned int len = iter->count; | |
138 | unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; | |
139 | ||
140 | bmd = bio_alloc_map_data(iter, gfp_mask); | |
141 | if (!bmd) | |
7589ad67 | 142 | return -ENOMEM; |
130879f1 CH |
143 | |
144 | /* | |
145 | * We need to do a deep copy of the iov_iter including the iovecs. | |
146 | * The caller provided iov might point to an on-stack or otherwise | |
147 | * shortlived one. | |
148 | */ | |
f3256075 | 149 | bmd->is_our_pages = !map_data; |
03859717 | 150 | bmd->is_null_mapped = (map_data && map_data->null_mapped); |
130879f1 | 151 | |
5f7136db | 152 | nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); |
130879f1 CH |
153 | |
154 | ret = -ENOMEM; | |
066ff571 | 155 | bio = bio_kmalloc(nr_pages, gfp_mask); |
130879f1 CH |
156 | if (!bio) |
157 | goto out_bmd; | |
066ff571 | 158 | bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, req_op(rq)); |
130879f1 CH |
159 | |
160 | if (map_data) { | |
161 | nr_pages = 1 << map_data->page_order; | |
162 | i = map_data->offset / PAGE_SIZE; | |
163 | } | |
164 | while (len) { | |
165 | unsigned int bytes = PAGE_SIZE; | |
166 | ||
167 | bytes -= offset; | |
168 | ||
169 | if (bytes > len) | |
170 | bytes = len; | |
171 | ||
172 | if (map_data) { | |
173 | if (i == map_data->nr_entries * nr_pages) { | |
174 | ret = -ENOMEM; | |
7589ad67 | 175 | goto cleanup; |
130879f1 CH |
176 | } |
177 | ||
178 | page = map_data->pages[i / nr_pages]; | |
179 | page += (i % nr_pages); | |
180 | ||
181 | i++; | |
182 | } else { | |
ce288e05 | 183 | page = alloc_page(GFP_NOIO | gfp_mask); |
130879f1 CH |
184 | if (!page) { |
185 | ret = -ENOMEM; | |
7589ad67 | 186 | goto cleanup; |
130879f1 CH |
187 | } |
188 | } | |
189 | ||
7589ad67 | 190 | if (bio_add_pc_page(rq->q, bio, page, bytes, offset) < bytes) { |
130879f1 CH |
191 | if (!map_data) |
192 | __free_page(page); | |
193 | break; | |
194 | } | |
195 | ||
196 | len -= bytes; | |
197 | offset = 0; | |
198 | } | |
199 | ||
130879f1 CH |
200 | if (map_data) |
201 | map_data->offset += bio->bi_iter.bi_size; | |
202 | ||
203 | /* | |
204 | * success | |
205 | */ | |
206 | if ((iov_iter_rw(iter) == WRITE && | |
207 | (!map_data || !map_data->null_mapped)) || | |
208 | (map_data && map_data->from_user)) { | |
209 | ret = bio_copy_from_iter(bio, iter); | |
210 | if (ret) | |
211 | goto cleanup; | |
212 | } else { | |
213 | if (bmd->is_our_pages) | |
214 | zero_fill_bio(bio); | |
215 | iov_iter_advance(iter, bio->bi_iter.bi_size); | |
216 | } | |
217 | ||
218 | bio->bi_private = bmd; | |
7589ad67 | 219 | |
393bb12e | 220 | ret = blk_rq_append_bio(rq, bio); |
7589ad67 CH |
221 | if (ret) |
222 | goto cleanup; | |
7589ad67 | 223 | return 0; |
130879f1 CH |
224 | cleanup: |
225 | if (!map_data) | |
226 | bio_free_pages(bio); | |
066ff571 CH |
227 | bio_uninit(bio); |
228 | kfree(bio); | |
130879f1 CH |
229 | out_bmd: |
230 | kfree(bmd); | |
7589ad67 | 231 | return ret; |
130879f1 CH |
232 | } |
233 | ||
7589ad67 CH |
234 | static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, |
235 | gfp_t gfp_mask) | |
130879f1 | 236 | { |
7589ad67 | 237 | unsigned int max_sectors = queue_max_hw_sectors(rq->q); |
066ff571 | 238 | unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS); |
393bb12e | 239 | struct bio *bio; |
130879f1 | 240 | int ret; |
7589ad67 | 241 | int j; |
130879f1 CH |
242 | |
243 | if (!iov_iter_count(iter)) | |
7589ad67 | 244 | return -EINVAL; |
130879f1 | 245 | |
066ff571 | 246 | bio = bio_kmalloc(nr_vecs, gfp_mask); |
130879f1 | 247 | if (!bio) |
7589ad67 | 248 | return -ENOMEM; |
066ff571 | 249 | bio_init(bio, NULL, bio->bi_inline_vecs, nr_vecs, req_op(rq)); |
130879f1 CH |
250 | |
251 | while (iov_iter_count(iter)) { | |
252 | struct page **pages; | |
253 | ssize_t bytes; | |
254 | size_t offs, added = 0; | |
255 | int npages; | |
256 | ||
480cb846 | 257 | bytes = iov_iter_get_pages_alloc2(iter, &pages, LONG_MAX, &offs); |
130879f1 CH |
258 | if (unlikely(bytes <= 0)) { |
259 | ret = bytes ? bytes : -EFAULT; | |
260 | goto out_unmap; | |
261 | } | |
262 | ||
263 | npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE); | |
264 | ||
7ab89db9 | 265 | if (unlikely(offs & queue_dma_alignment(rq->q))) |
130879f1 | 266 | j = 0; |
7ab89db9 | 267 | else { |
130879f1 CH |
268 | for (j = 0; j < npages; j++) { |
269 | struct page *page = pages[j]; | |
270 | unsigned int n = PAGE_SIZE - offs; | |
271 | bool same_page = false; | |
272 | ||
273 | if (n > bytes) | |
274 | n = bytes; | |
275 | ||
7589ad67 | 276 | if (!bio_add_hw_page(rq->q, bio, page, n, offs, |
e4581105 | 277 | max_sectors, &same_page)) { |
130879f1 CH |
278 | if (same_page) |
279 | put_page(page); | |
280 | break; | |
281 | } | |
282 | ||
283 | added += n; | |
284 | bytes -= n; | |
285 | offs = 0; | |
286 | } | |
130879f1 CH |
287 | } |
288 | /* | |
289 | * release the pages we didn't map into the bio, if any | |
290 | */ | |
291 | while (j < npages) | |
292 | put_page(pages[j++]); | |
293 | kvfree(pages); | |
294 | /* couldn't stuff something into bio? */ | |
480cb846 AV |
295 | if (bytes) { |
296 | iov_iter_revert(iter, bytes); | |
130879f1 | 297 | break; |
480cb846 | 298 | } |
130879f1 CH |
299 | } |
300 | ||
393bb12e | 301 | ret = blk_rq_append_bio(rq, bio); |
7589ad67 | 302 | if (ret) |
393bb12e | 303 | goto out_unmap; |
7589ad67 CH |
304 | return 0; |
305 | ||
130879f1 CH |
306 | out_unmap: |
307 | bio_release_pages(bio, false); | |
066ff571 CH |
308 | bio_uninit(bio); |
309 | kfree(bio); | |
7589ad67 | 310 | return ret; |
130879f1 CH |
311 | } |
312 | ||
130879f1 CH |
313 | static void bio_invalidate_vmalloc_pages(struct bio *bio) |
314 | { | |
f358afc5 | 315 | #ifdef ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE |
130879f1 CH |
316 | if (bio->bi_private && !op_is_write(bio_op(bio))) { |
317 | unsigned long i, len = 0; | |
318 | ||
319 | for (i = 0; i < bio->bi_vcnt; i++) | |
320 | len += bio->bi_io_vec[i].bv_len; | |
321 | invalidate_kernel_vmap_range(bio->bi_private, len); | |
322 | } | |
323 | #endif | |
324 | } | |
325 | ||
326 | static void bio_map_kern_endio(struct bio *bio) | |
327 | { | |
328 | bio_invalidate_vmalloc_pages(bio); | |
066ff571 CH |
329 | bio_uninit(bio); |
330 | kfree(bio); | |
130879f1 CH |
331 | } |
332 | ||
333 | /** | |
334 | * bio_map_kern - map kernel address into bio | |
335 | * @q: the struct request_queue for the bio | |
336 | * @data: pointer to buffer to map | |
337 | * @len: length in bytes | |
338 | * @gfp_mask: allocation flags for bio allocation | |
339 | * | |
340 | * Map the kernel address into a bio suitable for io to a block | |
341 | * device. Returns an error pointer in case of error. | |
342 | */ | |
343 | static struct bio *bio_map_kern(struct request_queue *q, void *data, | |
344 | unsigned int len, gfp_t gfp_mask) | |
345 | { | |
346 | unsigned long kaddr = (unsigned long)data; | |
347 | unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; | |
348 | unsigned long start = kaddr >> PAGE_SHIFT; | |
349 | const int nr_pages = end - start; | |
350 | bool is_vmalloc = is_vmalloc_addr(data); | |
351 | struct page *page; | |
352 | int offset, i; | |
353 | struct bio *bio; | |
354 | ||
066ff571 | 355 | bio = bio_kmalloc(nr_pages, gfp_mask); |
130879f1 CH |
356 | if (!bio) |
357 | return ERR_PTR(-ENOMEM); | |
066ff571 | 358 | bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); |
130879f1 CH |
359 | |
360 | if (is_vmalloc) { | |
361 | flush_kernel_vmap_range(data, len); | |
362 | bio->bi_private = data; | |
363 | } | |
364 | ||
365 | offset = offset_in_page(kaddr); | |
366 | for (i = 0; i < nr_pages; i++) { | |
367 | unsigned int bytes = PAGE_SIZE - offset; | |
368 | ||
369 | if (len <= 0) | |
370 | break; | |
371 | ||
372 | if (bytes > len) | |
373 | bytes = len; | |
374 | ||
375 | if (!is_vmalloc) | |
376 | page = virt_to_page(data); | |
377 | else | |
378 | page = vmalloc_to_page(data); | |
379 | if (bio_add_pc_page(q, bio, page, bytes, | |
380 | offset) < bytes) { | |
381 | /* we don't support partial mappings */ | |
066ff571 CH |
382 | bio_uninit(bio); |
383 | kfree(bio); | |
130879f1 CH |
384 | return ERR_PTR(-EINVAL); |
385 | } | |
386 | ||
387 | data += bytes; | |
388 | len -= bytes; | |
389 | offset = 0; | |
390 | } | |
391 | ||
392 | bio->bi_end_io = bio_map_kern_endio; | |
393 | return bio; | |
394 | } | |
395 | ||
396 | static void bio_copy_kern_endio(struct bio *bio) | |
397 | { | |
398 | bio_free_pages(bio); | |
066ff571 CH |
399 | bio_uninit(bio); |
400 | kfree(bio); | |
130879f1 CH |
401 | } |
402 | ||
403 | static void bio_copy_kern_endio_read(struct bio *bio) | |
404 | { | |
405 | char *p = bio->bi_private; | |
406 | struct bio_vec *bvec; | |
407 | struct bvec_iter_all iter_all; | |
408 | ||
409 | bio_for_each_segment_all(bvec, bio, iter_all) { | |
d24920e2 | 410 | memcpy_from_bvec(p, bvec); |
130879f1 CH |
411 | p += bvec->bv_len; |
412 | } | |
413 | ||
414 | bio_copy_kern_endio(bio); | |
415 | } | |
416 | ||
417 | /** | |
418 | * bio_copy_kern - copy kernel address into bio | |
419 | * @q: the struct request_queue for the bio | |
420 | * @data: pointer to buffer to copy | |
421 | * @len: length in bytes | |
422 | * @gfp_mask: allocation flags for bio and page allocation | |
423 | * @reading: data direction is READ | |
424 | * | |
425 | * copy the kernel address into a bio suitable for io to a block | |
426 | * device. Returns an error pointer in case of error. | |
427 | */ | |
428 | static struct bio *bio_copy_kern(struct request_queue *q, void *data, | |
429 | unsigned int len, gfp_t gfp_mask, int reading) | |
430 | { | |
431 | unsigned long kaddr = (unsigned long)data; | |
432 | unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; | |
433 | unsigned long start = kaddr >> PAGE_SHIFT; | |
434 | struct bio *bio; | |
435 | void *p = data; | |
436 | int nr_pages = 0; | |
437 | ||
438 | /* | |
439 | * Overflow, abort | |
440 | */ | |
441 | if (end < start) | |
442 | return ERR_PTR(-EINVAL); | |
443 | ||
444 | nr_pages = end - start; | |
066ff571 | 445 | bio = bio_kmalloc(nr_pages, gfp_mask); |
130879f1 CH |
446 | if (!bio) |
447 | return ERR_PTR(-ENOMEM); | |
066ff571 | 448 | bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); |
130879f1 CH |
449 | |
450 | while (len) { | |
451 | struct page *page; | |
452 | unsigned int bytes = PAGE_SIZE; | |
453 | ||
454 | if (bytes > len) | |
455 | bytes = len; | |
456 | ||
cc8f7fe1 | 457 | page = alloc_page(GFP_NOIO | __GFP_ZERO | gfp_mask); |
130879f1 CH |
458 | if (!page) |
459 | goto cleanup; | |
460 | ||
461 | if (!reading) | |
462 | memcpy(page_address(page), p, bytes); | |
463 | ||
464 | if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes) | |
465 | break; | |
466 | ||
467 | len -= bytes; | |
468 | p += bytes; | |
469 | } | |
470 | ||
471 | if (reading) { | |
472 | bio->bi_end_io = bio_copy_kern_endio_read; | |
473 | bio->bi_private = data; | |
474 | } else { | |
475 | bio->bi_end_io = bio_copy_kern_endio; | |
476 | } | |
477 | ||
478 | return bio; | |
479 | ||
480 | cleanup: | |
481 | bio_free_pages(bio); | |
066ff571 CH |
482 | bio_uninit(bio); |
483 | kfree(bio); | |
130879f1 CH |
484 | return ERR_PTR(-ENOMEM); |
485 | } | |
486 | ||
98d61d5b | 487 | /* |
0abc2a10 JA |
488 | * Append a bio to a passthrough request. Only works if the bio can be merged |
489 | * into the request based on the driver constraints. | |
98d61d5b | 490 | */ |
393bb12e | 491 | int blk_rq_append_bio(struct request *rq, struct bio *bio) |
86db1e29 | 492 | { |
14ccb66b CH |
493 | struct bvec_iter iter; |
494 | struct bio_vec bv; | |
495 | unsigned int nr_segs = 0; | |
0abc2a10 | 496 | |
393bb12e | 497 | bio_for_each_bvec(bv, bio, iter) |
14ccb66b CH |
498 | nr_segs++; |
499 | ||
98d61d5b | 500 | if (!rq->bio) { |
393bb12e | 501 | blk_rq_bio_prep(rq, bio, nr_segs); |
98d61d5b | 502 | } else { |
393bb12e | 503 | if (!ll_back_merge_fn(rq, bio, nr_segs)) |
98d61d5b | 504 | return -EINVAL; |
393bb12e CH |
505 | rq->biotail->bi_next = bio; |
506 | rq->biotail = bio; | |
507 | rq->__data_len += (bio)->bi_iter.bi_size; | |
508 | bio_crypt_free_ctx(bio); | |
86db1e29 | 509 | } |
98d61d5b | 510 | |
86db1e29 JA |
511 | return 0; |
512 | } | |
98d61d5b | 513 | EXPORT_SYMBOL(blk_rq_append_bio); |
86db1e29 | 514 | |
86db1e29 | 515 | /** |
aebf526b | 516 | * blk_rq_map_user_iov - map user data to a request, for passthrough requests |
86db1e29 JA |
517 | * @q: request queue where request should be inserted |
518 | * @rq: request to map data to | |
152e283f | 519 | * @map_data: pointer to the rq_map_data holding pages (if necessary) |
26e49cfc | 520 | * @iter: iovec iterator |
a3bce90e | 521 | * @gfp_mask: memory allocation flags |
86db1e29 JA |
522 | * |
523 | * Description: | |
710027a4 | 524 | * Data will be mapped directly for zero copy I/O, if possible. Otherwise |
86db1e29 JA |
525 | * a kernel bounce buffer is used. |
526 | * | |
710027a4 | 527 | * A matching blk_rq_unmap_user() must be issued at the end of I/O, while |
86db1e29 | 528 | * still in process context. |
86db1e29 JA |
529 | */ |
530 | int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, | |
26e49cfc KO |
531 | struct rq_map_data *map_data, |
532 | const struct iov_iter *iter, gfp_t gfp_mask) | |
86db1e29 | 533 | { |
357f435d AV |
534 | bool copy = false; |
535 | unsigned long align = q->dma_pad_mask | queue_dma_alignment(q); | |
4d6af73d CH |
536 | struct bio *bio = NULL; |
537 | struct iov_iter i; | |
69e0927b | 538 | int ret = -EINVAL; |
86db1e29 | 539 | |
a0ac402c LT |
540 | if (!iter_is_iovec(iter)) |
541 | goto fail; | |
542 | ||
357f435d AV |
543 | if (map_data) |
544 | copy = true; | |
393bb12e CH |
545 | else if (blk_queue_may_bounce(q)) |
546 | copy = true; | |
357f435d AV |
547 | else if (iov_iter_alignment(iter) & align) |
548 | copy = true; | |
549 | else if (queue_virt_boundary(q)) | |
550 | copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter); | |
afdc1a78 | 551 | |
4d6af73d CH |
552 | i = *iter; |
553 | do { | |
7589ad67 CH |
554 | if (copy) |
555 | ret = bio_copy_user_iov(rq, map_data, &i, gfp_mask); | |
556 | else | |
557 | ret = bio_map_user_iov(rq, &i, gfp_mask); | |
4d6af73d CH |
558 | if (ret) |
559 | goto unmap_rq; | |
560 | if (!bio) | |
561 | bio = rq->bio; | |
562 | } while (iov_iter_count(&i)); | |
86db1e29 | 563 | |
86db1e29 | 564 | return 0; |
4d6af73d CH |
565 | |
566 | unmap_rq: | |
3b7995a9 | 567 | blk_rq_unmap_user(bio); |
a0ac402c | 568 | fail: |
4d6af73d | 569 | rq->bio = NULL; |
69e0927b | 570 | return ret; |
86db1e29 | 571 | } |
152e283f | 572 | EXPORT_SYMBOL(blk_rq_map_user_iov); |
86db1e29 | 573 | |
ddad8dd0 CH |
574 | int blk_rq_map_user(struct request_queue *q, struct request *rq, |
575 | struct rq_map_data *map_data, void __user *ubuf, | |
576 | unsigned long len, gfp_t gfp_mask) | |
577 | { | |
26e49cfc KO |
578 | struct iovec iov; |
579 | struct iov_iter i; | |
8f7e885a | 580 | int ret = import_single_range(rq_data_dir(rq), ubuf, len, &iov, &i); |
ddad8dd0 | 581 | |
8f7e885a AV |
582 | if (unlikely(ret < 0)) |
583 | return ret; | |
ddad8dd0 | 584 | |
26e49cfc | 585 | return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask); |
ddad8dd0 CH |
586 | } |
587 | EXPORT_SYMBOL(blk_rq_map_user); | |
588 | ||
86db1e29 JA |
589 | /** |
590 | * blk_rq_unmap_user - unmap a request with user data | |
591 | * @bio: start of bio list | |
592 | * | |
593 | * Description: | |
594 | * Unmap a rq previously mapped by blk_rq_map_user(). The caller must | |
595 | * supply the original rq->bio from the blk_rq_map_user() return, since | |
710027a4 | 596 | * the I/O completion may have changed rq->bio. |
86db1e29 JA |
597 | */ |
598 | int blk_rq_unmap_user(struct bio *bio) | |
599 | { | |
393bb12e | 600 | struct bio *next_bio; |
86db1e29 JA |
601 | int ret = 0, ret2; |
602 | ||
603 | while (bio) { | |
3310eeba | 604 | if (bio->bi_private) { |
393bb12e | 605 | ret2 = bio_uncopy_user(bio); |
7b63c052 CH |
606 | if (ret2 && !ret) |
607 | ret = ret2; | |
3310eeba | 608 | } else { |
393bb12e | 609 | bio_release_pages(bio, bio_data_dir(bio) == READ); |
7b63c052 | 610 | } |
86db1e29 | 611 | |
393bb12e | 612 | next_bio = bio; |
86db1e29 | 613 | bio = bio->bi_next; |
066ff571 CH |
614 | bio_uninit(next_bio); |
615 | kfree(next_bio); | |
86db1e29 JA |
616 | } |
617 | ||
618 | return ret; | |
619 | } | |
86db1e29 JA |
620 | EXPORT_SYMBOL(blk_rq_unmap_user); |
621 | ||
622 | /** | |
aebf526b | 623 | * blk_rq_map_kern - map kernel data to a request, for passthrough requests |
86db1e29 JA |
624 | * @q: request queue where request should be inserted |
625 | * @rq: request to fill | |
626 | * @kbuf: the kernel buffer | |
627 | * @len: length of user data | |
628 | * @gfp_mask: memory allocation flags | |
68154e90 FT |
629 | * |
630 | * Description: | |
631 | * Data will be mapped directly if possible. Otherwise a bounce | |
e227867f | 632 | * buffer is used. Can be called multiple times to append multiple |
3a5a3927 | 633 | * buffers. |
86db1e29 JA |
634 | */ |
635 | int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, | |
636 | unsigned int len, gfp_t gfp_mask) | |
637 | { | |
68154e90 | 638 | int reading = rq_data_dir(rq) == READ; |
14417799 | 639 | unsigned long addr = (unsigned long) kbuf; |
393bb12e | 640 | struct bio *bio; |
3a5a3927 | 641 | int ret; |
86db1e29 | 642 | |
ae03bf63 | 643 | if (len > (queue_max_hw_sectors(q) << 9)) |
86db1e29 JA |
644 | return -EINVAL; |
645 | if (!len || !kbuf) | |
646 | return -EINVAL; | |
647 | ||
393bb12e CH |
648 | if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf) || |
649 | blk_queue_may_bounce(q)) | |
68154e90 FT |
650 | bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); |
651 | else | |
652 | bio = bio_map_kern(q, kbuf, len, gfp_mask); | |
653 | ||
86db1e29 JA |
654 | if (IS_ERR(bio)) |
655 | return PTR_ERR(bio); | |
656 | ||
aebf526b CH |
657 | bio->bi_opf &= ~REQ_OP_MASK; |
658 | bio->bi_opf |= req_op(rq); | |
86db1e29 | 659 | |
393bb12e | 660 | ret = blk_rq_append_bio(rq, bio); |
066ff571 CH |
661 | if (unlikely(ret)) { |
662 | bio_uninit(bio); | |
663 | kfree(bio); | |
664 | } | |
393bb12e | 665 | return ret; |
86db1e29 | 666 | } |
86db1e29 | 667 | EXPORT_SYMBOL(blk_rq_map_kern); |