1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/export.h>
3 #include <linux/bvec.h>
4 #include <linux/fault-inject-usercopy.h>
6 #include <linux/pagemap.h>
7 #include <linux/highmem.h>
8 #include <linux/slab.h>
9 #include <linux/vmalloc.h>
10 #include <linux/splice.h>
11 #include <linux/compat.h>
12 #include <linux/scatterlist.h>
13 #include <linux/instrumented.h>
14 #include <linux/iov_iter.h>
16 static __always_inline
17 size_t copy_to_user_iter(void __user
*iter_to
, size_t progress
,
18 size_t len
, void *from
, void *priv2
)
20 if (should_fail_usercopy())
22 if (access_ok(iter_to
, len
)) {
24 instrument_copy_to_user(iter_to
, from
, len
);
25 len
= raw_copy_to_user(iter_to
, from
, len
);
30 static __always_inline
31 size_t copy_to_user_iter_nofault(void __user
*iter_to
, size_t progress
,
32 size_t len
, void *from
, void *priv2
)
36 if (should_fail_usercopy())
40 res
= copy_to_user_nofault(iter_to
, from
, len
);
41 return res
< 0 ? len
: res
;
44 static __always_inline
45 size_t copy_from_user_iter(void __user
*iter_from
, size_t progress
,
46 size_t len
, void *to
, void *priv2
)
50 if (should_fail_usercopy())
52 if (access_ok(iter_from
, len
)) {
54 instrument_copy_from_user_before(to
, iter_from
, len
);
55 res
= raw_copy_from_user(to
, iter_from
, len
);
56 instrument_copy_from_user_after(to
, iter_from
, len
, res
);
61 static __always_inline
62 size_t memcpy_to_iter(void *iter_to
, size_t progress
,
63 size_t len
, void *from
, void *priv2
)
65 memcpy(iter_to
, from
+ progress
, len
);
69 static __always_inline
70 size_t memcpy_from_iter(void *iter_from
, size_t progress
,
71 size_t len
, void *to
, void *priv2
)
73 memcpy(to
+ progress
, iter_from
, len
);
78 * fault_in_iov_iter_readable - fault in iov iterator for reading
80 * @size: maximum length
82 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
83 * @size. For each iovec, fault in each page that constitutes the iovec.
85 * Returns the number of bytes not faulted in (like copy_to_user() and
88 * Always returns 0 for non-userspace iterators.
90 size_t fault_in_iov_iter_readable(const struct iov_iter
*i
, size_t size
)
92 if (iter_is_ubuf(i
)) {
93 size_t n
= min(size
, iov_iter_count(i
));
94 n
-= fault_in_readable(i
->ubuf
+ i
->iov_offset
, n
);
96 } else if (iter_is_iovec(i
)) {
97 size_t count
= min(size
, iov_iter_count(i
));
98 const struct iovec
*p
;
102 for (p
= iter_iov(i
), skip
= i
->iov_offset
; count
; p
++, skip
= 0) {
103 size_t len
= min(count
, p
->iov_len
- skip
);
108 ret
= fault_in_readable(p
->iov_base
+ skip
, len
);
117 EXPORT_SYMBOL(fault_in_iov_iter_readable
);
120 * fault_in_iov_iter_writeable - fault in iov iterator for writing
122 * @size: maximum length
124 * Faults in the iterator using get_user_pages(), i.e., without triggering
125 * hardware page faults. This is primarily useful when we already know that
126 * some or all of the pages in @i aren't in memory.
128 * Returns the number of bytes not faulted in, like copy_to_user() and
131 * Always returns 0 for non-user-space iterators.
133 size_t fault_in_iov_iter_writeable(const struct iov_iter
*i
, size_t size
)
135 if (iter_is_ubuf(i
)) {
136 size_t n
= min(size
, iov_iter_count(i
));
137 n
-= fault_in_safe_writeable(i
->ubuf
+ i
->iov_offset
, n
);
139 } else if (iter_is_iovec(i
)) {
140 size_t count
= min(size
, iov_iter_count(i
));
141 const struct iovec
*p
;
145 for (p
= iter_iov(i
), skip
= i
->iov_offset
; count
; p
++, skip
= 0) {
146 size_t len
= min(count
, p
->iov_len
- skip
);
151 ret
= fault_in_safe_writeable(p
->iov_base
+ skip
, len
);
160 EXPORT_SYMBOL(fault_in_iov_iter_writeable
);
162 void iov_iter_init(struct iov_iter
*i
, unsigned int direction
,
163 const struct iovec
*iov
, unsigned long nr_segs
,
166 WARN_ON(direction
& ~(READ
| WRITE
));
167 *i
= (struct iov_iter
) {
168 .iter_type
= ITER_IOVEC
,
171 .data_source
= direction
,
178 EXPORT_SYMBOL(iov_iter_init
);
180 size_t _copy_to_iter(const void *addr
, size_t bytes
, struct iov_iter
*i
)
182 if (WARN_ON_ONCE(i
->data_source
))
184 if (user_backed_iter(i
))
186 return iterate_and_advance(i
, bytes
, (void *)addr
,
187 copy_to_user_iter
, memcpy_to_iter
);
189 EXPORT_SYMBOL(_copy_to_iter
);
191 #ifdef CONFIG_ARCH_HAS_COPY_MC
192 static __always_inline
193 size_t copy_to_user_iter_mc(void __user
*iter_to
, size_t progress
,
194 size_t len
, void *from
, void *priv2
)
196 if (access_ok(iter_to
, len
)) {
198 instrument_copy_to_user(iter_to
, from
, len
);
199 len
= copy_mc_to_user(iter_to
, from
, len
);
204 static __always_inline
205 size_t memcpy_to_iter_mc(void *iter_to
, size_t progress
,
206 size_t len
, void *from
, void *priv2
)
208 return copy_mc_to_kernel(iter_to
, from
+ progress
, len
);
212 * _copy_mc_to_iter - copy to iter with source memory error exception handling
213 * @addr: source kernel address
214 * @bytes: total transfer length
215 * @i: destination iterator
217 * The pmem driver deploys this for the dax operation
218 * (dax_copy_to_iter()) for dax reads (bypass page-cache and the
219 * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
220 * successfully copied.
222 * The main differences between this and typical _copy_to_iter().
224 * * Typical tail/residue handling after a fault retries the copy
225 * byte-by-byte until the fault happens again. Re-triggering machine
226 * checks is potentially fatal so the implementation uses source
227 * alignment and poison alignment assumptions to avoid re-triggering
228 * hardware exceptions.
230 * * ITER_KVEC and ITER_BVEC can return short copies. Compare to
231 * copy_to_iter() where only ITER_IOVEC attempts might return a short copy.
233 * Return: number of bytes copied (may be %0)
235 size_t _copy_mc_to_iter(const void *addr
, size_t bytes
, struct iov_iter
*i
)
237 if (WARN_ON_ONCE(i
->data_source
))
239 if (user_backed_iter(i
))
241 return iterate_and_advance(i
, bytes
, (void *)addr
,
242 copy_to_user_iter_mc
, memcpy_to_iter_mc
);
244 EXPORT_SYMBOL_GPL(_copy_mc_to_iter
);
245 #endif /* CONFIG_ARCH_HAS_COPY_MC */
247 static __always_inline
248 size_t memcpy_from_iter_mc(void *iter_from
, size_t progress
,
249 size_t len
, void *to
, void *priv2
)
251 return copy_mc_to_kernel(to
+ progress
, iter_from
, len
);
254 static size_t __copy_from_iter_mc(void *addr
, size_t bytes
, struct iov_iter
*i
)
256 if (unlikely(i
->count
< bytes
))
258 if (unlikely(!bytes
))
260 return iterate_bvec(i
, bytes
, addr
, NULL
, memcpy_from_iter_mc
);
263 static __always_inline
264 size_t __copy_from_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
266 if (unlikely(iov_iter_is_copy_mc(i
)))
267 return __copy_from_iter_mc(addr
, bytes
, i
);
268 return iterate_and_advance(i
, bytes
, addr
,
269 copy_from_user_iter
, memcpy_from_iter
);
272 size_t _copy_from_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
274 if (WARN_ON_ONCE(!i
->data_source
))
277 if (user_backed_iter(i
))
279 return __copy_from_iter(addr
, bytes
, i
);
281 EXPORT_SYMBOL(_copy_from_iter
);
283 static __always_inline
284 size_t copy_from_user_iter_nocache(void __user
*iter_from
, size_t progress
,
285 size_t len
, void *to
, void *priv2
)
287 return __copy_from_user_inatomic_nocache(to
+ progress
, iter_from
, len
);
290 size_t _copy_from_iter_nocache(void *addr
, size_t bytes
, struct iov_iter
*i
)
292 if (WARN_ON_ONCE(!i
->data_source
))
295 return iterate_and_advance(i
, bytes
, addr
,
296 copy_from_user_iter_nocache
,
299 EXPORT_SYMBOL(_copy_from_iter_nocache
);
301 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
302 static __always_inline
303 size_t copy_from_user_iter_flushcache(void __user
*iter_from
, size_t progress
,
304 size_t len
, void *to
, void *priv2
)
306 return __copy_from_user_flushcache(to
+ progress
, iter_from
, len
);
309 static __always_inline
310 size_t memcpy_from_iter_flushcache(void *iter_from
, size_t progress
,
311 size_t len
, void *to
, void *priv2
)
313 memcpy_flushcache(to
+ progress
, iter_from
, len
);
318 * _copy_from_iter_flushcache - write destination through cpu cache
319 * @addr: destination kernel address
320 * @bytes: total transfer length
321 * @i: source iterator
323 * The pmem driver arranges for filesystem-dax to use this facility via
324 * dax_copy_from_iter() for ensuring that writes to persistent memory
325 * are flushed through the CPU cache. It is differentiated from
326 * _copy_from_iter_nocache() in that guarantees all data is flushed for
327 * all iterator types. The _copy_from_iter_nocache() only attempts to
328 * bypass the cache for the ITER_IOVEC case, and on some archs may use
329 * instructions that strand dirty-data in the cache.
331 * Return: number of bytes copied (may be %0)
333 size_t _copy_from_iter_flushcache(void *addr
, size_t bytes
, struct iov_iter
*i
)
335 if (WARN_ON_ONCE(!i
->data_source
))
338 return iterate_and_advance(i
, bytes
, addr
,
339 copy_from_user_iter_flushcache
,
340 memcpy_from_iter_flushcache
);
342 EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache
);
345 static inline bool page_copy_sane(struct page
*page
, size_t offset
, size_t n
)
348 size_t v
= n
+ offset
;
351 * The general case needs to access the page order in order
352 * to compute the page size.
353 * However, we mostly deal with order-0 pages and thus can
354 * avoid a possible cache line miss for requests that fit all
357 if (n
<= v
&& v
<= PAGE_SIZE
)
360 head
= compound_head(page
);
361 v
+= (page
- head
) << PAGE_SHIFT
;
363 if (WARN_ON(n
> v
|| v
> page_size(head
)))
368 size_t copy_page_to_iter(struct page
*page
, size_t offset
, size_t bytes
,
372 if (!page_copy_sane(page
, offset
, bytes
))
374 if (WARN_ON_ONCE(i
->data_source
))
376 page
+= offset
/ PAGE_SIZE
; // first subpage
379 void *kaddr
= kmap_local_page(page
);
380 size_t n
= min(bytes
, (size_t)PAGE_SIZE
- offset
);
381 n
= _copy_to_iter(kaddr
+ offset
, n
, i
);
388 if (offset
== PAGE_SIZE
) {
395 EXPORT_SYMBOL(copy_page_to_iter
);
397 size_t copy_page_to_iter_nofault(struct page
*page
, unsigned offset
, size_t bytes
,
402 if (!page_copy_sane(page
, offset
, bytes
))
404 if (WARN_ON_ONCE(i
->data_source
))
406 page
+= offset
/ PAGE_SIZE
; // first subpage
409 void *kaddr
= kmap_local_page(page
);
410 size_t n
= min(bytes
, (size_t)PAGE_SIZE
- offset
);
412 n
= iterate_and_advance(i
, n
, kaddr
+ offset
,
413 copy_to_user_iter_nofault
,
421 if (offset
== PAGE_SIZE
) {
428 EXPORT_SYMBOL(copy_page_to_iter_nofault
);
430 size_t copy_page_from_iter(struct page
*page
, size_t offset
, size_t bytes
,
434 if (!page_copy_sane(page
, offset
, bytes
))
436 page
+= offset
/ PAGE_SIZE
; // first subpage
439 void *kaddr
= kmap_local_page(page
);
440 size_t n
= min(bytes
, (size_t)PAGE_SIZE
- offset
);
441 n
= _copy_from_iter(kaddr
+ offset
, n
, i
);
448 if (offset
== PAGE_SIZE
) {
455 EXPORT_SYMBOL(copy_page_from_iter
);
457 static __always_inline
458 size_t zero_to_user_iter(void __user
*iter_to
, size_t progress
,
459 size_t len
, void *priv
, void *priv2
)
461 return clear_user(iter_to
, len
);
464 static __always_inline
465 size_t zero_to_iter(void *iter_to
, size_t progress
,
466 size_t len
, void *priv
, void *priv2
)
468 memset(iter_to
, 0, len
);
472 size_t iov_iter_zero(size_t bytes
, struct iov_iter
*i
)
474 return iterate_and_advance(i
, bytes
, NULL
,
475 zero_to_user_iter
, zero_to_iter
);
477 EXPORT_SYMBOL(iov_iter_zero
);
479 size_t copy_page_from_iter_atomic(struct page
*page
, size_t offset
,
480 size_t bytes
, struct iov_iter
*i
)
482 size_t n
, copied
= 0;
484 if (!page_copy_sane(page
, offset
, bytes
))
486 if (WARN_ON_ONCE(!i
->data_source
))
493 if (PageHighMem(page
)) {
494 page
+= offset
/ PAGE_SIZE
;
496 n
= min_t(size_t, n
, PAGE_SIZE
- offset
);
499 p
= kmap_atomic(page
) + offset
;
500 n
= __copy_from_iter(p
, n
, i
);
504 } while (PageHighMem(page
) && copied
!= bytes
&& n
> 0);
508 EXPORT_SYMBOL(copy_page_from_iter_atomic
);
510 static void iov_iter_bvec_advance(struct iov_iter
*i
, size_t size
)
512 const struct bio_vec
*bvec
, *end
;
518 size
+= i
->iov_offset
;
520 for (bvec
= i
->bvec
, end
= bvec
+ i
->nr_segs
; bvec
< end
; bvec
++) {
521 if (likely(size
< bvec
->bv_len
))
523 size
-= bvec
->bv_len
;
525 i
->iov_offset
= size
;
526 i
->nr_segs
-= bvec
- i
->bvec
;
530 static void iov_iter_iovec_advance(struct iov_iter
*i
, size_t size
)
532 const struct iovec
*iov
, *end
;
538 size
+= i
->iov_offset
; // from beginning of current segment
539 for (iov
= iter_iov(i
), end
= iov
+ i
->nr_segs
; iov
< end
; iov
++) {
540 if (likely(size
< iov
->iov_len
))
542 size
-= iov
->iov_len
;
544 i
->iov_offset
= size
;
545 i
->nr_segs
-= iov
- iter_iov(i
);
549 void iov_iter_advance(struct iov_iter
*i
, size_t size
)
551 if (unlikely(i
->count
< size
))
553 if (likely(iter_is_ubuf(i
)) || unlikely(iov_iter_is_xarray(i
))) {
554 i
->iov_offset
+= size
;
556 } else if (likely(iter_is_iovec(i
) || iov_iter_is_kvec(i
))) {
557 /* iovec and kvec have identical layouts */
558 iov_iter_iovec_advance(i
, size
);
559 } else if (iov_iter_is_bvec(i
)) {
560 iov_iter_bvec_advance(i
, size
);
561 } else if (iov_iter_is_discard(i
)) {
565 EXPORT_SYMBOL(iov_iter_advance
);
567 void iov_iter_revert(struct iov_iter
*i
, size_t unroll
)
571 if (WARN_ON(unroll
> MAX_RW_COUNT
))
574 if (unlikely(iov_iter_is_discard(i
)))
576 if (unroll
<= i
->iov_offset
) {
577 i
->iov_offset
-= unroll
;
580 unroll
-= i
->iov_offset
;
581 if (iov_iter_is_xarray(i
) || iter_is_ubuf(i
)) {
582 BUG(); /* We should never go beyond the start of the specified
583 * range since we might then be straying into pages that
586 } else if (iov_iter_is_bvec(i
)) {
587 const struct bio_vec
*bvec
= i
->bvec
;
589 size_t n
= (--bvec
)->bv_len
;
593 i
->iov_offset
= n
- unroll
;
598 } else { /* same logics for iovec and kvec */
599 const struct iovec
*iov
= iter_iov(i
);
601 size_t n
= (--iov
)->iov_len
;
605 i
->iov_offset
= n
- unroll
;
612 EXPORT_SYMBOL(iov_iter_revert
);
615 * Return the count of just the current iov_iter segment.
617 size_t iov_iter_single_seg_count(const struct iov_iter
*i
)
619 if (i
->nr_segs
> 1) {
620 if (likely(iter_is_iovec(i
) || iov_iter_is_kvec(i
)))
621 return min(i
->count
, iter_iov(i
)->iov_len
- i
->iov_offset
);
622 if (iov_iter_is_bvec(i
))
623 return min(i
->count
, i
->bvec
->bv_len
- i
->iov_offset
);
627 EXPORT_SYMBOL(iov_iter_single_seg_count
);
629 void iov_iter_kvec(struct iov_iter
*i
, unsigned int direction
,
630 const struct kvec
*kvec
, unsigned long nr_segs
,
633 WARN_ON(direction
& ~(READ
| WRITE
));
634 *i
= (struct iov_iter
){
635 .iter_type
= ITER_KVEC
,
637 .data_source
= direction
,
644 EXPORT_SYMBOL(iov_iter_kvec
);
646 void iov_iter_bvec(struct iov_iter
*i
, unsigned int direction
,
647 const struct bio_vec
*bvec
, unsigned long nr_segs
,
650 WARN_ON(direction
& ~(READ
| WRITE
));
651 *i
= (struct iov_iter
){
652 .iter_type
= ITER_BVEC
,
654 .data_source
= direction
,
661 EXPORT_SYMBOL(iov_iter_bvec
);
664 * iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray
665 * @i: The iterator to initialise.
666 * @direction: The direction of the transfer.
667 * @xarray: The xarray to access.
668 * @start: The start file position.
669 * @count: The size of the I/O buffer in bytes.
671 * Set up an I/O iterator to either draw data out of the pages attached to an
672 * inode or to inject data into those pages. The pages *must* be prevented
673 * from evaporation, either by taking a ref on them or locking them by the
676 void iov_iter_xarray(struct iov_iter
*i
, unsigned int direction
,
677 struct xarray
*xarray
, loff_t start
, size_t count
)
679 BUG_ON(direction
& ~1);
680 *i
= (struct iov_iter
) {
681 .iter_type
= ITER_XARRAY
,
683 .data_source
= direction
,
685 .xarray_start
= start
,
690 EXPORT_SYMBOL(iov_iter_xarray
);
693 * iov_iter_discard - Initialise an I/O iterator that discards data
694 * @i: The iterator to initialise.
695 * @direction: The direction of the transfer.
696 * @count: The size of the I/O buffer in bytes.
698 * Set up an I/O iterator that just discards everything that's written to it.
699 * It's only available as a READ iterator.
701 void iov_iter_discard(struct iov_iter
*i
, unsigned int direction
, size_t count
)
703 BUG_ON(direction
!= READ
);
704 *i
= (struct iov_iter
){
705 .iter_type
= ITER_DISCARD
,
707 .data_source
= false,
712 EXPORT_SYMBOL(iov_iter_discard
);
714 static bool iov_iter_aligned_iovec(const struct iov_iter
*i
, unsigned addr_mask
,
717 size_t size
= i
->count
;
718 size_t skip
= i
->iov_offset
;
721 for (k
= 0; k
< i
->nr_segs
; k
++, skip
= 0) {
722 const struct iovec
*iov
= iter_iov(i
) + k
;
723 size_t len
= iov
->iov_len
- skip
;
729 if ((unsigned long)(iov
->iov_base
+ skip
) & addr_mask
)
739 static bool iov_iter_aligned_bvec(const struct iov_iter
*i
, unsigned addr_mask
,
742 size_t size
= i
->count
;
743 unsigned skip
= i
->iov_offset
;
746 for (k
= 0; k
< i
->nr_segs
; k
++, skip
= 0) {
747 size_t len
= i
->bvec
[k
].bv_len
- skip
;
753 if ((unsigned long)(i
->bvec
[k
].bv_offset
+ skip
) & addr_mask
)
764 * iov_iter_is_aligned() - Check if the addresses and lengths of each segments
765 * are aligned to the parameters.
767 * @i: &struct iov_iter to restore
768 * @addr_mask: bit mask to check against the iov element's addresses
769 * @len_mask: bit mask to check against the iov element's lengths
771 * Return: false if any addresses or lengths intersect with the provided masks
773 bool iov_iter_is_aligned(const struct iov_iter
*i
, unsigned addr_mask
,
776 if (likely(iter_is_ubuf(i
))) {
777 if (i
->count
& len_mask
)
779 if ((unsigned long)(i
->ubuf
+ i
->iov_offset
) & addr_mask
)
784 if (likely(iter_is_iovec(i
) || iov_iter_is_kvec(i
)))
785 return iov_iter_aligned_iovec(i
, addr_mask
, len_mask
);
787 if (iov_iter_is_bvec(i
))
788 return iov_iter_aligned_bvec(i
, addr_mask
, len_mask
);
790 if (iov_iter_is_xarray(i
)) {
791 if (i
->count
& len_mask
)
793 if ((i
->xarray_start
+ i
->iov_offset
) & addr_mask
)
799 EXPORT_SYMBOL_GPL(iov_iter_is_aligned
);
801 static unsigned long iov_iter_alignment_iovec(const struct iov_iter
*i
)
803 unsigned long res
= 0;
804 size_t size
= i
->count
;
805 size_t skip
= i
->iov_offset
;
808 for (k
= 0; k
< i
->nr_segs
; k
++, skip
= 0) {
809 const struct iovec
*iov
= iter_iov(i
) + k
;
810 size_t len
= iov
->iov_len
- skip
;
812 res
|= (unsigned long)iov
->iov_base
+ skip
;
824 static unsigned long iov_iter_alignment_bvec(const struct iov_iter
*i
)
827 size_t size
= i
->count
;
828 unsigned skip
= i
->iov_offset
;
831 for (k
= 0; k
< i
->nr_segs
; k
++, skip
= 0) {
832 size_t len
= i
->bvec
[k
].bv_len
- skip
;
833 res
|= (unsigned long)i
->bvec
[k
].bv_offset
+ skip
;
844 unsigned long iov_iter_alignment(const struct iov_iter
*i
)
846 if (likely(iter_is_ubuf(i
))) {
847 size_t size
= i
->count
;
849 return ((unsigned long)i
->ubuf
+ i
->iov_offset
) | size
;
853 /* iovec and kvec have identical layouts */
854 if (likely(iter_is_iovec(i
) || iov_iter_is_kvec(i
)))
855 return iov_iter_alignment_iovec(i
);
857 if (iov_iter_is_bvec(i
))
858 return iov_iter_alignment_bvec(i
);
860 if (iov_iter_is_xarray(i
))
861 return (i
->xarray_start
+ i
->iov_offset
) | i
->count
;
865 EXPORT_SYMBOL(iov_iter_alignment
);
867 unsigned long iov_iter_gap_alignment(const struct iov_iter
*i
)
869 unsigned long res
= 0;
871 size_t size
= i
->count
;
877 if (WARN_ON(!iter_is_iovec(i
)))
880 for (k
= 0; k
< i
->nr_segs
; k
++) {
881 const struct iovec
*iov
= iter_iov(i
) + k
;
883 unsigned long base
= (unsigned long)iov
->iov_base
;
884 if (v
) // if not the first one
885 res
|= base
| v
; // this start | previous end
886 v
= base
+ iov
->iov_len
;
887 if (size
<= iov
->iov_len
)
889 size
-= iov
->iov_len
;
894 EXPORT_SYMBOL(iov_iter_gap_alignment
);
896 static int want_pages_array(struct page
***res
, size_t size
,
897 size_t start
, unsigned int maxpages
)
899 unsigned int count
= DIV_ROUND_UP(size
+ start
, PAGE_SIZE
);
901 if (count
> maxpages
)
903 WARN_ON(!count
); // caller should've prevented that
905 *res
= kvmalloc_array(count
, sizeof(struct page
*), GFP_KERNEL
);
912 static ssize_t
iter_xarray_populate_pages(struct page
**pages
, struct xarray
*xa
,
913 pgoff_t index
, unsigned int nr_pages
)
915 XA_STATE(xas
, xa
, index
);
917 unsigned int ret
= 0;
920 for (page
= xas_load(&xas
); page
; page
= xas_next(&xas
)) {
921 if (xas_retry(&xas
, page
))
924 /* Has the page moved or been split? */
925 if (unlikely(page
!= xas_reload(&xas
))) {
930 pages
[ret
] = find_subpage(page
, xas
.xa_index
);
931 get_page(pages
[ret
]);
932 if (++ret
== nr_pages
)
939 static ssize_t
iter_xarray_get_pages(struct iov_iter
*i
,
940 struct page
***pages
, size_t maxsize
,
941 unsigned maxpages
, size_t *_start_offset
)
943 unsigned nr
, offset
, count
;
947 pos
= i
->xarray_start
+ i
->iov_offset
;
948 index
= pos
>> PAGE_SHIFT
;
949 offset
= pos
& ~PAGE_MASK
;
950 *_start_offset
= offset
;
952 count
= want_pages_array(pages
, maxsize
, offset
, maxpages
);
955 nr
= iter_xarray_populate_pages(*pages
, i
->xarray
, index
, count
);
959 maxsize
= min_t(size_t, nr
* PAGE_SIZE
- offset
, maxsize
);
960 i
->iov_offset
+= maxsize
;
965 /* must be done on non-empty ITER_UBUF or ITER_IOVEC one */
966 static unsigned long first_iovec_segment(const struct iov_iter
*i
, size_t *size
)
972 return (unsigned long)i
->ubuf
+ i
->iov_offset
;
974 for (k
= 0, skip
= i
->iov_offset
; k
< i
->nr_segs
; k
++, skip
= 0) {
975 const struct iovec
*iov
= iter_iov(i
) + k
;
976 size_t len
= iov
->iov_len
- skip
;
982 return (unsigned long)iov
->iov_base
+ skip
;
984 BUG(); // if it had been empty, we wouldn't get called
987 /* must be done on non-empty ITER_BVEC one */
988 static struct page
*first_bvec_segment(const struct iov_iter
*i
,
989 size_t *size
, size_t *start
)
992 size_t skip
= i
->iov_offset
, len
;
994 len
= i
->bvec
->bv_len
- skip
;
997 skip
+= i
->bvec
->bv_offset
;
998 page
= i
->bvec
->bv_page
+ skip
/ PAGE_SIZE
;
999 *start
= skip
% PAGE_SIZE
;
1003 static ssize_t
__iov_iter_get_pages_alloc(struct iov_iter
*i
,
1004 struct page
***pages
, size_t maxsize
,
1005 unsigned int maxpages
, size_t *start
)
1007 unsigned int n
, gup_flags
= 0;
1009 if (maxsize
> i
->count
)
1013 if (maxsize
> MAX_RW_COUNT
)
1014 maxsize
= MAX_RW_COUNT
;
1016 if (likely(user_backed_iter(i
))) {
1020 if (iov_iter_rw(i
) != WRITE
)
1021 gup_flags
|= FOLL_WRITE
;
1023 gup_flags
|= FOLL_NOFAULT
;
1025 addr
= first_iovec_segment(i
, &maxsize
);
1026 *start
= addr
% PAGE_SIZE
;
1028 n
= want_pages_array(pages
, maxsize
, *start
, maxpages
);
1031 res
= get_user_pages_fast(addr
, n
, gup_flags
, *pages
);
1032 if (unlikely(res
<= 0))
1034 maxsize
= min_t(size_t, maxsize
, res
* PAGE_SIZE
- *start
);
1035 iov_iter_advance(i
, maxsize
);
1038 if (iov_iter_is_bvec(i
)) {
1042 page
= first_bvec_segment(i
, &maxsize
, start
);
1043 n
= want_pages_array(pages
, maxsize
, *start
, maxpages
);
1047 for (int k
= 0; k
< n
; k
++)
1048 get_page(p
[k
] = page
+ k
);
1049 maxsize
= min_t(size_t, maxsize
, n
* PAGE_SIZE
- *start
);
1050 i
->count
-= maxsize
;
1051 i
->iov_offset
+= maxsize
;
1052 if (i
->iov_offset
== i
->bvec
->bv_len
) {
1059 if (iov_iter_is_xarray(i
))
1060 return iter_xarray_get_pages(i
, pages
, maxsize
, maxpages
, start
);
1064 ssize_t
iov_iter_get_pages2(struct iov_iter
*i
, struct page
**pages
,
1065 size_t maxsize
, unsigned maxpages
, size_t *start
)
1071 return __iov_iter_get_pages_alloc(i
, &pages
, maxsize
, maxpages
, start
);
1073 EXPORT_SYMBOL(iov_iter_get_pages2
);
1075 ssize_t
iov_iter_get_pages_alloc2(struct iov_iter
*i
,
1076 struct page
***pages
, size_t maxsize
, size_t *start
)
1082 len
= __iov_iter_get_pages_alloc(i
, pages
, maxsize
, ~0U, start
);
1089 EXPORT_SYMBOL(iov_iter_get_pages_alloc2
);
1091 static int iov_npages(const struct iov_iter
*i
, int maxpages
)
1093 size_t skip
= i
->iov_offset
, size
= i
->count
;
1094 const struct iovec
*p
;
1097 for (p
= iter_iov(i
); size
; skip
= 0, p
++) {
1098 unsigned offs
= offset_in_page(p
->iov_base
+ skip
);
1099 size_t len
= min(p
->iov_len
- skip
, size
);
1103 npages
+= DIV_ROUND_UP(offs
+ len
, PAGE_SIZE
);
1104 if (unlikely(npages
> maxpages
))
1111 static int bvec_npages(const struct iov_iter
*i
, int maxpages
)
1113 size_t skip
= i
->iov_offset
, size
= i
->count
;
1114 const struct bio_vec
*p
;
1117 for (p
= i
->bvec
; size
; skip
= 0, p
++) {
1118 unsigned offs
= (p
->bv_offset
+ skip
) % PAGE_SIZE
;
1119 size_t len
= min(p
->bv_len
- skip
, size
);
1122 npages
+= DIV_ROUND_UP(offs
+ len
, PAGE_SIZE
);
1123 if (unlikely(npages
> maxpages
))
1129 int iov_iter_npages(const struct iov_iter
*i
, int maxpages
)
1131 if (unlikely(!i
->count
))
1133 if (likely(iter_is_ubuf(i
))) {
1134 unsigned offs
= offset_in_page(i
->ubuf
+ i
->iov_offset
);
1135 int npages
= DIV_ROUND_UP(offs
+ i
->count
, PAGE_SIZE
);
1136 return min(npages
, maxpages
);
1138 /* iovec and kvec have identical layouts */
1139 if (likely(iter_is_iovec(i
) || iov_iter_is_kvec(i
)))
1140 return iov_npages(i
, maxpages
);
1141 if (iov_iter_is_bvec(i
))
1142 return bvec_npages(i
, maxpages
);
1143 if (iov_iter_is_xarray(i
)) {
1144 unsigned offset
= (i
->xarray_start
+ i
->iov_offset
) % PAGE_SIZE
;
1145 int npages
= DIV_ROUND_UP(offset
+ i
->count
, PAGE_SIZE
);
1146 return min(npages
, maxpages
);
1150 EXPORT_SYMBOL(iov_iter_npages
);
1152 const void *dup_iter(struct iov_iter
*new, struct iov_iter
*old
, gfp_t flags
)
1155 if (iov_iter_is_bvec(new))
1156 return new->bvec
= kmemdup(new->bvec
,
1157 new->nr_segs
* sizeof(struct bio_vec
),
1159 else if (iov_iter_is_kvec(new) || iter_is_iovec(new))
1160 /* iovec and kvec have identical layout */
1161 return new->__iov
= kmemdup(new->__iov
,
1162 new->nr_segs
* sizeof(struct iovec
),
1166 EXPORT_SYMBOL(dup_iter
);
1168 static __noclone
int copy_compat_iovec_from_user(struct iovec
*iov
,
1169 const struct iovec __user
*uvec
, unsigned long nr_segs
)
1171 const struct compat_iovec __user
*uiov
=
1172 (const struct compat_iovec __user
*)uvec
;
1173 int ret
= -EFAULT
, i
;
1175 if (!user_access_begin(uiov
, nr_segs
* sizeof(*uiov
)))
1178 for (i
= 0; i
< nr_segs
; i
++) {
1182 unsafe_get_user(len
, &uiov
[i
].iov_len
, uaccess_end
);
1183 unsafe_get_user(buf
, &uiov
[i
].iov_base
, uaccess_end
);
1185 /* check for compat_size_t not fitting in compat_ssize_t .. */
1190 iov
[i
].iov_base
= compat_ptr(buf
);
1191 iov
[i
].iov_len
= len
;
1200 static __noclone
int copy_iovec_from_user(struct iovec
*iov
,
1201 const struct iovec __user
*uiov
, unsigned long nr_segs
)
1205 if (!user_access_begin(uiov
, nr_segs
* sizeof(*uiov
)))
1212 unsafe_get_user(len
, &uiov
->iov_len
, uaccess_end
);
1213 unsafe_get_user(buf
, &uiov
->iov_base
, uaccess_end
);
1215 /* check for size_t not fitting in ssize_t .. */
1216 if (unlikely(len
< 0)) {
1220 iov
->iov_base
= buf
;
1224 } while (--nr_segs
);
1232 struct iovec
*iovec_from_user(const struct iovec __user
*uvec
,
1233 unsigned long nr_segs
, unsigned long fast_segs
,
1234 struct iovec
*fast_iov
, bool compat
)
1236 struct iovec
*iov
= fast_iov
;
1240 * SuS says "The readv() function *may* fail if the iovcnt argument was
1241 * less than or equal to 0, or greater than {IOV_MAX}. Linux has
1242 * traditionally returned zero for zero segments, so...
1246 if (nr_segs
> UIO_MAXIOV
)
1247 return ERR_PTR(-EINVAL
);
1248 if (nr_segs
> fast_segs
) {
1249 iov
= kmalloc_array(nr_segs
, sizeof(struct iovec
), GFP_KERNEL
);
1251 return ERR_PTR(-ENOMEM
);
1254 if (unlikely(compat
))
1255 ret
= copy_compat_iovec_from_user(iov
, uvec
, nr_segs
);
1257 ret
= copy_iovec_from_user(iov
, uvec
, nr_segs
);
1259 if (iov
!= fast_iov
)
1261 return ERR_PTR(ret
);
1268 * Single segment iovec supplied by the user, import it as ITER_UBUF.
1270 static ssize_t
__import_iovec_ubuf(int type
, const struct iovec __user
*uvec
,
1271 struct iovec
**iovp
, struct iov_iter
*i
,
1274 struct iovec
*iov
= *iovp
;
1278 ret
= copy_compat_iovec_from_user(iov
, uvec
, 1);
1280 ret
= copy_iovec_from_user(iov
, uvec
, 1);
1284 ret
= import_ubuf(type
, iov
->iov_base
, iov
->iov_len
, i
);
1291 ssize_t
__import_iovec(int type
, const struct iovec __user
*uvec
,
1292 unsigned nr_segs
, unsigned fast_segs
, struct iovec
**iovp
,
1293 struct iov_iter
*i
, bool compat
)
1295 ssize_t total_len
= 0;
1300 return __import_iovec_ubuf(type
, uvec
, iovp
, i
, compat
);
1302 iov
= iovec_from_user(uvec
, nr_segs
, fast_segs
, *iovp
, compat
);
1305 return PTR_ERR(iov
);
1309 * According to the Single Unix Specification we should return EINVAL if
1310 * an element length is < 0 when cast to ssize_t or if the total length
1311 * would overflow the ssize_t return value of the system call.
1313 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
1316 for (seg
= 0; seg
< nr_segs
; seg
++) {
1317 ssize_t len
= (ssize_t
)iov
[seg
].iov_len
;
1319 if (!access_ok(iov
[seg
].iov_base
, len
)) {
1326 if (len
> MAX_RW_COUNT
- total_len
) {
1327 len
= MAX_RW_COUNT
- total_len
;
1328 iov
[seg
].iov_len
= len
;
1333 iov_iter_init(i
, type
, iov
, nr_segs
, total_len
);
1342 * import_iovec() - Copy an array of &struct iovec from userspace
1343 * into the kernel, check that it is valid, and initialize a new
1344 * &struct iov_iter iterator to access it.
1346 * @type: One of %READ or %WRITE.
1347 * @uvec: Pointer to the userspace array.
1348 * @nr_segs: Number of elements in userspace array.
1349 * @fast_segs: Number of elements in @iov.
1350 * @iovp: (input and output parameter) Pointer to pointer to (usually small
1351 * on-stack) kernel array.
1352 * @i: Pointer to iterator that will be initialized on success.
1354 * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1355 * then this function places %NULL in *@iov on return. Otherwise, a new
1356 * array will be allocated and the result placed in *@iov. This means that
1357 * the caller may call kfree() on *@iov regardless of whether the small
1358 * on-stack array was used or not (and regardless of whether this function
1359 * returns an error or not).
1361 * Return: Negative error code on error, bytes imported on success
1363 ssize_t
import_iovec(int type
, const struct iovec __user
*uvec
,
1364 unsigned nr_segs
, unsigned fast_segs
,
1365 struct iovec
**iovp
, struct iov_iter
*i
)
1367 return __import_iovec(type
, uvec
, nr_segs
, fast_segs
, iovp
, i
,
1368 in_compat_syscall());
1370 EXPORT_SYMBOL(import_iovec
);
1372 int import_single_range(int rw
, void __user
*buf
, size_t len
,
1373 struct iovec
*iov
, struct iov_iter
*i
)
1375 if (len
> MAX_RW_COUNT
)
1377 if (unlikely(!access_ok(buf
, len
)))
1380 iov_iter_ubuf(i
, rw
, buf
, len
);
1383 EXPORT_SYMBOL(import_single_range
);
1385 int import_ubuf(int rw
, void __user
*buf
, size_t len
, struct iov_iter
*i
)
1387 if (len
> MAX_RW_COUNT
)
1389 if (unlikely(!access_ok(buf
, len
)))
1392 iov_iter_ubuf(i
, rw
, buf
, len
);
1395 EXPORT_SYMBOL_GPL(import_ubuf
);
1398 * iov_iter_restore() - Restore a &struct iov_iter to the same state as when
1399 * iov_iter_save_state() was called.
1401 * @i: &struct iov_iter to restore
1402 * @state: state to restore from
1404 * Used after iov_iter_save_state() to bring restore @i, if operations may
1407 * Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC
1409 void iov_iter_restore(struct iov_iter
*i
, struct iov_iter_state
*state
)
1411 if (WARN_ON_ONCE(!iov_iter_is_bvec(i
) && !iter_is_iovec(i
) &&
1412 !iter_is_ubuf(i
)) && !iov_iter_is_kvec(i
))
1414 i
->iov_offset
= state
->iov_offset
;
1415 i
->count
= state
->count
;
1416 if (iter_is_ubuf(i
))
1419 * For the *vec iters, nr_segs + iov is constant - if we increment
1420 * the vec, then we also decrement the nr_segs count. Hence we don't
1421 * need to track both of these, just one is enough and we can deduct
1422 * the other from that. ITER_KVEC and ITER_IOVEC are the same struct
1423 * size, so we can just increment the iov pointer as they are unionzed.
1424 * ITER_BVEC _may_ be the same size on some archs, but on others it is
1425 * not. Be safe and handle it separately.
1427 BUILD_BUG_ON(sizeof(struct iovec
) != sizeof(struct kvec
));
1428 if (iov_iter_is_bvec(i
))
1429 i
->bvec
-= state
->nr_segs
- i
->nr_segs
;
1431 i
->__iov
-= state
->nr_segs
- i
->nr_segs
;
1432 i
->nr_segs
= state
->nr_segs
;
1436 * Extract a list of contiguous pages from an ITER_XARRAY iterator. This does not
1437 * get references on the pages, nor does it get a pin on them.
1439 static ssize_t
iov_iter_extract_xarray_pages(struct iov_iter
*i
,
1440 struct page
***pages
, size_t maxsize
,
1441 unsigned int maxpages
,
1442 iov_iter_extraction_t extraction_flags
,
1445 struct page
*page
, **p
;
1446 unsigned int nr
= 0, offset
;
1447 loff_t pos
= i
->xarray_start
+ i
->iov_offset
;
1448 pgoff_t index
= pos
>> PAGE_SHIFT
;
1449 XA_STATE(xas
, i
->xarray
, index
);
1451 offset
= pos
& ~PAGE_MASK
;
1454 maxpages
= want_pages_array(pages
, maxsize
, offset
, maxpages
);
1460 for (page
= xas_load(&xas
); page
; page
= xas_next(&xas
)) {
1461 if (xas_retry(&xas
, page
))
1464 /* Has the page moved or been split? */
1465 if (unlikely(page
!= xas_reload(&xas
))) {
1470 p
[nr
++] = find_subpage(page
, xas
.xa_index
);
1476 maxsize
= min_t(size_t, nr
* PAGE_SIZE
- offset
, maxsize
);
1477 iov_iter_advance(i
, maxsize
);
1482 * Extract a list of contiguous pages from an ITER_BVEC iterator. This does
1483 * not get references on the pages, nor does it get a pin on them.
1485 static ssize_t
iov_iter_extract_bvec_pages(struct iov_iter
*i
,
1486 struct page
***pages
, size_t maxsize
,
1487 unsigned int maxpages
,
1488 iov_iter_extraction_t extraction_flags
,
1491 struct page
**p
, *page
;
1492 size_t skip
= i
->iov_offset
, offset
, size
;
1496 if (i
->nr_segs
== 0)
1498 size
= min(maxsize
, i
->bvec
->bv_len
- skip
);
1507 skip
+= i
->bvec
->bv_offset
;
1508 page
= i
->bvec
->bv_page
+ skip
/ PAGE_SIZE
;
1509 offset
= skip
% PAGE_SIZE
;
1512 maxpages
= want_pages_array(pages
, size
, offset
, maxpages
);
1516 for (k
= 0; k
< maxpages
; k
++)
1519 size
= min_t(size_t, size
, maxpages
* PAGE_SIZE
- offset
);
1520 iov_iter_advance(i
, size
);
1525 * Extract a list of virtually contiguous pages from an ITER_KVEC iterator.
1526 * This does not get references on the pages, nor does it get a pin on them.
1528 static ssize_t
iov_iter_extract_kvec_pages(struct iov_iter
*i
,
1529 struct page
***pages
, size_t maxsize
,
1530 unsigned int maxpages
,
1531 iov_iter_extraction_t extraction_flags
,
1534 struct page
**p
, *page
;
1536 size_t skip
= i
->iov_offset
, offset
, len
, size
;
1540 if (i
->nr_segs
== 0)
1542 size
= min(maxsize
, i
->kvec
->iov_len
- skip
);
1551 kaddr
= i
->kvec
->iov_base
+ skip
;
1552 offset
= (unsigned long)kaddr
& ~PAGE_MASK
;
1555 maxpages
= want_pages_array(pages
, size
, offset
, maxpages
);
1561 len
= offset
+ size
;
1562 for (k
= 0; k
< maxpages
; k
++) {
1563 size_t seg
= min_t(size_t, len
, PAGE_SIZE
);
1565 if (is_vmalloc_or_module_addr(kaddr
))
1566 page
= vmalloc_to_page(kaddr
);
1568 page
= virt_to_page(kaddr
);
1575 size
= min_t(size_t, size
, maxpages
* PAGE_SIZE
- offset
);
1576 iov_iter_advance(i
, size
);
1581 * Extract a list of contiguous pages from a user iterator and get a pin on
1582 * each of them. This should only be used if the iterator is user-backed
1585 * It does not get refs on the pages, but the pages must be unpinned by the
1586 * caller once the transfer is complete.
1588 * This is safe to be used where background IO/DMA *is* going to be modifying
1589 * the buffer; using a pin rather than a ref makes forces fork() to give the
1590 * child a copy of the page.
1592 static ssize_t
iov_iter_extract_user_pages(struct iov_iter
*i
,
1593 struct page
***pages
,
1595 unsigned int maxpages
,
1596 iov_iter_extraction_t extraction_flags
,
1600 unsigned int gup_flags
= 0;
1604 if (i
->data_source
== ITER_DEST
)
1605 gup_flags
|= FOLL_WRITE
;
1606 if (extraction_flags
& ITER_ALLOW_P2PDMA
)
1607 gup_flags
|= FOLL_PCI_P2PDMA
;
1609 gup_flags
|= FOLL_NOFAULT
;
1611 addr
= first_iovec_segment(i
, &maxsize
);
1612 *offset0
= offset
= addr
% PAGE_SIZE
;
1614 maxpages
= want_pages_array(pages
, maxsize
, offset
, maxpages
);
1617 res
= pin_user_pages_fast(addr
, maxpages
, gup_flags
, *pages
);
1618 if (unlikely(res
<= 0))
1620 maxsize
= min_t(size_t, maxsize
, res
* PAGE_SIZE
- offset
);
1621 iov_iter_advance(i
, maxsize
);
1626 * iov_iter_extract_pages - Extract a list of contiguous pages from an iterator
1627 * @i: The iterator to extract from
1628 * @pages: Where to return the list of pages
1629 * @maxsize: The maximum amount of iterator to extract
1630 * @maxpages: The maximum size of the list of pages
1631 * @extraction_flags: Flags to qualify request
1632 * @offset0: Where to return the starting offset into (*@pages)[0]
1634 * Extract a list of contiguous pages from the current point of the iterator,
1635 * advancing the iterator. The maximum number of pages and the maximum amount
1636 * of page contents can be set.
1638 * If *@pages is NULL, a page list will be allocated to the required size and
1639 * *@pages will be set to its base. If *@pages is not NULL, it will be assumed
1640 * that the caller allocated a page list at least @maxpages in size and this
1641 * will be filled in.
1643 * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA
1644 * be allowed on the pages extracted.
1646 * The iov_iter_extract_will_pin() function can be used to query how cleanup
1647 * should be performed.
1649 * Extra refs or pins on the pages may be obtained as follows:
1651 * (*) If the iterator is user-backed (ITER_IOVEC/ITER_UBUF), pins will be
1652 * added to the pages, but refs will not be taken.
1653 * iov_iter_extract_will_pin() will return true.
1655 * (*) If the iterator is ITER_KVEC, ITER_BVEC or ITER_XARRAY, the pages are
1656 * merely listed; no extra refs or pins are obtained.
1657 * iov_iter_extract_will_pin() will return 0.
1661 * (*) Use with ITER_DISCARD is not supported as that has no content.
1663 * On success, the function sets *@pages to the new pagelist, if allocated, and
1664 * sets *offset0 to the offset into the first page.
1666 * It may also return -ENOMEM and -EFAULT.
1668 ssize_t
iov_iter_extract_pages(struct iov_iter
*i
,
1669 struct page
***pages
,
1671 unsigned int maxpages
,
1672 iov_iter_extraction_t extraction_flags
,
1675 maxsize
= min_t(size_t, min_t(size_t, maxsize
, i
->count
), MAX_RW_COUNT
);
1679 if (likely(user_backed_iter(i
)))
1680 return iov_iter_extract_user_pages(i
, pages
, maxsize
,
1681 maxpages
, extraction_flags
,
1683 if (iov_iter_is_kvec(i
))
1684 return iov_iter_extract_kvec_pages(i
, pages
, maxsize
,
1685 maxpages
, extraction_flags
,
1687 if (iov_iter_is_bvec(i
))
1688 return iov_iter_extract_bvec_pages(i
, pages
, maxsize
,
1689 maxpages
, extraction_flags
,
1691 if (iov_iter_is_xarray(i
))
1692 return iov_iter_extract_xarray_pages(i
, pages
, maxsize
,
1693 maxpages
, extraction_flags
,
1697 EXPORT_SYMBOL_GPL(iov_iter_extract_pages
);