1 #include <linux/export.h>
2 #include <linux/bvec.h>
4 #include <linux/pagemap.h>
5 #include <linux/slab.h>
6 #include <linux/vmalloc.h>
7 #include <linux/splice.h>
8 #include <net/checksum.h>
9 #include <linux/scatterlist.h>
11 #define PIPE_PARANOIA /* for now */
13 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
17 __v.iov_len = min(n, __p->iov_len - skip); \
18 if (likely(__v.iov_len)) { \
19 __v.iov_base = __p->iov_base + skip; \
21 __v.iov_len -= left; \
22 skip += __v.iov_len; \
27 while (unlikely(!left && n)) { \
29 __v.iov_len = min(n, __p->iov_len); \
30 if (unlikely(!__v.iov_len)) \
32 __v.iov_base = __p->iov_base; \
34 __v.iov_len -= left; \
41 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
44 __v.iov_len = min(n, __p->iov_len - skip); \
45 if (likely(__v.iov_len)) { \
46 __v.iov_base = __p->iov_base + skip; \
48 skip += __v.iov_len; \
51 while (unlikely(n)) { \
53 __v.iov_len = min(n, __p->iov_len); \
54 if (unlikely(!__v.iov_len)) \
56 __v.iov_base = __p->iov_base; \
64 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
65 struct bvec_iter __start; \
66 __start.bi_size = n; \
67 __start.bi_bvec_done = skip; \
69 for_each_bvec(__v, i->bvec, __bi, __start) { \
76 #define iterate_all_kinds(i, n, v, I, B, K) { \
78 size_t skip = i->iov_offset; \
79 if (unlikely(i->type & ITER_BVEC)) { \
81 struct bvec_iter __bi; \
82 iterate_bvec(i, n, v, __bi, skip, (B)) \
83 } else if (unlikely(i->type & ITER_KVEC)) { \
84 const struct kvec *kvec; \
86 iterate_kvec(i, n, v, kvec, skip, (K)) \
87 } else if (unlikely(i->type & ITER_DISCARD)) { \
89 const struct iovec *iov; \
91 iterate_iovec(i, n, v, iov, skip, (I)) \
96 #define iterate_and_advance(i, n, v, I, B, K) { \
97 if (unlikely(i->count < n)) \
100 size_t skip = i->iov_offset; \
101 if (unlikely(i->type & ITER_BVEC)) { \
102 const struct bio_vec *bvec = i->bvec; \
104 struct bvec_iter __bi; \
105 iterate_bvec(i, n, v, __bi, skip, (B)) \
106 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
107 i->nr_segs -= i->bvec - bvec; \
108 skip = __bi.bi_bvec_done; \
109 } else if (unlikely(i->type & ITER_KVEC)) { \
110 const struct kvec *kvec; \
112 iterate_kvec(i, n, v, kvec, skip, (K)) \
113 if (skip == kvec->iov_len) { \
117 i->nr_segs -= kvec - i->kvec; \
119 } else if (unlikely(i->type & ITER_DISCARD)) { \
122 const struct iovec *iov; \
124 iterate_iovec(i, n, v, iov, skip, (I)) \
125 if (skip == iov->iov_len) { \
129 i->nr_segs -= iov - i->iov; \
133 i->iov_offset = skip; \
137 static int copyout(void __user
*to
, const void *from
, size_t n
)
139 if (access_ok(to
, n
)) {
140 kasan_check_read(from
, n
);
141 n
= raw_copy_to_user(to
, from
, n
);
146 static int copyin(void *to
, const void __user
*from
, size_t n
)
148 if (access_ok(from
, n
)) {
149 kasan_check_write(to
, n
);
150 n
= raw_copy_from_user(to
, from
, n
);
155 static size_t copy_page_to_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
158 size_t skip
, copy
, left
, wanted
;
159 const struct iovec
*iov
;
163 if (unlikely(bytes
> i
->count
))
166 if (unlikely(!bytes
))
172 skip
= i
->iov_offset
;
173 buf
= iov
->iov_base
+ skip
;
174 copy
= min(bytes
, iov
->iov_len
- skip
);
176 if (IS_ENABLED(CONFIG_HIGHMEM
) && !fault_in_pages_writeable(buf
, copy
)) {
177 kaddr
= kmap_atomic(page
);
178 from
= kaddr
+ offset
;
180 /* first chunk, usually the only one */
181 left
= copyout(buf
, from
, copy
);
187 while (unlikely(!left
&& bytes
)) {
190 copy
= min(bytes
, iov
->iov_len
);
191 left
= copyout(buf
, from
, copy
);
197 if (likely(!bytes
)) {
198 kunmap_atomic(kaddr
);
201 offset
= from
- kaddr
;
203 kunmap_atomic(kaddr
);
204 copy
= min(bytes
, iov
->iov_len
- skip
);
206 /* Too bad - revert to non-atomic kmap */
209 from
= kaddr
+ offset
;
210 left
= copyout(buf
, from
, copy
);
215 while (unlikely(!left
&& bytes
)) {
218 copy
= min(bytes
, iov
->iov_len
);
219 left
= copyout(buf
, from
, copy
);
228 if (skip
== iov
->iov_len
) {
232 i
->count
-= wanted
- bytes
;
233 i
->nr_segs
-= iov
- i
->iov
;
235 i
->iov_offset
= skip
;
236 return wanted
- bytes
;
239 static size_t copy_page_from_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
242 size_t skip
, copy
, left
, wanted
;
243 const struct iovec
*iov
;
247 if (unlikely(bytes
> i
->count
))
250 if (unlikely(!bytes
))
256 skip
= i
->iov_offset
;
257 buf
= iov
->iov_base
+ skip
;
258 copy
= min(bytes
, iov
->iov_len
- skip
);
260 if (IS_ENABLED(CONFIG_HIGHMEM
) && !fault_in_pages_readable(buf
, copy
)) {
261 kaddr
= kmap_atomic(page
);
264 /* first chunk, usually the only one */
265 left
= copyin(to
, buf
, copy
);
271 while (unlikely(!left
&& bytes
)) {
274 copy
= min(bytes
, iov
->iov_len
);
275 left
= copyin(to
, buf
, copy
);
281 if (likely(!bytes
)) {
282 kunmap_atomic(kaddr
);
287 kunmap_atomic(kaddr
);
288 copy
= min(bytes
, iov
->iov_len
- skip
);
290 /* Too bad - revert to non-atomic kmap */
294 left
= copyin(to
, buf
, copy
);
299 while (unlikely(!left
&& bytes
)) {
302 copy
= min(bytes
, iov
->iov_len
);
303 left
= copyin(to
, buf
, copy
);
312 if (skip
== iov
->iov_len
) {
316 i
->count
-= wanted
- bytes
;
317 i
->nr_segs
-= iov
- i
->iov
;
319 i
->iov_offset
= skip
;
320 return wanted
- bytes
;
324 static bool sanity(const struct iov_iter
*i
)
326 struct pipe_inode_info
*pipe
= i
->pipe
;
328 int next
= pipe
->curbuf
+ pipe
->nrbufs
;
330 struct pipe_buffer
*p
;
331 if (unlikely(!pipe
->nrbufs
))
332 goto Bad
; // pipe must be non-empty
333 if (unlikely(idx
!= ((next
- 1) & (pipe
->buffers
- 1))))
334 goto Bad
; // must be at the last buffer...
336 p
= &pipe
->bufs
[idx
];
337 if (unlikely(p
->offset
+ p
->len
!= i
->iov_offset
))
338 goto Bad
; // ... at the end of segment
340 if (idx
!= (next
& (pipe
->buffers
- 1)))
341 goto Bad
; // must be right after the last buffer
345 printk(KERN_ERR
"idx = %d, offset = %zd\n", i
->idx
, i
->iov_offset
);
346 printk(KERN_ERR
"curbuf = %d, nrbufs = %d, buffers = %d\n",
347 pipe
->curbuf
, pipe
->nrbufs
, pipe
->buffers
);
348 for (idx
= 0; idx
< pipe
->buffers
; idx
++)
349 printk(KERN_ERR
"[%p %p %d %d]\n",
351 pipe
->bufs
[idx
].page
,
352 pipe
->bufs
[idx
].offset
,
353 pipe
->bufs
[idx
].len
);
358 #define sanity(i) true
361 static inline int next_idx(int idx
, struct pipe_inode_info
*pipe
)
363 return (idx
+ 1) & (pipe
->buffers
- 1);
366 static size_t copy_page_to_iter_pipe(struct page
*page
, size_t offset
, size_t bytes
,
369 struct pipe_inode_info
*pipe
= i
->pipe
;
370 struct pipe_buffer
*buf
;
374 if (unlikely(bytes
> i
->count
))
377 if (unlikely(!bytes
))
385 buf
= &pipe
->bufs
[idx
];
387 if (offset
== off
&& buf
->page
== page
) {
388 /* merge with the last one */
390 i
->iov_offset
+= bytes
;
393 idx
= next_idx(idx
, pipe
);
394 buf
= &pipe
->bufs
[idx
];
396 if (idx
== pipe
->curbuf
&& pipe
->nrbufs
)
399 buf
->ops
= &page_cache_pipe_buf_ops
;
400 get_page(buf
->page
= page
);
401 buf
->offset
= offset
;
403 i
->iov_offset
= offset
+ bytes
;
411 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
412 * bytes. For each iovec, fault in each page that constitutes the iovec.
414 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
415 * because it is an invalid address).
417 int iov_iter_fault_in_readable(struct iov_iter
*i
, size_t bytes
)
419 size_t skip
= i
->iov_offset
;
420 const struct iovec
*iov
;
424 if (!(i
->type
& (ITER_BVEC
|ITER_KVEC
))) {
425 iterate_iovec(i
, bytes
, v
, iov
, skip
, ({
426 err
= fault_in_pages_readable(v
.iov_base
, v
.iov_len
);
433 EXPORT_SYMBOL(iov_iter_fault_in_readable
);
435 void iov_iter_init(struct iov_iter
*i
, unsigned int direction
,
436 const struct iovec
*iov
, unsigned long nr_segs
,
439 WARN_ON(direction
& ~(READ
| WRITE
));
440 direction
&= READ
| WRITE
;
442 /* It will get better. Eventually... */
443 if (uaccess_kernel()) {
444 i
->type
= ITER_KVEC
| direction
;
445 i
->kvec
= (struct kvec
*)iov
;
447 i
->type
= ITER_IOVEC
| direction
;
450 i
->nr_segs
= nr_segs
;
454 EXPORT_SYMBOL(iov_iter_init
);
456 static void memcpy_from_page(char *to
, struct page
*page
, size_t offset
, size_t len
)
458 char *from
= kmap_atomic(page
);
459 memcpy(to
, from
+ offset
, len
);
463 static void memcpy_to_page(struct page
*page
, size_t offset
, const char *from
, size_t len
)
465 char *to
= kmap_atomic(page
);
466 memcpy(to
+ offset
, from
, len
);
470 static void memzero_page(struct page
*page
, size_t offset
, size_t len
)
472 char *addr
= kmap_atomic(page
);
473 memset(addr
+ offset
, 0, len
);
477 static inline bool allocated(struct pipe_buffer
*buf
)
479 return buf
->ops
== &default_pipe_buf_ops
;
482 static inline void data_start(const struct iov_iter
*i
, int *idxp
, size_t *offp
)
484 size_t off
= i
->iov_offset
;
486 if (off
&& (!allocated(&i
->pipe
->bufs
[idx
]) || off
== PAGE_SIZE
)) {
487 idx
= next_idx(idx
, i
->pipe
);
494 static size_t push_pipe(struct iov_iter
*i
, size_t size
,
495 int *idxp
, size_t *offp
)
497 struct pipe_inode_info
*pipe
= i
->pipe
;
502 if (unlikely(size
> i
->count
))
508 data_start(i
, &idx
, &off
);
512 left
-= PAGE_SIZE
- off
;
514 pipe
->bufs
[idx
].len
+= size
;
517 pipe
->bufs
[idx
].len
= PAGE_SIZE
;
518 idx
= next_idx(idx
, pipe
);
520 while (idx
!= pipe
->curbuf
|| !pipe
->nrbufs
) {
521 struct page
*page
= alloc_page(GFP_USER
);
525 pipe
->bufs
[idx
].ops
= &default_pipe_buf_ops
;
526 pipe
->bufs
[idx
].page
= page
;
527 pipe
->bufs
[idx
].offset
= 0;
528 if (left
<= PAGE_SIZE
) {
529 pipe
->bufs
[idx
].len
= left
;
532 pipe
->bufs
[idx
].len
= PAGE_SIZE
;
534 idx
= next_idx(idx
, pipe
);
539 static size_t copy_pipe_to_iter(const void *addr
, size_t bytes
,
542 struct pipe_inode_info
*pipe
= i
->pipe
;
549 bytes
= n
= push_pipe(i
, bytes
, &idx
, &off
);
552 for ( ; n
; idx
= next_idx(idx
, pipe
), off
= 0) {
553 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- off
);
554 memcpy_to_page(pipe
->bufs
[idx
].page
, off
, addr
, chunk
);
556 i
->iov_offset
= off
+ chunk
;
564 static __wsum
csum_and_memcpy(void *to
, const void *from
, size_t len
,
565 __wsum sum
, size_t off
)
567 __wsum next
= csum_partial_copy_nocheck(from
, to
, len
, 0);
568 return csum_block_add(sum
, next
, off
);
571 static size_t csum_and_copy_to_pipe_iter(const void *addr
, size_t bytes
,
572 __wsum
*csum
, struct iov_iter
*i
)
574 struct pipe_inode_info
*pipe
= i
->pipe
;
583 bytes
= n
= push_pipe(i
, bytes
, &idx
, &r
);
586 for ( ; n
; idx
= next_idx(idx
, pipe
), r
= 0) {
587 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- r
);
588 char *p
= kmap_atomic(pipe
->bufs
[idx
].page
);
589 sum
= csum_and_memcpy(p
+ r
, addr
, chunk
, sum
, off
);
592 i
->iov_offset
= r
+ chunk
;
602 size_t _copy_to_iter(const void *addr
, size_t bytes
, struct iov_iter
*i
)
604 const char *from
= addr
;
605 if (unlikely(iov_iter_is_pipe(i
)))
606 return copy_pipe_to_iter(addr
, bytes
, i
);
607 if (iter_is_iovec(i
))
609 iterate_and_advance(i
, bytes
, v
,
610 copyout(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
, v
.iov_len
),
611 memcpy_to_page(v
.bv_page
, v
.bv_offset
,
612 (from
+= v
.bv_len
) - v
.bv_len
, v
.bv_len
),
613 memcpy(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
, v
.iov_len
)
618 EXPORT_SYMBOL(_copy_to_iter
);
620 #ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE
621 static int copyout_mcsafe(void __user
*to
, const void *from
, size_t n
)
623 if (access_ok(to
, n
)) {
624 kasan_check_read(from
, n
);
625 n
= copy_to_user_mcsafe((__force
void *) to
, from
, n
);
630 static unsigned long memcpy_mcsafe_to_page(struct page
*page
, size_t offset
,
631 const char *from
, size_t len
)
636 to
= kmap_atomic(page
);
637 ret
= memcpy_mcsafe(to
+ offset
, from
, len
);
643 static size_t copy_pipe_to_iter_mcsafe(const void *addr
, size_t bytes
,
646 struct pipe_inode_info
*pipe
= i
->pipe
;
647 size_t n
, off
, xfer
= 0;
653 bytes
= n
= push_pipe(i
, bytes
, &idx
, &off
);
656 for ( ; n
; idx
= next_idx(idx
, pipe
), off
= 0) {
657 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- off
);
660 rem
= memcpy_mcsafe_to_page(pipe
->bufs
[idx
].page
, off
, addr
,
663 i
->iov_offset
= off
+ chunk
- rem
;
675 * _copy_to_iter_mcsafe - copy to user with source-read error exception handling
676 * @addr: source kernel address
677 * @bytes: total transfer length
678 * @iter: destination iterator
680 * The pmem driver arranges for filesystem-dax to use this facility via
681 * dax_copy_to_iter() for protecting read/write to persistent memory.
682 * Unless / until an architecture can guarantee identical performance
683 * between _copy_to_iter_mcsafe() and _copy_to_iter() it would be a
684 * performance regression to switch more users to the mcsafe version.
686 * Otherwise, the main differences between this and typical _copy_to_iter().
688 * * Typical tail/residue handling after a fault retries the copy
689 * byte-by-byte until the fault happens again. Re-triggering machine
690 * checks is potentially fatal so the implementation uses source
691 * alignment and poison alignment assumptions to avoid re-triggering
692 * hardware exceptions.
694 * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
695 * Compare to copy_to_iter() where only ITER_IOVEC attempts might return
698 * See MCSAFE_TEST for self-test.
700 size_t _copy_to_iter_mcsafe(const void *addr
, size_t bytes
, struct iov_iter
*i
)
702 const char *from
= addr
;
703 unsigned long rem
, curr_addr
, s_addr
= (unsigned long) addr
;
705 if (unlikely(iov_iter_is_pipe(i
)))
706 return copy_pipe_to_iter_mcsafe(addr
, bytes
, i
);
707 if (iter_is_iovec(i
))
709 iterate_and_advance(i
, bytes
, v
,
710 copyout_mcsafe(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
, v
.iov_len
),
712 rem
= memcpy_mcsafe_to_page(v
.bv_page
, v
.bv_offset
,
713 (from
+= v
.bv_len
) - v
.bv_len
, v
.bv_len
);
715 curr_addr
= (unsigned long) from
;
716 bytes
= curr_addr
- s_addr
- rem
;
721 rem
= memcpy_mcsafe(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
,
724 curr_addr
= (unsigned long) from
;
725 bytes
= curr_addr
- s_addr
- rem
;
733 EXPORT_SYMBOL_GPL(_copy_to_iter_mcsafe
);
734 #endif /* CONFIG_ARCH_HAS_UACCESS_MCSAFE */
736 size_t _copy_from_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
739 if (unlikely(iov_iter_is_pipe(i
))) {
743 if (iter_is_iovec(i
))
745 iterate_and_advance(i
, bytes
, v
,
746 copyin((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
),
747 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
748 v
.bv_offset
, v
.bv_len
),
749 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
754 EXPORT_SYMBOL(_copy_from_iter
);
756 bool _copy_from_iter_full(void *addr
, size_t bytes
, struct iov_iter
*i
)
759 if (unlikely(iov_iter_is_pipe(i
))) {
763 if (unlikely(i
->count
< bytes
))
766 if (iter_is_iovec(i
))
768 iterate_all_kinds(i
, bytes
, v
, ({
769 if (copyin((to
+= v
.iov_len
) - v
.iov_len
,
770 v
.iov_base
, v
.iov_len
))
773 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
774 v
.bv_offset
, v
.bv_len
),
775 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
778 iov_iter_advance(i
, bytes
);
781 EXPORT_SYMBOL(_copy_from_iter_full
);
783 size_t _copy_from_iter_nocache(void *addr
, size_t bytes
, struct iov_iter
*i
)
786 if (unlikely(iov_iter_is_pipe(i
))) {
790 iterate_and_advance(i
, bytes
, v
,
791 __copy_from_user_inatomic_nocache((to
+= v
.iov_len
) - v
.iov_len
,
792 v
.iov_base
, v
.iov_len
),
793 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
794 v
.bv_offset
, v
.bv_len
),
795 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
800 EXPORT_SYMBOL(_copy_from_iter_nocache
);
802 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
804 * _copy_from_iter_flushcache - write destination through cpu cache
805 * @addr: destination kernel address
806 * @bytes: total transfer length
807 * @iter: source iterator
809 * The pmem driver arranges for filesystem-dax to use this facility via
810 * dax_copy_from_iter() for ensuring that writes to persistent memory
811 * are flushed through the CPU cache. It is differentiated from
812 * _copy_from_iter_nocache() in that guarantees all data is flushed for
813 * all iterator types. The _copy_from_iter_nocache() only attempts to
814 * bypass the cache for the ITER_IOVEC case, and on some archs may use
815 * instructions that strand dirty-data in the cache.
817 size_t _copy_from_iter_flushcache(void *addr
, size_t bytes
, struct iov_iter
*i
)
820 if (unlikely(iov_iter_is_pipe(i
))) {
824 iterate_and_advance(i
, bytes
, v
,
825 __copy_from_user_flushcache((to
+= v
.iov_len
) - v
.iov_len
,
826 v
.iov_base
, v
.iov_len
),
827 memcpy_page_flushcache((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
828 v
.bv_offset
, v
.bv_len
),
829 memcpy_flushcache((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
,
835 EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache
);
838 bool _copy_from_iter_full_nocache(void *addr
, size_t bytes
, struct iov_iter
*i
)
841 if (unlikely(iov_iter_is_pipe(i
))) {
845 if (unlikely(i
->count
< bytes
))
847 iterate_all_kinds(i
, bytes
, v
, ({
848 if (__copy_from_user_inatomic_nocache((to
+= v
.iov_len
) - v
.iov_len
,
849 v
.iov_base
, v
.iov_len
))
852 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
853 v
.bv_offset
, v
.bv_len
),
854 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
857 iov_iter_advance(i
, bytes
);
860 EXPORT_SYMBOL(_copy_from_iter_full_nocache
);
862 static inline bool page_copy_sane(struct page
*page
, size_t offset
, size_t n
)
864 struct page
*head
= compound_head(page
);
865 size_t v
= n
+ offset
+ page_address(page
) - page_address(head
);
867 if (likely(n
<= v
&& v
<= (PAGE_SIZE
<< compound_order(head
))))
873 size_t copy_page_to_iter(struct page
*page
, size_t offset
, size_t bytes
,
876 if (unlikely(!page_copy_sane(page
, offset
, bytes
)))
878 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
879 void *kaddr
= kmap_atomic(page
);
880 size_t wanted
= copy_to_iter(kaddr
+ offset
, bytes
, i
);
881 kunmap_atomic(kaddr
);
883 } else if (unlikely(iov_iter_is_discard(i
)))
885 else if (likely(!iov_iter_is_pipe(i
)))
886 return copy_page_to_iter_iovec(page
, offset
, bytes
, i
);
888 return copy_page_to_iter_pipe(page
, offset
, bytes
, i
);
890 EXPORT_SYMBOL(copy_page_to_iter
);
892 size_t copy_page_from_iter(struct page
*page
, size_t offset
, size_t bytes
,
895 if (unlikely(!page_copy_sane(page
, offset
, bytes
)))
897 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
901 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
902 void *kaddr
= kmap_atomic(page
);
903 size_t wanted
= _copy_from_iter(kaddr
+ offset
, bytes
, i
);
904 kunmap_atomic(kaddr
);
907 return copy_page_from_iter_iovec(page
, offset
, bytes
, i
);
909 EXPORT_SYMBOL(copy_page_from_iter
);
911 static size_t pipe_zero(size_t bytes
, struct iov_iter
*i
)
913 struct pipe_inode_info
*pipe
= i
->pipe
;
920 bytes
= n
= push_pipe(i
, bytes
, &idx
, &off
);
924 for ( ; n
; idx
= next_idx(idx
, pipe
), off
= 0) {
925 size_t chunk
= min_t(size_t, n
, PAGE_SIZE
- off
);
926 memzero_page(pipe
->bufs
[idx
].page
, off
, chunk
);
928 i
->iov_offset
= off
+ chunk
;
935 size_t iov_iter_zero(size_t bytes
, struct iov_iter
*i
)
937 if (unlikely(iov_iter_is_pipe(i
)))
938 return pipe_zero(bytes
, i
);
939 iterate_and_advance(i
, bytes
, v
,
940 clear_user(v
.iov_base
, v
.iov_len
),
941 memzero_page(v
.bv_page
, v
.bv_offset
, v
.bv_len
),
942 memset(v
.iov_base
, 0, v
.iov_len
)
947 EXPORT_SYMBOL(iov_iter_zero
);
949 size_t iov_iter_copy_from_user_atomic(struct page
*page
,
950 struct iov_iter
*i
, unsigned long offset
, size_t bytes
)
952 char *kaddr
= kmap_atomic(page
), *p
= kaddr
+ offset
;
953 if (unlikely(!page_copy_sane(page
, offset
, bytes
))) {
954 kunmap_atomic(kaddr
);
957 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
958 kunmap_atomic(kaddr
);
962 iterate_all_kinds(i
, bytes
, v
,
963 copyin((p
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
),
964 memcpy_from_page((p
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
965 v
.bv_offset
, v
.bv_len
),
966 memcpy((p
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
968 kunmap_atomic(kaddr
);
971 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic
);
973 static inline void pipe_truncate(struct iov_iter
*i
)
975 struct pipe_inode_info
*pipe
= i
->pipe
;
977 size_t off
= i
->iov_offset
;
979 int nrbufs
= (idx
- pipe
->curbuf
) & (pipe
->buffers
- 1);
981 pipe
->bufs
[idx
].len
= off
- pipe
->bufs
[idx
].offset
;
982 idx
= next_idx(idx
, pipe
);
985 while (pipe
->nrbufs
> nrbufs
) {
986 pipe_buf_release(pipe
, &pipe
->bufs
[idx
]);
987 idx
= next_idx(idx
, pipe
);
993 static void pipe_advance(struct iov_iter
*i
, size_t size
)
995 struct pipe_inode_info
*pipe
= i
->pipe
;
996 if (unlikely(i
->count
< size
))
999 struct pipe_buffer
*buf
;
1000 size_t off
= i
->iov_offset
, left
= size
;
1002 if (off
) /* make it relative to the beginning of buffer */
1003 left
+= off
- pipe
->bufs
[idx
].offset
;
1005 buf
= &pipe
->bufs
[idx
];
1006 if (left
<= buf
->len
)
1009 idx
= next_idx(idx
, pipe
);
1012 i
->iov_offset
= buf
->offset
+ left
;
1015 /* ... and discard everything past that point */
1019 void iov_iter_advance(struct iov_iter
*i
, size_t size
)
1021 if (unlikely(iov_iter_is_pipe(i
))) {
1022 pipe_advance(i
, size
);
1025 if (unlikely(iov_iter_is_discard(i
))) {
1029 iterate_and_advance(i
, size
, v
, 0, 0, 0)
1031 EXPORT_SYMBOL(iov_iter_advance
);
1033 void iov_iter_revert(struct iov_iter
*i
, size_t unroll
)
1037 if (WARN_ON(unroll
> MAX_RW_COUNT
))
1040 if (unlikely(iov_iter_is_pipe(i
))) {
1041 struct pipe_inode_info
*pipe
= i
->pipe
;
1043 size_t off
= i
->iov_offset
;
1045 size_t n
= off
- pipe
->bufs
[idx
].offset
;
1051 if (!unroll
&& idx
== i
->start_idx
) {
1056 idx
= pipe
->buffers
- 1;
1057 off
= pipe
->bufs
[idx
].offset
+ pipe
->bufs
[idx
].len
;
1059 i
->iov_offset
= off
;
1064 if (unlikely(iov_iter_is_discard(i
)))
1066 if (unroll
<= i
->iov_offset
) {
1067 i
->iov_offset
-= unroll
;
1070 unroll
-= i
->iov_offset
;
1071 if (iov_iter_is_bvec(i
)) {
1072 const struct bio_vec
*bvec
= i
->bvec
;
1074 size_t n
= (--bvec
)->bv_len
;
1078 i
->iov_offset
= n
- unroll
;
1083 } else { /* same logics for iovec and kvec */
1084 const struct iovec
*iov
= i
->iov
;
1086 size_t n
= (--iov
)->iov_len
;
1090 i
->iov_offset
= n
- unroll
;
1097 EXPORT_SYMBOL(iov_iter_revert
);
1100 * Return the count of just the current iov_iter segment.
1102 size_t iov_iter_single_seg_count(const struct iov_iter
*i
)
1104 if (unlikely(iov_iter_is_pipe(i
)))
1105 return i
->count
; // it is a silly place, anyway
1106 if (i
->nr_segs
== 1)
1108 if (unlikely(iov_iter_is_discard(i
)))
1110 else if (iov_iter_is_bvec(i
))
1111 return min(i
->count
, i
->bvec
->bv_len
- i
->iov_offset
);
1113 return min(i
->count
, i
->iov
->iov_len
- i
->iov_offset
);
1115 EXPORT_SYMBOL(iov_iter_single_seg_count
);
1117 void iov_iter_kvec(struct iov_iter
*i
, unsigned int direction
,
1118 const struct kvec
*kvec
, unsigned long nr_segs
,
1121 WARN_ON(direction
& ~(READ
| WRITE
));
1122 i
->type
= ITER_KVEC
| (direction
& (READ
| WRITE
));
1124 i
->nr_segs
= nr_segs
;
1128 EXPORT_SYMBOL(iov_iter_kvec
);
1130 void iov_iter_bvec(struct iov_iter
*i
, unsigned int direction
,
1131 const struct bio_vec
*bvec
, unsigned long nr_segs
,
1134 WARN_ON(direction
& ~(READ
| WRITE
));
1135 i
->type
= ITER_BVEC
| (direction
& (READ
| WRITE
));
1137 i
->nr_segs
= nr_segs
;
1141 EXPORT_SYMBOL(iov_iter_bvec
);
1143 void iov_iter_pipe(struct iov_iter
*i
, unsigned int direction
,
1144 struct pipe_inode_info
*pipe
,
1147 BUG_ON(direction
!= READ
);
1148 WARN_ON(pipe
->nrbufs
== pipe
->buffers
);
1149 i
->type
= ITER_PIPE
| READ
;
1151 i
->idx
= (pipe
->curbuf
+ pipe
->nrbufs
) & (pipe
->buffers
- 1);
1154 i
->start_idx
= i
->idx
;
1156 EXPORT_SYMBOL(iov_iter_pipe
);
1159 * iov_iter_discard - Initialise an I/O iterator that discards data
1160 * @i: The iterator to initialise.
1161 * @direction: The direction of the transfer.
1162 * @count: The size of the I/O buffer in bytes.
1164 * Set up an I/O iterator that just discards everything that's written to it.
1165 * It's only available as a READ iterator.
1167 void iov_iter_discard(struct iov_iter
*i
, unsigned int direction
, size_t count
)
1169 BUG_ON(direction
!= READ
);
1170 i
->type
= ITER_DISCARD
| READ
;
1174 EXPORT_SYMBOL(iov_iter_discard
);
1176 unsigned long iov_iter_alignment(const struct iov_iter
*i
)
1178 unsigned long res
= 0;
1179 size_t size
= i
->count
;
1181 if (unlikely(iov_iter_is_pipe(i
))) {
1182 if (size
&& i
->iov_offset
&& allocated(&i
->pipe
->bufs
[i
->idx
]))
1183 return size
| i
->iov_offset
;
1186 iterate_all_kinds(i
, size
, v
,
1187 (res
|= (unsigned long)v
.iov_base
| v
.iov_len
, 0),
1188 res
|= v
.bv_offset
| v
.bv_len
,
1189 res
|= (unsigned long)v
.iov_base
| v
.iov_len
1193 EXPORT_SYMBOL(iov_iter_alignment
);
1195 unsigned long iov_iter_gap_alignment(const struct iov_iter
*i
)
1197 unsigned long res
= 0;
1198 size_t size
= i
->count
;
1200 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
1205 iterate_all_kinds(i
, size
, v
,
1206 (res
|= (!res
? 0 : (unsigned long)v
.iov_base
) |
1207 (size
!= v
.iov_len
? size
: 0), 0),
1208 (res
|= (!res
? 0 : (unsigned long)v
.bv_offset
) |
1209 (size
!= v
.bv_len
? size
: 0)),
1210 (res
|= (!res
? 0 : (unsigned long)v
.iov_base
) |
1211 (size
!= v
.iov_len
? size
: 0))
1215 EXPORT_SYMBOL(iov_iter_gap_alignment
);
1217 static inline ssize_t
__pipe_get_pages(struct iov_iter
*i
,
1219 struct page
**pages
,
1223 struct pipe_inode_info
*pipe
= i
->pipe
;
1224 ssize_t n
= push_pipe(i
, maxsize
, &idx
, start
);
1231 get_page(*pages
++ = pipe
->bufs
[idx
].page
);
1232 idx
= next_idx(idx
, pipe
);
1239 static ssize_t
pipe_get_pages(struct iov_iter
*i
,
1240 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
1253 data_start(i
, &idx
, start
);
1254 /* some of this one + all after this one */
1255 npages
= ((i
->pipe
->curbuf
- idx
- 1) & (i
->pipe
->buffers
- 1)) + 1;
1256 capacity
= min(npages
,maxpages
) * PAGE_SIZE
- *start
;
1258 return __pipe_get_pages(i
, min(maxsize
, capacity
), pages
, idx
, start
);
1261 ssize_t
iov_iter_get_pages(struct iov_iter
*i
,
1262 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
1265 if (maxsize
> i
->count
)
1268 if (unlikely(iov_iter_is_pipe(i
)))
1269 return pipe_get_pages(i
, pages
, maxsize
, maxpages
, start
);
1270 if (unlikely(iov_iter_is_discard(i
)))
1273 iterate_all_kinds(i
, maxsize
, v
, ({
1274 unsigned long addr
= (unsigned long)v
.iov_base
;
1275 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
1279 if (len
> maxpages
* PAGE_SIZE
)
1280 len
= maxpages
* PAGE_SIZE
;
1281 addr
&= ~(PAGE_SIZE
- 1);
1282 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
1283 res
= get_user_pages_fast(addr
, n
, iov_iter_rw(i
) != WRITE
, pages
);
1284 if (unlikely(res
< 0))
1286 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
1288 /* can't be more than PAGE_SIZE */
1289 *start
= v
.bv_offset
;
1290 get_page(*pages
= v
.bv_page
);
1298 EXPORT_SYMBOL(iov_iter_get_pages
);
1300 static struct page
**get_pages_array(size_t n
)
1302 return kvmalloc_array(n
, sizeof(struct page
*), GFP_KERNEL
);
1305 static ssize_t
pipe_get_pages_alloc(struct iov_iter
*i
,
1306 struct page
***pages
, size_t maxsize
,
1320 data_start(i
, &idx
, start
);
1321 /* some of this one + all after this one */
1322 npages
= ((i
->pipe
->curbuf
- idx
- 1) & (i
->pipe
->buffers
- 1)) + 1;
1323 n
= npages
* PAGE_SIZE
- *start
;
1327 npages
= DIV_ROUND_UP(maxsize
+ *start
, PAGE_SIZE
);
1328 p
= get_pages_array(npages
);
1331 n
= __pipe_get_pages(i
, maxsize
, p
, idx
, start
);
1339 ssize_t
iov_iter_get_pages_alloc(struct iov_iter
*i
,
1340 struct page
***pages
, size_t maxsize
,
1345 if (maxsize
> i
->count
)
1348 if (unlikely(iov_iter_is_pipe(i
)))
1349 return pipe_get_pages_alloc(i
, pages
, maxsize
, start
);
1350 if (unlikely(iov_iter_is_discard(i
)))
1353 iterate_all_kinds(i
, maxsize
, v
, ({
1354 unsigned long addr
= (unsigned long)v
.iov_base
;
1355 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
1359 addr
&= ~(PAGE_SIZE
- 1);
1360 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
1361 p
= get_pages_array(n
);
1364 res
= get_user_pages_fast(addr
, n
, iov_iter_rw(i
) != WRITE
, p
);
1365 if (unlikely(res
< 0)) {
1370 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
1372 /* can't be more than PAGE_SIZE */
1373 *start
= v
.bv_offset
;
1374 *pages
= p
= get_pages_array(1);
1377 get_page(*p
= v
.bv_page
);
1385 EXPORT_SYMBOL(iov_iter_get_pages_alloc
);
1387 size_t csum_and_copy_from_iter(void *addr
, size_t bytes
, __wsum
*csum
,
1394 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
1398 iterate_and_advance(i
, bytes
, v
, ({
1400 next
= csum_and_copy_from_user(v
.iov_base
,
1401 (to
+= v
.iov_len
) - v
.iov_len
,
1402 v
.iov_len
, 0, &err
);
1404 sum
= csum_block_add(sum
, next
, off
);
1407 err
? v
.iov_len
: 0;
1409 char *p
= kmap_atomic(v
.bv_page
);
1410 sum
= csum_and_memcpy((to
+= v
.bv_len
) - v
.bv_len
,
1411 p
+ v
.bv_offset
, v
.bv_len
,
1416 sum
= csum_and_memcpy((to
+= v
.iov_len
) - v
.iov_len
,
1417 v
.iov_base
, v
.iov_len
,
1425 EXPORT_SYMBOL(csum_and_copy_from_iter
);
1427 bool csum_and_copy_from_iter_full(void *addr
, size_t bytes
, __wsum
*csum
,
1434 if (unlikely(iov_iter_is_pipe(i
) || iov_iter_is_discard(i
))) {
1438 if (unlikely(i
->count
< bytes
))
1440 iterate_all_kinds(i
, bytes
, v
, ({
1442 next
= csum_and_copy_from_user(v
.iov_base
,
1443 (to
+= v
.iov_len
) - v
.iov_len
,
1444 v
.iov_len
, 0, &err
);
1447 sum
= csum_block_add(sum
, next
, off
);
1451 char *p
= kmap_atomic(v
.bv_page
);
1452 sum
= csum_and_memcpy((to
+= v
.bv_len
) - v
.bv_len
,
1453 p
+ v
.bv_offset
, v
.bv_len
,
1458 sum
= csum_and_memcpy((to
+= v
.iov_len
) - v
.iov_len
,
1459 v
.iov_base
, v
.iov_len
,
1465 iov_iter_advance(i
, bytes
);
1468 EXPORT_SYMBOL(csum_and_copy_from_iter_full
);
1470 size_t csum_and_copy_to_iter(const void *addr
, size_t bytes
, void *csump
,
1473 const char *from
= addr
;
1474 __wsum
*csum
= csump
;
1478 if (unlikely(iov_iter_is_pipe(i
)))
1479 return csum_and_copy_to_pipe_iter(addr
, bytes
, csum
, i
);
1482 if (unlikely(iov_iter_is_discard(i
))) {
1483 WARN_ON(1); /* for now */
1486 iterate_and_advance(i
, bytes
, v
, ({
1488 next
= csum_and_copy_to_user((from
+= v
.iov_len
) - v
.iov_len
,
1490 v
.iov_len
, 0, &err
);
1492 sum
= csum_block_add(sum
, next
, off
);
1495 err
? v
.iov_len
: 0;
1497 char *p
= kmap_atomic(v
.bv_page
);
1498 sum
= csum_and_memcpy(p
+ v
.bv_offset
,
1499 (from
+= v
.bv_len
) - v
.bv_len
,
1500 v
.bv_len
, sum
, off
);
1504 sum
= csum_and_memcpy(v
.iov_base
,
1505 (from
+= v
.iov_len
) - v
.iov_len
,
1506 v
.iov_len
, sum
, off
);
1513 EXPORT_SYMBOL(csum_and_copy_to_iter
);
1515 size_t hash_and_copy_to_iter(const void *addr
, size_t bytes
, void *hashp
,
1518 struct ahash_request
*hash
= hashp
;
1519 struct scatterlist sg
;
1522 copied
= copy_to_iter(addr
, bytes
, i
);
1523 sg_init_one(&sg
, addr
, copied
);
1524 ahash_request_set_crypt(hash
, &sg
, NULL
, copied
);
1525 crypto_ahash_update(hash
);
1528 EXPORT_SYMBOL(hash_and_copy_to_iter
);
1530 int iov_iter_npages(const struct iov_iter
*i
, int maxpages
)
1532 size_t size
= i
->count
;
1537 if (unlikely(iov_iter_is_discard(i
)))
1540 if (unlikely(iov_iter_is_pipe(i
))) {
1541 struct pipe_inode_info
*pipe
= i
->pipe
;
1548 data_start(i
, &idx
, &off
);
1549 /* some of this one + all after this one */
1550 npages
= ((pipe
->curbuf
- idx
- 1) & (pipe
->buffers
- 1)) + 1;
1551 if (npages
>= maxpages
)
1553 } else iterate_all_kinds(i
, size
, v
, ({
1554 unsigned long p
= (unsigned long)v
.iov_base
;
1555 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
1557 if (npages
>= maxpages
)
1561 if (npages
>= maxpages
)
1564 unsigned long p
= (unsigned long)v
.iov_base
;
1565 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
1567 if (npages
>= maxpages
)
1573 EXPORT_SYMBOL(iov_iter_npages
);
1575 const void *dup_iter(struct iov_iter
*new, struct iov_iter
*old
, gfp_t flags
)
1578 if (unlikely(iov_iter_is_pipe(new))) {
1582 if (unlikely(iov_iter_is_discard(new)))
1584 if (iov_iter_is_bvec(new))
1585 return new->bvec
= kmemdup(new->bvec
,
1586 new->nr_segs
* sizeof(struct bio_vec
),
1589 /* iovec and kvec have identical layout */
1590 return new->iov
= kmemdup(new->iov
,
1591 new->nr_segs
* sizeof(struct iovec
),
1594 EXPORT_SYMBOL(dup_iter
);
1597 * import_iovec() - Copy an array of &struct iovec from userspace
1598 * into the kernel, check that it is valid, and initialize a new
1599 * &struct iov_iter iterator to access it.
1601 * @type: One of %READ or %WRITE.
1602 * @uvector: Pointer to the userspace array.
1603 * @nr_segs: Number of elements in userspace array.
1604 * @fast_segs: Number of elements in @iov.
1605 * @iov: (input and output parameter) Pointer to pointer to (usually small
1606 * on-stack) kernel array.
1607 * @i: Pointer to iterator that will be initialized on success.
1609 * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1610 * then this function places %NULL in *@iov on return. Otherwise, a new
1611 * array will be allocated and the result placed in *@iov. This means that
1612 * the caller may call kfree() on *@iov regardless of whether the small
1613 * on-stack array was used or not (and regardless of whether this function
1614 * returns an error or not).
1616 * Return: 0 on success or negative error code on error.
1618 int import_iovec(int type
, const struct iovec __user
* uvector
,
1619 unsigned nr_segs
, unsigned fast_segs
,
1620 struct iovec
**iov
, struct iov_iter
*i
)
1624 n
= rw_copy_check_uvector(type
, uvector
, nr_segs
, fast_segs
,
1632 iov_iter_init(i
, type
, p
, nr_segs
, n
);
1633 *iov
= p
== *iov
? NULL
: p
;
1636 EXPORT_SYMBOL(import_iovec
);
1638 #ifdef CONFIG_COMPAT
1639 #include <linux/compat.h>
1641 int compat_import_iovec(int type
, const struct compat_iovec __user
* uvector
,
1642 unsigned nr_segs
, unsigned fast_segs
,
1643 struct iovec
**iov
, struct iov_iter
*i
)
1647 n
= compat_rw_copy_check_uvector(type
, uvector
, nr_segs
, fast_segs
,
1655 iov_iter_init(i
, type
, p
, nr_segs
, n
);
1656 *iov
= p
== *iov
? NULL
: p
;
1661 int import_single_range(int rw
, void __user
*buf
, size_t len
,
1662 struct iovec
*iov
, struct iov_iter
*i
)
1664 if (len
> MAX_RW_COUNT
)
1666 if (unlikely(!access_ok(buf
, len
)))
1669 iov
->iov_base
= buf
;
1671 iov_iter_init(i
, rw
, iov
, 1, len
);
1674 EXPORT_SYMBOL(import_single_range
);
1676 int iov_iter_for_each_range(struct iov_iter
*i
, size_t bytes
,
1677 int (*f
)(struct kvec
*vec
, void *context
),
1685 iterate_all_kinds(i
, bytes
, v
, -EINVAL
, ({
1686 w
.iov_base
= kmap(v
.bv_page
) + v
.bv_offset
;
1687 w
.iov_len
= v
.bv_len
;
1688 err
= f(&w
, context
);
1692 err
= f(&w
, context
);})
1696 EXPORT_SYMBOL(iov_iter_for_each_range
);