io_uring/rsrc: store folio shift and mask into imu

Store the folio shift and folio mask into imu struct and use it in
iov_iter adjust, as we will have non PAGE_SIZE'd chunks if a
multi-hugepage buffer get coalesced.

Signed-off-by: Chenliang Li <cliang01.li@samsung.com>
Reviewed-by: Anuj Gupta <anuj20.g@samsung.com>
Reviewed-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/20240731090133.4106-2-cliang01.li@samsung.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index a860516bf4484071d2677d9134010782010ce78e..64152dc6f293363fc76c8895a511bf806f35386b 100644 (file)
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -915,6 +915,8 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
        imu->ubuf = (unsigned long) iov->iov_base;
        imu->ubuf_end = imu->ubuf + iov->iov_len;
        imu->nr_bvecs = nr_pages;
+       imu->folio_shift = PAGE_SHIFT;
+       imu->folio_mask = PAGE_MASK;
        *pimu = imu;
        ret = 0;
 
@@ -1031,23 +1033,18 @@ int io_import_fixed(int ddir, struct iov_iter *iter,
                 * we know that:
                 *
                 * 1) it's a BVEC iter, we set it up
-                * 2) all bvecs are PAGE_SIZE in size, except potentially the
+                * 2) all bvecs are the same in size, except potentially the
                 *    first and last bvec
                 *
                 * So just find our index, and adjust the iterator afterwards.
                 * If the offset is within the first bvec (or the whole first
                 * bvec, just use iov_iter_advance(). This makes it easier
                 * since we can just skip the first segment, which may not
-                * be PAGE_SIZE aligned.
+                * be folio_size aligned.
                 */
                const struct bio_vec *bvec = imu->bvec;
 
                if (offset < bvec->bv_len) {
-                       /*
-                        * Note, huge pages buffers consists of one large
-                        * bvec entry and should always go this way. The other
-                        * branch doesn't expect non PAGE_SIZE'd chunks.
-                        */
                        iter->bvec = bvec;
                        iter->count -= offset;
                        iter->iov_offset = offset;
@@ -1056,12 +1053,12 @@ int io_import_fixed(int ddir, struct iov_iter *iter,
 
                        /* skip first vec */
                        offset -= bvec->bv_len;
-                       seg_skip = 1 + (offset >> PAGE_SHIFT);
+                       seg_skip = 1 + (offset >> imu->folio_shift);
 
                        iter->bvec = bvec + seg_skip;
                        iter->nr_segs -= seg_skip;
                        iter->count -= bvec->bv_len + offset;
-                       iter->iov_offset = offset & ~PAGE_MASK;
+                       iter->iov_offset = offset & ~imu->folio_mask;
                }
        }
 

diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index c032ca3436ca99a531059090d36997a6130678cc..ee77e53328bfa30d1e2d817de923f38f365d10c7 100644 (file)
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -46,7 +46,9 @@ struct io_mapped_ubuf {
        u64             ubuf;
        u64             ubuf_end;
        unsigned int    nr_bvecs;
+       unsigned int    folio_shift;
        unsigned long   acct_pages;
+       unsigned long   folio_mask;
        struct bio_vec  bvec[] __counted_by(nr_bvecs);
 };