io_uring/kbuf: introduce struct io_br_sel

Rather than return addresses directly from buffer selection, add a
struct around it. No functional changes in this patch, it's in
preparation for storing more buffer related information locally, rather
than in struct io_kiocb.

Link: https://lore.kernel.org/r/20250821020750.598432-7-axboe@kernel.dk
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index b8b2f6dee7547ffb80cae991df32ce9aeb830c87..61d9a8d439ba0d1d19045bd7ec3dd13836de01f9 100644 (file)
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -151,18 +151,18 @@ static int io_provided_buffers_select(struct io_kiocb *req, size_t *len,
        return 1;
 }
 
-static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,
-                                         struct io_buffer_list *bl,
-                                         unsigned int issue_flags)
+static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len,
+                                             struct io_buffer_list *bl,
+                                             unsigned int issue_flags)
 {
        struct io_uring_buf_ring *br = bl->buf_ring;
        __u16 tail, head = bl->head;
+       struct io_br_sel sel = { };
        struct io_uring_buf *buf;
-       void __user *ret;
 
        tail = smp_load_acquire(&br->tail);
        if (unlikely(tail == head))
-               return NULL;
+               return sel;
 
        if (head + 1 == tail)
                req->flags |= REQ_F_BL_EMPTY;
@@ -173,7 +173,7 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,
        req->flags |= REQ_F_BUFFER_RING | REQ_F_BUFFERS_COMMIT;
        req->buf_list = bl;
        req->buf_index = buf->bid;
-       ret = u64_to_user_ptr(buf->addr);
+       sel.addr = u64_to_user_ptr(buf->addr);
 
        if (issue_flags & IO_URING_F_UNLOCKED || !io_file_can_poll(req)) {
                /*
@@ -189,27 +189,27 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,
                io_kbuf_commit(req, bl, *len, 1);
                req->buf_list = NULL;
        }
-       return ret;
+       return sel;
 }
 
-void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
-                             unsigned buf_group, unsigned int issue_flags)
+struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len,
+                                 unsigned buf_group, unsigned int issue_flags)
 {
        struct io_ring_ctx *ctx = req->ctx;
+       struct io_br_sel sel = { };
        struct io_buffer_list *bl;
-       void __user *ret = NULL;
 
        io_ring_submit_lock(req->ctx, issue_flags);
 
        bl = io_buffer_get_list(ctx, buf_group);
        if (likely(bl)) {
                if (bl->flags & IOBL_BUF_RING)
-                       ret = io_ring_buffer_select(req, len, bl, issue_flags);
+                       sel = io_ring_buffer_select(req, len, bl, issue_flags);
                else
-                       ret = io_provided_buffer_select(req, len, bl);
+                       sel.addr = io_provided_buffer_select(req, len, bl);
        }
        io_ring_submit_unlock(req->ctx, issue_flags);
-       return ret;
+       return sel;
 }
 
 /* cap it at a reasonable 256, will be one page even for 4K */

diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h
index 58451ab4ab8ac9987dc3be45ce4b255f4059766d..e14a43b125c3d448c2a7965ca011454e3b23e681 100644 (file)
--- a/io_uring/kbuf.h
+++ b/io_uring/kbuf.h
@@ -62,8 +62,23 @@ struct buf_sel_arg {
        unsigned short partial_map;
 };
 
-void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
-                             unsigned buf_group, unsigned int issue_flags);
+/*
+ * Return value from io_buffer_list selection. Just returns the error or
+ * user address for now, will be extended to return the buffer list in the
+ * future.
+ */
+struct io_br_sel {
+       /*
+        * Some selection parts return the user address, others return an error.
+        */
+       union {
+               void __user *addr;
+               ssize_t val;
+       };
+};
+
+struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len,
+                                 unsigned buf_group, unsigned int issue_flags);
 int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg,
                      unsigned int issue_flags);
 int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg);

diff --git a/io_uring/net.c b/io_uring/net.c
index 54e13d1b0d0b30e3697dec29cb4f5daafccef2e9..12dcc21f2259ba3f4653723b2313e4d5ace9383e 100644 (file)
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -1035,22 +1035,22 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
 
 retry_multishot:
        if (io_do_buffer_select(req)) {
-               void __user *buf;
+               struct io_br_sel sel;
                size_t len = sr->len;
 
-               buf = io_buffer_select(req, &len, sr->buf_group, issue_flags);
-               if (!buf)
+               sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
+               if (!sel.addr)
                        return -ENOBUFS;
 
                if (req->flags & REQ_F_APOLL_MULTISHOT) {
-                       ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len);
+                       ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len);
                        if (ret) {
                                io_kbuf_recycle(req, req->buf_list, issue_flags);
                                return ret;
                        }
                }
 
-               iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, buf, len);
+               iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len);
        }
 
        kmsg->msg.msg_get_inq = 1;
@@ -1153,13 +1153,13 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg
                iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,
                                arg.out_len);
        } else {
-               void __user *buf;
+               struct io_br_sel sel;
 
                *len = sr->len;
-               buf = io_buffer_select(req, len, sr->buf_group, issue_flags);
-               if (!buf)
+               sel = io_buffer_select(req, len, sr->buf_group, issue_flags);
+               if (!sel.addr)
                        return -ENOBUFS;
-               sr->buf = buf;
+               sr->buf = sel.addr;
                sr->len = *len;
 map_ubuf:
                ret = import_ubuf(ITER_DEST, sr->buf, sr->len,

diff --git a/io_uring/rw.c b/io_uring/rw.c
index 7fe188872279412fb4009b414cd181f2833c815c..ef94cdde5f1f761fb16aa96d956ec6b5b10beb3a 100644 (file)
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -107,34 +107,35 @@ static int io_import_vec(int ddir, struct io_kiocb *req,
 }
 
 static int __io_import_rw_buffer(int ddir, struct io_kiocb *req,
-                            struct io_async_rw *io,
-                            unsigned int issue_flags)
+                                struct io_async_rw *io, struct io_br_sel *sel,
+                                unsigned int issue_flags)
 {
        const struct io_issue_def *def = &io_issue_defs[req->opcode];
        struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
-       void __user *buf = u64_to_user_ptr(rw->addr);
        size_t sqe_len = rw->len;
 
+       sel->addr = u64_to_user_ptr(rw->addr);
        if (def->vectored && !(req->flags & REQ_F_BUFFER_SELECT))
-               return io_import_vec(ddir, req, io, buf, sqe_len);
+               return io_import_vec(ddir, req, io, sel->addr, sqe_len);
 
        if (io_do_buffer_select(req)) {
-               buf = io_buffer_select(req, &sqe_len, io->buf_group, issue_flags);
-               if (!buf)
+               *sel = io_buffer_select(req, &sqe_len, io->buf_group, issue_flags);
+               if (!sel->addr)
                        return -ENOBUFS;
-               rw->addr = (unsigned long) buf;
+               rw->addr = (unsigned long) sel->addr;
                rw->len = sqe_len;
        }
-       return import_ubuf(ddir, buf, sqe_len, &io->iter);
+       return import_ubuf(ddir, sel->addr, sqe_len, &io->iter);
 }
 
 static inline int io_import_rw_buffer(int rw, struct io_kiocb *req,
                                      struct io_async_rw *io,
+                                     struct io_br_sel *sel,
                                      unsigned int issue_flags)
 {
        int ret;
 
-       ret = __io_import_rw_buffer(rw, req, io, issue_flags);
+       ret = __io_import_rw_buffer(rw, req, io, sel, issue_flags);
        if (unlikely(ret < 0))
                return ret;
 
@@ -306,10 +307,12 @@ static int __io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 
 static int io_rw_do_import(struct io_kiocb *req, int ddir)
 {
+       struct io_br_sel sel = { };
+
        if (io_do_buffer_select(req))
                return 0;
 
-       return io_import_rw_buffer(ddir, req, req->async_data, 0);
+       return io_import_rw_buffer(ddir, req, req->async_data, &sel, 0);
 }
 
 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
@@ -899,7 +902,8 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode, int rw_type)
        return 0;
 }
 
-static int __io_read(struct io_kiocb *req, unsigned int issue_flags)
+static int __io_read(struct io_kiocb *req, struct io_br_sel *sel,
+                    unsigned int issue_flags)
 {
        bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
        struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
@@ -913,7 +917,7 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags)
                if (unlikely(ret))
                        return ret;
        } else if (io_do_buffer_select(req)) {
-               ret = io_import_rw_buffer(ITER_DEST, req, io, issue_flags);
+               ret = io_import_rw_buffer(ITER_DEST, req, io, sel, issue_flags);
                if (unlikely(ret < 0))
                        return ret;
        }
@@ -1015,9 +1019,10 @@ done:
 
 int io_read(struct io_kiocb *req, unsigned int issue_flags)
 {
+       struct io_br_sel sel = { };
        int ret;
 
-       ret = __io_read(req, issue_flags);
+       ret = __io_read(req, &sel, issue_flags);
        if (ret >= 0)
                return kiocb_done(req, ret, issue_flags);
 
@@ -1027,6 +1032,7 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags)
 int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)
 {
        struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
+       struct io_br_sel sel = { };
        unsigned int cflags = 0;
        int ret;
 
@@ -1038,7 +1044,7 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)
 
        /* make it sync, multishot doesn't support async execution */
        rw->kiocb.ki_complete = NULL;
-       ret = __io_read(req, issue_flags);
+       ret = __io_read(req, &sel, issue_flags);
 
        /*
         * If we get -EAGAIN, recycle our buffer and just let normal poll