io_uring/kbuf: unify legacy buf provision and removal

Combine IORING_OP_PROVIDE_BUFFERS and IORING_OP_REMOVE_BUFFERS
->issue(), so that we can deduplicate ring locking and list lookups.
This way we further reduce code for legacy provided buffers. Locking is
also separated from buffer related handling, which makes it a bit
simpler with label jumps.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/f61af131622ad4337c2fb9f7c453d5b0102c7b90.1747150490.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index df8aeb42e910618771137fd4e06bd6dd6075e49d..823e7eb15fb2a8deb68e209ea2d744d1d78162be 100644 (file)
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -450,30 +450,6 @@ int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
        return 0;
 }
 
-int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
-{
-       struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
-       struct io_ring_ctx *ctx = req->ctx;
-       struct io_buffer_list *bl;
-       int ret = 0;
-
-       io_ring_submit_lock(ctx, issue_flags);
-
-       ret = -ENOENT;
-       bl = io_buffer_get_list(ctx, p->bgid);
-       if (bl) {
-               ret = -EINVAL;
-               /* can't use provide/remove buffers command on mapped buffers */
-               if (!(bl->flags & IOBL_BUF_RING))
-                       ret = io_remove_buffers_legacy(ctx, bl, p->nbufs);
-       }
-       io_ring_submit_unlock(ctx, issue_flags);
-       if (ret < 0)
-               req_set_fail(req);
-       io_req_set_res(req, ret, 0);
-       return IOU_OK;
-}
-
 int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
        unsigned long size, tmp_check;
@@ -535,37 +511,44 @@ static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf,
        return i ? 0 : -ENOMEM;
 }
 
-int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
+static int __io_manage_buffers_legacy(struct io_kiocb *req,
+                                       struct io_buffer_list *bl)
 {
        struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
-       struct io_ring_ctx *ctx = req->ctx;
-       struct io_buffer_list *bl;
-       int ret = 0;
-
-       io_ring_submit_lock(ctx, issue_flags);
+       int ret;
 
-       bl = io_buffer_get_list(ctx, p->bgid);
-       if (unlikely(!bl)) {
+       if (!bl) {
+               if (req->opcode != IORING_OP_PROVIDE_BUFFERS)
+                       return -ENOENT;
                bl = kzalloc(sizeof(*bl), GFP_KERNEL_ACCOUNT);
-               if (!bl) {
-                       ret = -ENOMEM;
-                       goto err;
-               }
+               if (!bl)
+                       return -ENOMEM;
+
                INIT_LIST_HEAD(&bl->buf_list);
-               ret = io_buffer_add_list(ctx, bl, p->bgid);
+               ret = io_buffer_add_list(req->ctx, bl, p->bgid);
                if (ret) {
                        kfree(bl);
-                       goto err;
+                       return ret;
                }
        }
-       /* can't add buffers via this command for a mapped buffer ring */
-       if (bl->flags & IOBL_BUF_RING) {
-               ret = -EINVAL;
-               goto err;
-       }
+       /* can't use provide/remove buffers command on mapped buffers */
+       if (bl->flags & IOBL_BUF_RING)
+               return -EINVAL;
+       if (req->opcode == IORING_OP_PROVIDE_BUFFERS)
+               return io_add_buffers(req->ctx, p, bl);
+       return io_remove_buffers_legacy(req->ctx, bl, p->nbufs);
+}
+
+int io_manage_buffers_legacy(struct io_kiocb *req, unsigned int issue_flags)
+{
+       struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf);
+       struct io_ring_ctx *ctx = req->ctx;
+       struct io_buffer_list *bl;
+       int ret;
 
-       ret = io_add_buffers(ctx, p, bl);
-err:
+       io_ring_submit_lock(ctx, issue_flags);
+       bl = io_buffer_get_list(ctx, p->bgid);
+       ret = __io_manage_buffers_legacy(req, bl);
        io_ring_submit_unlock(ctx, issue_flags);
 
        if (ret < 0)

diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h
index 0798a732e6cb0de925f6357789346ebe2cc30b57..4d2c209d1a4112d050bec6b1aa5be403da154176 100644 (file)
--- a/io_uring/kbuf.h
+++ b/io_uring/kbuf.h
@@ -66,10 +66,8 @@ int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg);
 void io_destroy_buffers(struct io_ring_ctx *ctx);
 
 int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
-int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags);
-
 int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
-int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags);
+int io_manage_buffers_legacy(struct io_kiocb *req, unsigned int issue_flags);
 
 int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
 int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);

diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index db36433c2294ef8a62542e45286723b4de8fb9cd..6e0882b051f93bae2974cc623678b199b39262e5 100644 (file)
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -333,13 +333,13 @@ const struct io_issue_def io_issue_defs[] = {
                .audit_skip             = 1,
                .iopoll                 = 1,
                .prep                   = io_provide_buffers_prep,
-               .issue                  = io_provide_buffers,
+               .issue                  = io_manage_buffers_legacy,
        },
        [IORING_OP_REMOVE_BUFFERS] = {
                .audit_skip             = 1,
                .iopoll                 = 1,
                .prep                   = io_remove_buffers_prep,
-               .issue                  = io_remove_buffers,
+               .issue                  = io_manage_buffers_legacy,
        },
        [IORING_OP_TEE] = {
                .needs_file             = 1,