io_uring/kbuf: hold io_buffer_list reference over mmap

commit 561e4f9451d65fc2f7eef564e0064373e3019793 upstream.

If we look up the kbuf, ensure that it doesn't get unregistered until
after we're done with it. Since we're inside mmap, we cannot safely use
the io_uring lock. Rely on the fact that we can lookup the buffer list
under RCU now and grab a reference to it, preventing it from being
unregistered until we're done with it. The lookup returns the
io_buffer_list directly with it referenced.

Cc: stable@vger.kernel.org # v6.4+
Fixes: 5cf4f52e6d8a ("io_uring: free io_buffer_list entries via RCU")
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 233e35dd72452c1c0341b04f2928c082973bb5f6..2c0a9a98272ca0e2eb005c2522cc99d590f0ba67 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -3429,14 +3429,15 @@ static void *io_uring_validate_mmap_request(struct file *file,
                ptr = ctx->sq_sqes;
                break;
        case IORING_OFF_PBUF_RING: {
+               struct io_buffer_list *bl;
                unsigned int bgid;
 
                bgid = (offset & ~IORING_OFF_MMAP_MASK) >> IORING_OFF_PBUF_SHIFT;
-               rcu_read_lock();
-               ptr = io_pbuf_get_address(ctx, bgid);
-               rcu_read_unlock();
-               if (!ptr)
-                       return ERR_PTR(-EINVAL);
+               bl = io_pbuf_get_bl(ctx, bgid);
+               if (IS_ERR(bl))
+                       return bl;
+               ptr = bl->buf_ring;
+               io_put_bl(ctx, bl);
                break;
                }
        default:

diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index f7d1dcc8ae1d721d816f7eac6f299e823abd5bb3..26a00920042c45f43d596a7721f417852a79c1d2 100644 (file)
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -273,7 +273,7 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx,
        return i;
 }
 
-static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl)
+void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl)
 {
        if (atomic_dec_and_test(&bl->refs)) {
                __io_remove_buffers(ctx, bl, -1U);
@@ -689,16 +689,35 @@ int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
        return 0;
 }
 
-void *io_pbuf_get_address(struct io_ring_ctx *ctx, unsigned long bgid)
+struct io_buffer_list *io_pbuf_get_bl(struct io_ring_ctx *ctx,
+                                     unsigned long bgid)
 {
        struct io_buffer_list *bl;
+       bool ret;
 
-       bl = __io_buffer_get_list(ctx, bgid);
-
-       if (!bl || !bl->is_mmap)
-               return NULL;
-
-       return bl->buf_ring;
+       /*
+        * We have to be a bit careful here - we're inside mmap and cannot grab
+        * the uring_lock. This means the buffer_list could be simultaneously
+        * going away, if someone is trying to be sneaky. Look it up under rcu
+        * so we know it's not going away, and attempt to grab a reference to
+        * it. If the ref is already zero, then fail the mapping. If successful,
+        * the caller will call io_put_bl() to drop the the reference at at the
+        * end. This may then safely free the buffer_list (and drop the pages)
+        * at that point, vm_insert_pages() would've already grabbed the
+        * necessary vma references.
+        */
+       rcu_read_lock();
+       bl = xa_load(&ctx->io_bl_xa, bgid);
+       /* must be a mmap'able buffer ring and have pages */
+       ret = false;
+       if (bl && bl->is_mmap)
+               ret = atomic_inc_not_zero(&bl->refs);
+       rcu_read_unlock();
+
+       if (ret)
+               return bl;
+
+       return ERR_PTR(-EINVAL);
 }
 
 /*

diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h
index b5c45b1d148649df29f352dc0abbe9ab87e24e5f..8d7929369501d2a884cdbab57e758bbc9abd809c 100644 (file)
--- a/io_uring/kbuf.h
+++ b/io_uring/kbuf.h
@@ -60,7 +60,9 @@ unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags);
 
 void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
 
-void *io_pbuf_get_address(struct io_ring_ctx *ctx, unsigned long bgid);
+void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl);
+struct io_buffer_list *io_pbuf_get_bl(struct io_ring_ctx *ctx,
+                                     unsigned long bgid);
 
 static inline void io_kbuf_recycle_ring(struct io_kiocb *req)
 {