]> git.ipfire.org Git - thirdparty/kernel/stable-queue.git/blob - queue-6.6/io_uring-kbuf-hold-io_buffer_list-reference-over-mmap.patch
Linux 6.1.85
[thirdparty/kernel/stable-queue.git] / queue-6.6 / io_uring-kbuf-hold-io_buffer_list-reference-over-mmap.patch
1 From 561e4f9451d65fc2f7eef564e0064373e3019793 Mon Sep 17 00:00:00 2001
2 From: Jens Axboe <axboe@kernel.dk>
3 Date: Tue, 2 Apr 2024 16:16:03 -0600
4 Subject: io_uring/kbuf: hold io_buffer_list reference over mmap
5
6 From: Jens Axboe <axboe@kernel.dk>
7
8 commit 561e4f9451d65fc2f7eef564e0064373e3019793 upstream.
9
10 If we look up the kbuf, ensure that it doesn't get unregistered until
11 after we're done with it. Since we're inside mmap, we cannot safely use
12 the io_uring lock. Rely on the fact that we can lookup the buffer list
13 under RCU now and grab a reference to it, preventing it from being
14 unregistered until we're done with it. The lookup returns the
15 io_buffer_list directly with it referenced.
16
17 Cc: stable@vger.kernel.org # v6.4+
18 Fixes: 5cf4f52e6d8a ("io_uring: free io_buffer_list entries via RCU")
19 Signed-off-by: Jens Axboe <axboe@kernel.dk>
20 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
21 ---
22 io_uring/io_uring.c | 11 ++++++-----
23 io_uring/kbuf.c | 31 +++++++++++++++++++++++++------
24 io_uring/kbuf.h | 4 +++-
25 3 files changed, 34 insertions(+), 12 deletions(-)
26
27 --- a/io_uring/io_uring.c
28 +++ b/io_uring/io_uring.c
29 @@ -3429,14 +3429,15 @@ static void *io_uring_validate_mmap_requ
30 ptr = ctx->sq_sqes;
31 break;
32 case IORING_OFF_PBUF_RING: {
33 + struct io_buffer_list *bl;
34 unsigned int bgid;
35
36 bgid = (offset & ~IORING_OFF_MMAP_MASK) >> IORING_OFF_PBUF_SHIFT;
37 - rcu_read_lock();
38 - ptr = io_pbuf_get_address(ctx, bgid);
39 - rcu_read_unlock();
40 - if (!ptr)
41 - return ERR_PTR(-EINVAL);
42 + bl = io_pbuf_get_bl(ctx, bgid);
43 + if (IS_ERR(bl))
44 + return bl;
45 + ptr = bl->buf_ring;
46 + io_put_bl(ctx, bl);
47 break;
48 }
49 default:
50 --- a/io_uring/kbuf.c
51 +++ b/io_uring/kbuf.c
52 @@ -273,7 +273,7 @@ static int __io_remove_buffers(struct io
53 return i;
54 }
55
56 -static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl)
57 +void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl)
58 {
59 if (atomic_dec_and_test(&bl->refs)) {
60 __io_remove_buffers(ctx, bl, -1U);
61 @@ -689,16 +689,35 @@ int io_unregister_pbuf_ring(struct io_ri
62 return 0;
63 }
64
65 -void *io_pbuf_get_address(struct io_ring_ctx *ctx, unsigned long bgid)
66 +struct io_buffer_list *io_pbuf_get_bl(struct io_ring_ctx *ctx,
67 + unsigned long bgid)
68 {
69 struct io_buffer_list *bl;
70 + bool ret;
71
72 - bl = __io_buffer_get_list(ctx, bgid);
73 + /*
74 + * We have to be a bit careful here - we're inside mmap and cannot grab
75 + * the uring_lock. This means the buffer_list could be simultaneously
76 + * going away, if someone is trying to be sneaky. Look it up under rcu
77 + * so we know it's not going away, and attempt to grab a reference to
78 + * it. If the ref is already zero, then fail the mapping. If successful,
79 + * the caller will call io_put_bl() to drop the the reference at at the
80 + * end. This may then safely free the buffer_list (and drop the pages)
81 + * at that point, vm_insert_pages() would've already grabbed the
82 + * necessary vma references.
83 + */
84 + rcu_read_lock();
85 + bl = xa_load(&ctx->io_bl_xa, bgid);
86 + /* must be a mmap'able buffer ring and have pages */
87 + ret = false;
88 + if (bl && bl->is_mmap)
89 + ret = atomic_inc_not_zero(&bl->refs);
90 + rcu_read_unlock();
91
92 - if (!bl || !bl->is_mmap)
93 - return NULL;
94 + if (ret)
95 + return bl;
96
97 - return bl->buf_ring;
98 + return ERR_PTR(-EINVAL);
99 }
100
101 /*
102 --- a/io_uring/kbuf.h
103 +++ b/io_uring/kbuf.h
104 @@ -60,7 +60,9 @@ unsigned int __io_put_kbuf(struct io_kio
105
106 void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
107
108 -void *io_pbuf_get_address(struct io_ring_ctx *ctx, unsigned long bgid);
109 +void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl);
110 +struct io_buffer_list *io_pbuf_get_bl(struct io_ring_ctx *ctx,
111 + unsigned long bgid);
112
113 static inline void io_kbuf_recycle_ring(struct io_kiocb *req)
114 {