1 From 09ab7eff38202159271534d2f5ad45526168f2a5 Mon Sep 17 00:00:00 2001
2 From: Jens Axboe <axboe@kernel.dk>
3 Date: Thu, 14 Mar 2024 10:45:07 -0600
4 Subject: io_uring/kbuf: get rid of lower BGID lists
6 From: Jens Axboe <axboe@kernel.dk>
8 commit 09ab7eff38202159271534d2f5ad45526168f2a5 upstream.
10 Just rely on the xarray for any kind of bgid. This simplifies things, and
11 it really doesn't bring us much, if anything.
13 Cc: stable@vger.kernel.org # v6.4+
14 Signed-off-by: Jens Axboe <axboe@kernel.dk>
15 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
17 include/linux/io_uring_types.h | 1
18 io_uring/io_uring.c | 2 -
19 io_uring/kbuf.c | 70 ++++-------------------------------------
20 3 files changed, 8 insertions(+), 65 deletions(-)
22 --- a/include/linux/io_uring_types.h
23 +++ b/include/linux/io_uring_types.h
24 @@ -250,7 +250,6 @@ struct io_ring_ctx {
26 struct io_submit_state submit_state;
28 - struct io_buffer_list *io_bl;
29 struct xarray io_bl_xa;
31 struct io_hash_table cancel_table_locked;
32 --- a/io_uring/io_uring.c
33 +++ b/io_uring/io_uring.c
34 @@ -343,7 +343,6 @@ static __cold struct io_ring_ctx *io_rin
36 kfree(ctx->cancel_table.hbs);
37 kfree(ctx->cancel_table_locked.hbs);
39 xa_destroy(&ctx->io_bl_xa);
42 @@ -2934,7 +2933,6 @@ static __cold void io_ring_ctx_free(stru
43 io_wq_put_hash(ctx->hash_map);
44 kfree(ctx->cancel_table.hbs);
45 kfree(ctx->cancel_table_locked.hbs);
47 xa_destroy(&ctx->io_bl_xa);
54 #define IO_BUFFER_LIST_BUF_PER_PAGE (PAGE_SIZE / sizeof(struct io_uring_buf))
56 -#define BGID_ARRAY 64
58 /* BIDs are addressed by a 16-bit field in a CQE */
59 #define MAX_BIDS_PER_BGID (1 << 16)
61 @@ -31,13 +29,9 @@ struct io_provide_buf {
65 -static struct io_buffer_list *__io_buffer_get_list(struct io_ring_ctx *ctx,
66 - struct io_buffer_list *bl,
68 +static inline struct io_buffer_list *__io_buffer_get_list(struct io_ring_ctx *ctx,
71 - if (bl && bgid < BGID_ARRAY)
74 return xa_load(&ctx->io_bl_xa, bgid);
77 @@ -53,7 +47,7 @@ static inline struct io_buffer_list *io_
79 lockdep_assert_held(&ctx->uring_lock);
81 - return __io_buffer_get_list(ctx, ctx->io_bl, bgid);
82 + return __io_buffer_get_list(ctx, bgid);
85 static int io_buffer_add_list(struct io_ring_ctx *ctx,
86 @@ -66,10 +60,6 @@ static int io_buffer_add_list(struct io_
89 smp_store_release(&bl->is_ready, 1);
91 - if (bgid < BGID_ARRAY)
94 return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL));
97 @@ -215,24 +205,6 @@ void __user *io_buffer_select(struct io_
101 -static __cold int io_init_bl_list(struct io_ring_ctx *ctx)
103 - struct io_buffer_list *bl;
106 - bl = kcalloc(BGID_ARRAY, sizeof(struct io_buffer_list), GFP_KERNEL);
110 - for (i = 0; i < BGID_ARRAY; i++) {
111 - INIT_LIST_HEAD(&bl[i].buf_list);
115 - smp_store_release(&ctx->io_bl, bl);
120 * Mark the given mapped range as free for reuse
122 @@ -305,13 +277,6 @@ void io_destroy_buffers(struct io_ring_c
124 struct io_buffer_list *bl;
128 - for (i = 0; i < BGID_ARRAY; i++) {
131 - __io_remove_buffers(ctx, &ctx->io_bl[i], -1U);
134 xa_for_each(&ctx->io_bl_xa, index, bl) {
135 xa_erase(&ctx->io_bl_xa, bl->bgid);
136 @@ -485,12 +450,6 @@ int io_provide_buffers(struct io_kiocb *
138 io_ring_submit_lock(ctx, issue_flags);
140 - if (unlikely(p->bgid < BGID_ARRAY && !ctx->io_bl)) {
141 - ret = io_init_bl_list(ctx);
146 bl = io_buffer_get_list(ctx, p->bgid);
148 bl = kzalloc(sizeof(*bl), GFP_KERNEL_ACCOUNT);
149 @@ -503,14 +462,9 @@ int io_provide_buffers(struct io_kiocb *
152 * Doesn't need rcu free as it was never visible, but
153 - * let's keep it consistent throughout. Also can't
154 - * be a lower indexed array group, as adding one
155 - * where lookup failed cannot happen.
156 + * let's keep it consistent throughout.
158 - if (p->bgid >= BGID_ARRAY)
159 - kfree_rcu(bl, rcu);
162 + kfree_rcu(bl, rcu);
166 @@ -675,12 +629,6 @@ int io_register_pbuf_ring(struct io_ring
167 if (reg.ring_entries >= 65536)
170 - if (unlikely(reg.bgid < BGID_ARRAY && !ctx->io_bl)) {
171 - int ret = io_init_bl_list(ctx);
176 bl = io_buffer_get_list(ctx, reg.bgid);
178 /* if mapped buffer ring OR classic exists, don't allow */
179 @@ -730,10 +678,8 @@ int io_unregister_pbuf_ring(struct io_ri
182 __io_remove_buffers(ctx, bl, -1U);
183 - if (bl->bgid >= BGID_ARRAY) {
184 - xa_erase(&ctx->io_bl_xa, bl->bgid);
185 - kfree_rcu(bl, rcu);
187 + xa_erase(&ctx->io_bl_xa, bl->bgid);
188 + kfree_rcu(bl, rcu);
192 @@ -741,7 +687,7 @@ void *io_pbuf_get_address(struct io_ring
194 struct io_buffer_list *bl;
196 - bl = __io_buffer_get_list(ctx, smp_load_acquire(&ctx->io_bl), bgid);
197 + bl = __io_buffer_get_list(ctx, bgid);
199 if (!bl || !bl->is_mmap)