From: Jens Axboe Date: Wed, 10 Jun 2026 21:19:35 +0000 (-0600) Subject: io_uring: switch local task_work to a mpscq X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d46ab2c98ababa19b41a5709b6921d7b1add7f74;p=thirdparty%2Flinux.git io_uring: switch local task_work to a mpscq The local (DEFER_TASKRUN) task_work list is an llist, which is LIFO ordered, and hence __io_run_local_work() has to restore the right running order with an O(n) llist_reverse_order() pass first. On top of that, a batch that gets capped by max_events needs the leftover entries parked on a separate ->retry_llist, as they can't be pushed back to the shared list. Switch it to the FIFO mpscq. Adds are wait-free instead of a cmpxchg retry loop, entries are popped in queue order with no reversal pass, capping a run simply leaves the remainder on the queue, and ->retry_llist goes away entirely. The consumer cursor, ->work_head, lives with the rest of the ->uring_lock protected state rather than next to the queue, so that popping entries doesn't dirty the producer side cacheline. For low amounts of task_work, this ends up being a bit more efficient than the existing scheme. As an example of that, doing multishot receives for 8 clients has the following task_work overhead: 1.02% sock-test [kernel.kallsyms] [k] io_req_local_work_add 0.88% sock-test [kernel.kallsyms] [k] __io_run_local_work_loop 0.60% sock-test [kernel.kallsyms] [k] llist_reverse_order 0.14% sock-test [kernel.kallsyms] [k] __io_run_local_work 2.64% at ~46Gb/sec and after this change: 1.08% sock-test [kernel.kallsyms] [k] io_req_local_work_add 1.03% sock-test [kernel.kallsyms] [k] __io_run_local_work 2.11% at ~53Gb/sec which has less overhead even though that test run was faster. For a case of having 1024 clients on a single ring: 2.22% sock-test [kernel.kallsyms] [k] llist_reverse_order 0.84% sock-test [kernel.kallsyms] [k] __io_run_local_work_loop 0.42% sock-test [kernel.kallsyms] [k] io_req_local_work_add 0.02% sock-test [kernel.kallsyms] [k] __io_run_local_work 3.50% at ~24Gb/sec we start to see the llist reversing taking a considerable amount of time, and the total add+run task_work overhead is around 3.5%. After the change: 0.90% sock-test [kernel.kallsyms] [k] __io_run_local_work 0.42% sock-test [kernel.kallsyms] [k] io_req_local_work_add 1.32% at ~26Gb/sec most of that overhead is gone, and performance is better as well. Caleb Sander Mateos reports that it improves the performance of a ublk 4kb workload by 4% [1], while testing v1 of this patchset. [1] https://lore.kernel.org/io-uring/CADUfDZr-MMYBaP-e+y9+xuRhuiunO2sBTUCmwZyd7AgT8sVtiQ@mail.gmail.com/ Signed-off-by: Jens Axboe --- diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 85e12b4884a5d..3e07c7059d7b2 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -360,6 +360,14 @@ struct io_ring_ctx { bool poll_multi_queue; struct list_head iopoll_list; + /* + * Consumer cursor for ->work_list, protected by ->uring_lock. + * Deliberately kept away from the producer side of the queue, + * as it's written for every popped entry, and the producer + * cacheline is contended enough as it is. + */ + struct llist_node *work_head; + struct io_file_table file_table; struct io_rsrc_data buf_table; struct io_alloc_cache node_cache; @@ -417,8 +425,7 @@ struct io_ring_ctx { */ struct { struct io_rings __rcu *rings_rcu; - struct llist_head work_llist; - struct llist_head retry_llist; + struct mpscq work_list; unsigned long check_cq; atomic_t cq_wait_nr; atomic_t cq_timeouts; @@ -742,8 +749,6 @@ struct io_kiocb { */ u16 buf_index; - unsigned nr_tw; - /* REQ_F_* flags */ io_req_flags_t flags; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 02c02e14f3926..0809fc70c91d6 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -280,7 +280,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->defer_list); INIT_LIST_HEAD(&ctx->timeout_list); INIT_LIST_HEAD(&ctx->ltimeout_list); - init_llist_head(&ctx->work_llist); + mpscq_init(&ctx->work_list, &ctx->work_head); INIT_LIST_HEAD(&ctx->tctx_list); mutex_init(&ctx->tctx_lock); ctx->submit_state.free_list.next = NULL; diff --git a/io_uring/tw.c b/io_uring/tw.c index f4335c8d50d93..79feeb4f671a1 100644 --- a/io_uring/tw.c +++ b/io_uring/tw.c @@ -14,6 +14,7 @@ #include "rw.h" #include "eventfd.h" #include "wait.h" +#include "mpscq.h" void io_fallback_req_func(struct work_struct *work) { @@ -170,11 +171,7 @@ static void io_ctx_mark_taskrun(struct io_ring_ctx *ctx) void io_req_local_work_add(struct io_kiocb *req, unsigned flags) { struct io_ring_ctx *ctx = req->ctx; - unsigned nr_wait, nr_tw, nr_tw_prev; - struct llist_node *head; - - /* See comment above IO_CQ_WAKE_INIT */ - BUILD_BUG_ON(IO_CQ_WAKE_FORCE <= IORING_MAX_CQ_ENTRIES); + int nr_wait; /* * We don't know how many requests there are in the link and whether @@ -183,56 +180,45 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags) if (req->flags & IO_REQ_LINK_FLAGS) flags &= ~IOU_F_TWQ_LAZY_WAKE; - guard(rcu)(); - - head = READ_ONCE(ctx->work_llist.first); - do { - nr_tw_prev = 0; - if (head) { - struct io_kiocb *first_req = container_of(head, - struct io_kiocb, - io_task_work.node); - /* - * Might be executed at any moment, rely on - * SLAB_TYPESAFE_BY_RCU to keep it alive. - */ - nr_tw_prev = READ_ONCE(first_req->nr_tw); - } - - /* - * Theoretically, it can overflow, but that's fine as one of - * previous adds should've tried to wake the task. - */ - nr_tw = nr_tw_prev + 1; - if (!(flags & IOU_F_TWQ_LAZY_WAKE)) - nr_tw = IO_CQ_WAKE_FORCE; - - req->nr_tw = nr_tw; - req->io_task_work.node.next = head; - } while (!try_cmpxchg(&ctx->work_llist.first, &head, - &req->io_task_work.node)); - /* - * cmpxchg implies a full barrier, which pairs with the barrier - * in set_current_state() on the io_cqring_wait() side. It's used - * to ensure that either we see updated ->cq_wait_nr, or waiters - * going to sleep will observe the work added to the list, which - * is similar to the wait/wawke task state sync. + * The xchg() in mpscq_push() implies a full barrier, which pairs with + * the barrier in set_current_state() on the io_cqring_wait() side. This + * ensures that either we see the updated ->cq_wait_nr, or waiters going + * to sleep will observe the work added to the list, which is similar to + * the wait/wake task state sync. */ - - if (!head) { + if (mpscq_push(&ctx->work_list, &req->io_task_work.node)) { io_ctx_mark_taskrun(ctx); if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD) io_eventfd_signal(ctx, false); } + /* + * No one is waiting (IO_CQ_WAKE_INIT), or this cycle's wake up has + * already been issued (zero or negative, see below). + */ nr_wait = atomic_read(&ctx->cq_wait_nr); - /* not enough or no one is waiting */ - if (nr_tw < nr_wait) + if (nr_wait <= 0) return; - /* the previous add has already woken it up */ - if (nr_tw_prev >= nr_wait) + if (flags & IOU_F_TWQ_LAZY_WAKE) { + /* + * ->cq_wait_nr counts down the number of lazy adds, once it + * hits zero we're good to wake the waiter. A producer that + * gets delayed between pushing its entry and getting here + * may count down a later wait cycle. That's OK, it'll be an + * early wake, not a lost one. + */ + if (!atomic_dec_and_test(&ctx->cq_wait_nr)) + return; + } else if (atomic_xchg(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT) <= 0) { + /* + * Potentially raced with lazy add, claim the wake. A value + * <= 0 means a lazy add hit zero or another forced add + * claimed IO_CQ_WAKE_INIT. Either way, the wake up for this + * wait cycle has already been done. + */ return; + } wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE); } @@ -273,21 +259,27 @@ void io_req_task_work_add_remote(struct io_kiocb *req, unsigned flags) void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) { - struct llist_node *node; + struct llist_node *node, *first = NULL, **tail = &first; /* - * Running the work items may utilize ->retry_llist as a means - * for capping the number of task_work entries run at the same - * time. But that list can potentially race with moving the work - * from here, if the task is exiting. As any normal task_work - * running holds ->uring_lock already, just guard this slow path - * with ->uring_lock to avoid racing on ->retry_llist. + * The work list consumer side is serialized by ->uring_lock, see + * __io_run_local_work(). Grab it to guard against racing with normal + * task_work running, as the task may be exiting. */ guard(mutex)(&ctx->uring_lock); - node = llist_del_all(&ctx->work_llist); - __io_fallback_tw(node, false); - node = llist_del_all(&ctx->retry_llist); - __io_fallback_tw(node, false); + + while (!mpscq_empty(&ctx->work_list)) { + node = mpscq_pop(&ctx->work_list, &ctx->work_head); + if (!node) { + /* a producer is mid-push, wait for it to link */ + cpu_relax(); + continue; + } + *tail = node; + tail = &node->next; + } + *tail = NULL; + __io_fallback_tw(first, false); } static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events, @@ -302,22 +294,23 @@ static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events, return false; } -static int __io_run_local_work_loop(struct llist_node **node, +static int __io_run_local_work_loop(struct io_ring_ctx *ctx, io_tw_token_t tw, int events) { int ret = 0; - while (*node) { - struct llist_node *next = (*node)->next; - struct io_kiocb *req = container_of(*node, struct io_kiocb, - io_task_work.node); + while (ret < events) { + struct llist_node *node = mpscq_pop(&ctx->work_list, &ctx->work_head); + struct io_kiocb *req; + + if (!node) + break; + req = container_of(node, struct io_kiocb, io_task_work.node); INDIRECT_CALL_2(req->io_task_work.func, io_poll_task_func, io_req_rw_complete, (struct io_tw_req){req}, tw); - *node = next; - if (++ret >= events) - break; + ret++; } return ret; @@ -326,7 +319,6 @@ static int __io_run_local_work_loop(struct llist_node **node, static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw, int min_events, int max_events) { - struct llist_node *node; unsigned int loops = 0; int ret = 0; @@ -335,24 +327,21 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw, if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); again: - tw.cancel = io_should_terminate_tw(ctx); - min_events -= ret; - ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events); - if (ctx->retry_llist.first) - goto retry_done; - /* - * llists are in reverse order, flip it back the right way before - * running the pending items. + * If the last loop made no progress while work is still pending, + * a producer has published a node but hasn't linked it into the + * queue yet (see mpscq_pop()). Give it a chance to finish rather + * than spinning on the queue. */ - node = llist_reverse_order(llist_del_all(&ctx->work_llist)); - ret += __io_run_local_work_loop(&node, tw, max_events - ret); - ctx->retry_llist.first = node; + if (unlikely(loops && !ret)) + cond_resched(); + tw.cancel = io_should_terminate_tw(ctx); + min_events -= ret; + ret = __io_run_local_work_loop(ctx, tw, max_events); loops++; if (io_run_local_work_continue(ctx, ret, min_events)) goto again; -retry_done: io_submit_flush_completions(ctx); if (io_run_local_work_continue(ctx, ret, min_events)) goto again; diff --git a/io_uring/tw.h b/io_uring/tw.h index 415e330fabdeb..f42db5fdbdede 100644 --- a/io_uring/tw.h +++ b/io_uring/tw.h @@ -6,6 +6,8 @@ #include #include +#include "mpscq.h" + #define IO_LOCAL_TW_DEFAULT_MAX 20 /* @@ -89,7 +91,7 @@ static inline int io_run_task_work(void) static inline bool io_local_work_pending(struct io_ring_ctx *ctx) { - return !llist_empty(&ctx->work_llist) || !llist_empty(&ctx->retry_llist); + return !mpscq_empty(&ctx->work_list); } static inline bool io_task_work_pending(struct io_ring_ctx *ctx) diff --git a/io_uring/wait.c b/io_uring/wait.c index ec01e78a216d6..c5fc34d2ce97d 100644 --- a/io_uring/wait.c +++ b/io_uring/wait.c @@ -98,7 +98,7 @@ static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer) if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) { atomic_set(&ctx->cq_wait_nr, 1); smp_mb(); - if (!llist_empty(&ctx->work_llist)) + if (io_local_work_pending(ctx)) goto out_wake; } diff --git a/io_uring/wait.h b/io_uring/wait.h index a4274b137f817..b7b9c46b1b013 100644 --- a/io_uring/wait.h +++ b/io_uring/wait.h @@ -5,12 +5,14 @@ #include /* - * No waiters. It's larger than any valid value of the tw counter - * so that tests against ->cq_wait_nr would fail and skip wake_up(). + * ->cq_wait_nr is armed with the number of lazy task_work adds the waiter + * still needs, and counted down by the add side, with the add reaching zero + * issuing the (single) wake up for this wait cycle. Zero and below means no + * wake up is to be issued: IO_CQ_WAKE_INIT when no task is waiting (also + * what a forced wake up resets it to when claiming one), zero once the + * countdown has fired. */ -#define IO_CQ_WAKE_INIT (-1U) -/* Forced wake up if there is a waiter regardless of ->cq_wait_nr */ -#define IO_CQ_WAKE_FORCE (IO_CQ_WAKE_INIT >> 1) +#define IO_CQ_WAKE_INIT (-1) struct ext_arg { size_t argsz;