io_uring: introduce type alias for io_tw_state

In preparation for changing how io_tw_state is passed, introduce a type
alias io_tw_token_t for struct io_tw_state *. This allows for changing
the representation in one place, without having to update the many
functions that just forward their struct io_tw_state * argument.

Also add a comment to struct io_tw_state to explain its purpose.

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
Link: https://lore.kernel.org/r/20250217022511.1150145-1-csander@purestorage.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index e2fef264ff8b8efceb1cc66abd7fca362286ce84..ea4694ee9d199b669ac01e6da392929edf295100 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -436,8 +436,15 @@ struct io_ring_ctx {
        struct io_mapped_region         param_region;
 };
 
+/*
+ * Token indicating function is called in task work context:
+ * ctx->uring_lock is held and any completions generated will be flushed.
+ * ONLY core io_uring.c should instantiate this struct.
+ */
 struct io_tw_state {
 };
+/* Alias to use in code that doesn't instantiate struct io_tw_state */
+typedef struct io_tw_state *io_tw_token_t;
 
 enum {
        REQ_F_FIXED_FILE_BIT    = IOSQE_FIXED_FILE_BIT,
@@ -563,7 +570,7 @@ enum {
        REQ_F_HAS_METADATA      = IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),
 };
 
-typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);
+typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);
 
 struct io_task_work {
        struct llist_node               node;

diff --git a/io_uring/futex.c b/io_uring/futex.c
index ede6279cadc6a65360125c2a8cb6cd78a30dbcb0..b7581766406ce0d3d97ec6a6beb4262b374aa869 100644 (file)
--- a/io_uring/futex.c
+++ b/io_uring/futex.c
@@ -44,30 +44,30 @@ void io_futex_cache_free(struct io_ring_ctx *ctx)
        io_alloc_cache_free(&ctx->futex_cache, kfree);
 }
 
-static void __io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)
+static void __io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)
 {
        req->async_data = NULL;
        hlist_del_init(&req->hash_node);
-       io_req_task_complete(req, ts);
+       io_req_task_complete(req, tw);
 }
 
-static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)
+static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)
 {
        struct io_futex_data *ifd = req->async_data;
        struct io_ring_ctx *ctx = req->ctx;
 
-       io_tw_lock(ctx, ts);
+       io_tw_lock(ctx, tw);
        if (!io_alloc_cache_put(&ctx->futex_cache, ifd))
                kfree(ifd);
-       __io_futex_complete(req, ts);
+       __io_futex_complete(req, tw);
 }
 
-static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)
+static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw)
 {
        struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
        struct futex_vector *futexv = req->async_data;
 
-       io_tw_lock(req->ctx, ts);
+       io_tw_lock(req->ctx, tw);
 
        if (!iof->futexv_unqueued) {
                int res;
@@ -79,7 +79,7 @@ static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)
 
        kfree(req->async_data);
        req->flags &= ~REQ_F_ASYNC_DATA;
-       __io_futex_complete(req, ts);
+       __io_futex_complete(req, tw);
 }
 
 static bool io_futexv_claim(struct io_futex *iof)

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 4a0944a57d96309e60d2f740a118df307eb744b1..b44ff88717258faa55e8e81c7d69c501c76ae61d 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -543,7 +543,7 @@ static void io_queue_iowq(struct io_kiocb *req)
                io_queue_linked_timeout(link);
 }
 
-static void io_req_queue_iowq_tw(struct io_kiocb *req, struct io_tw_state *ts)
+static void io_req_queue_iowq_tw(struct io_kiocb *req, io_tw_token_t tw)
 {
        io_queue_iowq(req);
 }
@@ -1022,7 +1022,7 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
        return nxt;
 }
 
-static void ctx_flush_and_put(struct io_ring_ctx *ctx, struct io_tw_state *ts)
+static void ctx_flush_and_put(struct io_ring_ctx *ctx, io_tw_token_t tw)
 {
        if (!ctx)
                return;
@@ -1277,7 +1277,7 @@ static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events,
 }
 
 static int __io_run_local_work_loop(struct llist_node **node,
-                                   struct io_tw_state *ts,
+                                   io_tw_token_t tw,
                                    int events)
 {
        int ret = 0;
@@ -1288,7 +1288,7 @@ static int __io_run_local_work_loop(struct llist_node **node,
                                                    io_task_work.node);
                INDIRECT_CALL_2(req->io_task_work.func,
                                io_poll_task_func, io_req_rw_complete,
-                               req, ts);
+                               req, tw);
                *node = next;
                if (++ret >= events)
                        break;
@@ -1297,7 +1297,7 @@ static int __io_run_local_work_loop(struct llist_node **node,
        return ret;
 }
 
-static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,
+static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw,
                               int min_events, int max_events)
 {
        struct llist_node *node;
@@ -1310,7 +1310,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,
                atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
 again:
        min_events -= ret;
-       ret = __io_run_local_work_loop(&ctx->retry_llist.first, ts, max_events);
+       ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events);
        if (ctx->retry_llist.first)
                goto retry_done;
 
@@ -1319,7 +1319,7 @@ again:
         * running the pending items.
         */
        node = llist_reverse_order(llist_del_all(&ctx->work_llist));
-       ret += __io_run_local_work_loop(&node, ts, max_events - ret);
+       ret += __io_run_local_work_loop(&node, tw, max_events - ret);
        ctx->retry_llist.first = node;
        loops++;
 
@@ -1357,15 +1357,15 @@ static int io_run_local_work(struct io_ring_ctx *ctx, int min_events,
        return ret;
 }
 
-static void io_req_task_cancel(struct io_kiocb *req, struct io_tw_state *ts)
+static void io_req_task_cancel(struct io_kiocb *req, io_tw_token_t tw)
 {
-       io_tw_lock(req->ctx, ts);
+       io_tw_lock(req->ctx, tw);
        io_req_defer_failed(req, req->cqe.res);
 }
 
-void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts)
+void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw)
 {
-       io_tw_lock(req->ctx, ts);
+       io_tw_lock(req->ctx, tw);
        if (unlikely(io_should_terminate_tw()))
                io_req_defer_failed(req, -EFAULT);
        else if (req->flags & REQ_F_FORCE_ASYNC)
@@ -1583,7 +1583,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
        return 0;
 }
 
-void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts)
+void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw)
 {
        io_req_complete_defer(req);
 }
@@ -1763,9 +1763,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
        return ret;
 }
 
-int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts)
+int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw)
 {
-       io_tw_lock(req->ctx, ts);
+       io_tw_lock(req->ctx, tw);
        return io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_MULTISHOT|
                                 IO_URING_F_COMPLETE_DEFER);
 }

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 85bc8f76ca19092c21aba6df8d50475179c1f228..6c46d9cdd7aa828d08dae95d9a309f552bb1ac4f 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -90,9 +90,9 @@ void io_req_task_work_add_remote(struct io_kiocb *req, struct io_ring_ctx *ctx,
                                 unsigned flags);
 bool io_alloc_async_data(struct io_kiocb *req);
 void io_req_task_queue(struct io_kiocb *req);
-void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts);
+void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw);
 void io_req_task_queue_fail(struct io_kiocb *req, int ret);
-void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts);
+void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw);
 struct llist_node *io_handle_tw_list(struct llist_node *node, unsigned int *count, unsigned int max_entries);
 struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, unsigned int *count);
 void tctx_task_work(struct callback_head *cb);
@@ -104,7 +104,7 @@ int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file,
                                     int start, int end);
 void io_req_queue_iowq(struct io_kiocb *req);
 
-int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts);
+int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw);
 int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);
 int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);
 void __io_submit_flush_completions(struct io_ring_ctx *ctx);
@@ -376,7 +376,7 @@ static inline bool io_task_work_pending(struct io_ring_ctx *ctx)
        return task_work_pending(current) || io_local_work_pending(ctx);
 }
 
-static inline void io_tw_lock(struct io_ring_ctx *ctx, struct io_tw_state *ts)
+static inline void io_tw_lock(struct io_ring_ctx *ctx, io_tw_token_t tw)
 {
        lockdep_assert_held(&ctx->uring_lock);
 }

diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c
index 7e6f68e911f10cdec35cc8b0ca49e569fbdd7856..0bbcbbcdebfd90c8bfddf69cbb31f641e7310326 100644 (file)
--- a/io_uring/msg_ring.c
+++ b/io_uring/msg_ring.c
@@ -71,7 +71,7 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)
        return target_ctx->task_complete;
 }
 
-static void io_msg_tw_complete(struct io_kiocb *req, struct io_tw_state *ts)
+static void io_msg_tw_complete(struct io_kiocb *req, io_tw_token_t tw)
 {
        struct io_ring_ctx *ctx = req->ctx;
 

diff --git a/io_uring/notif.c b/io_uring/notif.c
index ee3a33510b3c2ad84b60962377e75ed1e1910e86..7bd92538dccbc7e87742603f2c4b7caa606fe056 100644 (file)
--- a/io_uring/notif.c
+++ b/io_uring/notif.c
@@ -11,7 +11,7 @@
 
 static const struct ubuf_info_ops io_ubuf_ops;
 
-static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)
+static void io_notif_tw_complete(struct io_kiocb *notif, io_tw_token_t tw)
 {
        struct io_notif_data *nd = io_notif_to_data(notif);
 
@@ -29,7 +29,7 @@ static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)
                }
 
                nd = nd->next;
-               io_req_task_complete(notif, ts);
+               io_req_task_complete(notif, tw);
        } while (nd);
 }
 

diff --git a/io_uring/poll.c b/io_uring/poll.c
index bb1c0cd4f809ae05ba690ec092a87bdd0f70a355..176854882ba661fb9c6406646eab17ce87c186d3 100644 (file)
--- a/io_uring/poll.c
+++ b/io_uring/poll.c
@@ -220,7 +220,7 @@ static inline void io_poll_execute(struct io_kiocb *req, int res)
  * req->cqe.res. IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot
  * poll and that the result is stored in req->cqe.
  */
-static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
+static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
 {
        int v;
 
@@ -288,7 +288,7 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
                                return IOU_POLL_REMOVE_POLL_USE_RES;
                        }
                } else {
-                       int ret = io_poll_issue(req, ts);
+                       int ret = io_poll_issue(req, tw);
                        if (ret == IOU_STOP_MULTISHOT)
                                return IOU_POLL_REMOVE_POLL_USE_RES;
                        else if (ret == IOU_REQUEUE)
@@ -311,11 +311,11 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
        return IOU_POLL_NO_ACTION;
 }
 
-void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
+void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw)
 {
        int ret;
 
-       ret = io_poll_check_events(req, ts);
+       ret = io_poll_check_events(req, tw);
        if (ret == IOU_POLL_NO_ACTION) {
                io_kbuf_recycle(req, 0);
                return;
@@ -335,7 +335,7 @@ void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
                        poll = io_kiocb_to_cmd(req, struct io_poll);
                        req->cqe.res = mangle_poll(req->cqe.res & poll->events);
                } else if (ret == IOU_POLL_REISSUE) {
-                       io_req_task_submit(req, ts);
+                       io_req_task_submit(req, tw);
                        return;
                } else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) {
                        req->cqe.res = ret;
@@ -343,14 +343,14 @@ void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
                }
 
                io_req_set_res(req, req->cqe.res, 0);
-               io_req_task_complete(req, ts);
+               io_req_task_complete(req, tw);
        } else {
-               io_tw_lock(req->ctx, ts);
+               io_tw_lock(req->ctx, tw);
 
                if (ret == IOU_POLL_REMOVE_POLL_USE_RES)
-                       io_req_task_complete(req, ts);
+                       io_req_task_complete(req, tw);
                else if (ret == IOU_POLL_DONE || ret == IOU_POLL_REISSUE)
-                       io_req_task_submit(req, ts);
+                       io_req_task_submit(req, tw);
                else
                        io_req_defer_failed(req, ret);
        }

diff --git a/io_uring/poll.h b/io_uring/poll.h
index 04ede93113dc72b888a7baec1966bf64dc6a8148..27e2db2ed4aee52b972202074377b68fbde868c3 100644 (file)
--- a/io_uring/poll.h
+++ b/io_uring/poll.h
@@ -1,5 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
+#include <linux/io_uring_types.h>
+
 #define IO_POLL_ALLOC_CACHE_MAX 32
 
 enum {
@@ -43,4 +45,4 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags);
 bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
                        bool cancel_all);
 
-void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts);
+void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw);

diff --git a/io_uring/rw.c b/io_uring/rw.c
index 7aa1e4c9f64a31753ed4ce29ef873aff39804086..16f12f94943f7f33783d4665d4dc0bab82734c9d 100644 (file)
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -511,7 +511,7 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res)
        return res;
 }
 
-void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts)
+void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw)
 {
        struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
        struct kiocb *kiocb = &rw->kiocb;
@@ -528,7 +528,7 @@ void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts)
                req->cqe.flags |= io_put_kbuf(req, req->cqe.res, 0);
 
        io_req_rw_cleanup(req, 0);
-       io_req_task_complete(req, ts);
+       io_req_task_complete(req, tw);
 }
 
 static void io_complete_rw(struct kiocb *kiocb, long res)

diff --git a/io_uring/rw.h b/io_uring/rw.h
index eaa59bd64870973429a42be644c286eec6e3dd34..a45e0c71b59d6e2ac9e2444e7928ff484ec559b0 100644 (file)
--- a/io_uring/rw.h
+++ b/io_uring/rw.h
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 
+#include <linux/io_uring_types.h>
 #include <linux/pagemap.h>
 
 struct io_meta_state {
@@ -39,7 +40,7 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags);
 int io_write(struct io_kiocb *req, unsigned int issue_flags);
 void io_readv_writev_cleanup(struct io_kiocb *req);
 void io_rw_fail(struct io_kiocb *req);
-void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts);
+void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw);
 int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
 int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags);
 void io_rw_cache_free(const void *entry);

diff --git a/io_uring/timeout.c b/io_uring/timeout.c
index 48fc8cf707843bfbf94afa1dca7d91f4fb51fe8b..fec6ec7beb62e59a253c5861a0e050125baf456e 100644 (file)
--- a/io_uring/timeout.c
+++ b/io_uring/timeout.c
@@ -65,7 +65,7 @@ static inline bool io_timeout_finish(struct io_timeout *timeout,
 
 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer);
 
-static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)
+static void io_timeout_complete(struct io_kiocb *req, io_tw_token_t tw)
 {
        struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
        struct io_timeout_data *data = req->async_data;
@@ -82,7 +82,7 @@ static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)
                }
        }
 
-       io_req_task_complete(req, ts);
+       io_req_task_complete(req, tw);
 }
 
 static __cold bool io_flush_killed_timeouts(struct list_head *list, int err)
@@ -154,9 +154,9 @@ __cold void io_flush_timeouts(struct io_ring_ctx *ctx)
        io_flush_killed_timeouts(&list, 0);
 }
 
-static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts)
+static void io_req_tw_fail_links(struct io_kiocb *link, io_tw_token_t tw)
 {
-       io_tw_lock(link->ctx, ts);
+       io_tw_lock(link->ctx, tw);
        while (link) {
                struct io_kiocb *nxt = link->link;
                long res = -ECANCELED;
@@ -165,7 +165,7 @@ static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts)
                        res = link->cqe.res;
                link->link = NULL;
                io_req_set_res(link, res, 0);
-               io_req_task_complete(link, ts);
+               io_req_task_complete(link, tw);
                link = nxt;
        }
 }
@@ -312,7 +312,7 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
        return 0;
 }
 
-static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts)
+static void io_req_task_link_timeout(struct io_kiocb *req, io_tw_token_t tw)
 {
        struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
        struct io_kiocb *prev = timeout->prev;
@@ -330,11 +330,11 @@ static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *t
                        ret = -ECANCELED;
                }
                io_req_set_res(req, ret ?: -ETIME, 0);
-               io_req_task_complete(req, ts);
+               io_req_task_complete(req, tw);
                io_put_req(prev);
        } else {
                io_req_set_res(req, -ETIME, 0);
-               io_req_task_complete(req, ts);
+               io_req_task_complete(req, tw);
        }
 }
 

diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index e6701b7aa14743547ad91c429fd9d12df5314564..8bdf2c9b3fef9676ccc189bd5f1df742455997e4 100644 (file)
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -102,7 +102,7 @@ void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
 }
 EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable);
 
-static void io_uring_cmd_work(struct io_kiocb *req, struct io_tw_state *ts)
+static void io_uring_cmd_work(struct io_kiocb *req, io_tw_token_t tw)
 {
        struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
        unsigned int flags = IO_URING_F_COMPLETE_DEFER;

diff --git a/io_uring/waitid.c b/io_uring/waitid.c
index 5c443e5f6d9286d8ffc22d6a8af59f92163567b6..347b8f53efa7da3051dc7e382df9e7559d9d60c6 100644 (file)
--- a/io_uring/waitid.c
+++ b/io_uring/waitid.c
@@ -16,7 +16,7 @@
 #include "waitid.h"
 #include "../kernel/exit.h"
 
-static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts);
+static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw);
 
 #define IO_WAITID_CANCEL_FLAG  BIT(31)
 #define IO_WAITID_REF_MASK     GENMASK(30, 0)
@@ -185,13 +185,13 @@ static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req)
        return true;
 }
 
-static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts)
+static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw)
 {
        struct io_waitid_async *iwa = req->async_data;
        struct io_ring_ctx *ctx = req->ctx;
        int ret;
 
-       io_tw_lock(ctx, ts);
+       io_tw_lock(ctx, tw);
 
        ret = __do_wait(&iwa->wo);
 
@@ -221,7 +221,7 @@ static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts)
        }
 
        io_waitid_complete(req, ret);
-       io_req_task_complete(req, ts);
+       io_req_task_complete(req, tw);
 }
 
 static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode,