io_uring: run local task_work from ring exit IOPOLL reaping

In preparation for needing to shift NVMe passthrough to always use
task_work for polled IO completions, ensure that those are suitably
run at exit time. See commit:

9ce6c9875f3e ("nvme: always punt polled uring_cmd end_io work to task_work")

for details on why that is necessary.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 4e32f808d07df2e683a9b53e97f5a02c420b8cb1..5111ec040c53424e815072cd062a64bfbaf52956 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1523,6 +1523,9 @@ static __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
                }
        }
        mutex_unlock(&ctx->uring_lock);
+
+       if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+               io_move_task_work_from_local(ctx);
 }
 
 static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events)