io_uring: Fix race condition when sqp thread goes to sleep

commit 997135017716c33f3405e86cca5da9567b40a08e upstream.

If an asynchronous completion happens before the task is preparing
itself to wait and set its state to TASK_INTERRUPTIBLE, the completion
will not wake up the sqp thread.

Cc: stable@vger.kernel.org
Signed-off-by: Olivier Langlois <olivier@trillion01.com>
Reviewed-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/d1419dc32ec6a97b453bee34dc03fa6a02797142.1624473200.git.olivier@trillion01.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 8843f48ace27da7c5272ba3a3d5d09620723cedb..5cc371798f135a6a8c6db55947f0224b125d1512 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -6876,7 +6876,8 @@ static int io_sq_thread(void *data)
                }
 
                prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE);
-               if (!test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)) {
+               if (!test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) &&
+                   !io_run_task_work()) {
                        list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
                                io_ring_set_wakeup_flag(ctx);