io_uring: add missing REQ_F_COMP_LOCKED for nested requests

commit 9b7adba9eaec28e0e4343c96d0dbeb9578802f5f upstream.

When we traverse into failing links or timeouts, we need to ensure we
propagate the REQ_F_COMP_LOCKED flag to ensure that we correctly signal
to the completion side that we already hold the completion lock.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index f2c71b494b52768979430730ffd2c711bd0e38f3..b15cd2a50ef34c7f0fb5cc475308ae7de17d3376 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -1484,12 +1484,9 @@ static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
 /*
  * Called if REQ_F_LINK_HEAD is set, and we fail the head request
  */
-static void io_fail_links(struct io_kiocb *req)
+static void __io_fail_links(struct io_kiocb *req)
 {
        struct io_ring_ctx *ctx = req->ctx;
-       unsigned long flags;
-
-       spin_lock_irqsave(&ctx->completion_lock, flags);
 
        while (!list_empty(&req->link_list)) {
                struct io_kiocb *link = list_first_entry(&req->link_list,
@@ -1503,13 +1500,29 @@ static void io_fail_links(struct io_kiocb *req)
                        io_link_cancel_timeout(link);
                } else {
                        io_cqring_fill_event(link, -ECANCELED);
+                       link->flags |= REQ_F_COMP_LOCKED;
                        __io_double_put_req(link);
                }
                req->flags &= ~REQ_F_LINK_TIMEOUT;
        }
 
        io_commit_cqring(ctx);
-       spin_unlock_irqrestore(&ctx->completion_lock, flags);
+}
+
+static void io_fail_links(struct io_kiocb *req)
+{
+       struct io_ring_ctx *ctx = req->ctx;
+
+       if (!(req->flags & REQ_F_COMP_LOCKED)) {
+               unsigned long flags;
+
+               spin_lock_irqsave(&ctx->completion_lock, flags);
+               __io_fail_links(req);
+               spin_unlock_irqrestore(&ctx->completion_lock, flags);
+       } else {
+               __io_fail_links(req);
+       }
+
        io_cqring_ev_posted(ctx);
 }
 
@@ -4828,6 +4841,7 @@ static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
                return -EALREADY;
 
        req_set_fail_links(req);
+       req->flags |= REQ_F_COMP_LOCKED;
        io_cqring_fill_event(req, -ECANCELED);
        io_put_req(req);
        return 0;