io_uring: always arm linked timeouts prior to issue

There are a few spots where linked timeouts are armed, and not all of
them adhere to the pre-arm, attempt issue, post-arm pattern. This can
be problematic if the linked request returns that it will trigger a
callback later, and does so before the linked timeout is fully armed.

Consolidate all the linked timeout handling into __io_issue_sqe(),
rather than have it spread throughout the various issue entry points.

Cc: stable@vger.kernel.org
Link: https://github.com/axboe/liburing/issues/1390
Reported-by: Chase Hiltz <chase@path.net>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index a2b256e96d5da388495a20cee9539d82ca1378ef..769814d7115302a5bb07e3a8eb09d3b63c9e1a12 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -448,24 +448,6 @@ static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)
        return req->link;
 }
 
-static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
-{
-       if (likely(!(req->flags & REQ_F_ARM_LTIMEOUT)))
-               return NULL;
-       return __io_prep_linked_timeout(req);
-}
-
-static noinline void __io_arm_ltimeout(struct io_kiocb *req)
-{
-       io_queue_linked_timeout(__io_prep_linked_timeout(req));
-}
-
-static inline void io_arm_ltimeout(struct io_kiocb *req)
-{
-       if (unlikely(req->flags & REQ_F_ARM_LTIMEOUT))
-               __io_arm_ltimeout(req);
-}
-
 static void io_prep_async_work(struct io_kiocb *req)
 {
        const struct io_issue_def *def = &io_issue_defs[req->opcode];
@@ -518,7 +500,6 @@ static void io_prep_async_link(struct io_kiocb *req)
 
 static void io_queue_iowq(struct io_kiocb *req)
 {
-       struct io_kiocb *link = io_prep_linked_timeout(req);
        struct io_uring_task *tctx = req->tctx;
 
        BUG_ON(!tctx);
@@ -543,8 +524,6 @@ static void io_queue_iowq(struct io_kiocb *req)
 
        trace_io_uring_queue_async_work(req, io_wq_is_hashed(&req->work));
        io_wq_enqueue(tctx->io_wq, &req->work);
-       if (link)
-               io_queue_linked_timeout(link);
 }
 
 static void io_req_queue_iowq_tw(struct io_kiocb *req, io_tw_token_t tw)
@@ -1724,15 +1703,22 @@ static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,
        return !!req->file;
 }
 
+#define REQ_ISSUE_SLOW_FLAGS   (REQ_F_CREDS | REQ_F_ARM_LTIMEOUT)
+
 static inline int __io_issue_sqe(struct io_kiocb *req,
                                 unsigned int issue_flags,
                                 const struct io_issue_def *def)
 {
        const struct cred *creds = NULL;
+       struct io_kiocb *link = NULL;
        int ret;
 
-       if (unlikely((req->flags & REQ_F_CREDS) && req->creds != current_cred()))
-               creds = override_creds(req->creds);
+       if (unlikely(req->flags & REQ_ISSUE_SLOW_FLAGS)) {
+               if ((req->flags & REQ_F_CREDS) && req->creds != current_cred())
+                       creds = override_creds(req->creds);
+               if (req->flags & REQ_F_ARM_LTIMEOUT)
+                       link = __io_prep_linked_timeout(req);
+       }
 
        if (!def->audit_skip)
                audit_uring_entry(req->opcode);
@@ -1742,8 +1728,12 @@ static inline int __io_issue_sqe(struct io_kiocb *req,
        if (!def->audit_skip)
                audit_uring_exit(!ret, ret);
 
-       if (creds)
-               revert_creds(creds);
+       if (unlikely(creds || link)) {
+               if (creds)
+                       revert_creds(creds);
+               if (link)
+                       io_queue_linked_timeout(link);
+       }
 
        return ret;
 }
@@ -1769,7 +1759,6 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
 
        if (ret == IOU_ISSUE_SKIP_COMPLETE) {
                ret = 0;
-               io_arm_ltimeout(req);
 
                /* If the op doesn't have a file, we're not polling for it */
                if ((req->ctx->flags & IORING_SETUP_IOPOLL) && def->iopoll_queue)
@@ -1824,8 +1813,6 @@ void io_wq_submit_work(struct io_wq_work *work)
        else
                req_ref_get(req);
 
-       io_arm_ltimeout(req);
-
        /* either cancelled or io-wq is dying, so don't touch tctx->iowq */
        if (atomic_read(&work->flags) & IO_WQ_WORK_CANCEL) {
 fail:
@@ -1941,15 +1928,11 @@ struct file *io_file_get_normal(struct io_kiocb *req, int fd)
 static void io_queue_async(struct io_kiocb *req, int ret)
        __must_hold(&req->ctx->uring_lock)
 {
-       struct io_kiocb *linked_timeout;
-
        if (ret != -EAGAIN || (req->flags & REQ_F_NOWAIT)) {
                io_req_defer_failed(req, ret);
                return;
        }
 
-       linked_timeout = io_prep_linked_timeout(req);
-
        switch (io_arm_poll_handler(req, 0)) {
        case IO_APOLL_READY:
                io_kbuf_recycle(req, 0);
@@ -1962,9 +1945,6 @@ static void io_queue_async(struct io_kiocb *req, int ret)
        case IO_APOLL_OK:
                break;
        }
-
-       if (linked_timeout)
-               io_queue_linked_timeout(linked_timeout);
 }
 
 static inline void io_queue_sqe(struct io_kiocb *req)