io_uring: make io_poll_issue() sturdier

io_poll_issue() forwards the call to io_issue_sqe() and thus inherits
some of the handling. That's not particularly failure resistant, as for
example returning an innocently looking IOU_OK from a multishot issue
will lead to severe bugs.

Reimplement io_poll_issue() without io_issue_sqe()'s request completion
logic. Remove extra checks as we know that req->file is already set,
linked timeout are armed, and iopoll is not supported. Also cover it
with warnings for now.

The patch should be useful by itself, but it's also preparing the
codebase for other future clean ups.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/3096d7b1026d9a52426a598bdfc8d9d324555545.1740331076.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index bd1ab21ed5393f317a380c85ebb3dba9632ef42c..d169043fc35a986279d786036b1b104f1a43d18c 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1720,15 +1720,13 @@ static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,
        return !!req->file;
 }
 
-static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
+static inline int __io_issue_sqe(struct io_kiocb *req,
+                                unsigned int issue_flags,
+                                const struct io_issue_def *def)
 {
-       const struct io_issue_def *def = &io_issue_defs[req->opcode];
        const struct cred *creds = NULL;
        int ret;
 
-       if (unlikely(!io_assign_file(req, def, issue_flags)))
-               return -EBADF;
-
        if (unlikely((req->flags & REQ_F_CREDS) && req->creds != current_cred()))
                creds = override_creds(req->creds);
 
@@ -1743,6 +1741,19 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
        if (creds)
                revert_creds(creds);
 
+       return ret;
+}
+
+static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
+{
+       const struct io_issue_def *def = &io_issue_defs[req->opcode];
+       int ret;
+
+       if (unlikely(!io_assign_file(req, def, issue_flags)))
+               return -EBADF;
+
+       ret = __io_issue_sqe(req, issue_flags, def);
+
        if (ret == IOU_OK) {
                if (issue_flags & IO_URING_F_COMPLETE_DEFER)
                        io_req_complete_defer(req);
@@ -1765,9 +1776,24 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
 
 int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw)
 {
+       const unsigned int issue_flags = IO_URING_F_NONBLOCK |
+                                        IO_URING_F_MULTISHOT |
+                                        IO_URING_F_COMPLETE_DEFER;
+       int ret;
+
        io_tw_lock(req->ctx, tw);
-       return io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_MULTISHOT|
-                                IO_URING_F_COMPLETE_DEFER);
+
+       WARN_ON_ONCE(!req->file);
+       if (WARN_ON_ONCE(req->ctx->flags & IORING_SETUP_IOPOLL))
+               return -EFAULT;
+
+       ret = __io_issue_sqe(req, issue_flags, &io_issue_defs[req->opcode]);
+
+       WARN_ON_ONCE(ret == IOU_OK);
+
+       if (ret == IOU_ISSUE_SKIP_COMPLETE)
+               ret = 0;
+       return ret;
 }
 
 struct io_wq_work *io_wq_free_work(struct io_wq_work *work)