Merge tag 'scmi-fixes-6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/sudeep...

[thirdparty/linux.git] / io_uring / io_uring.c

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 8d1bc6cdfe712e75638ddf99c2f9ebac2d32d1f5..ed254076c723332c3e9338b65ea3505be1608e71 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -92,6 +92,8 @@
 #include "cancel.h"
 #include "net.h"
 #include "notif.h"
+#include "waitid.h"
+#include "futex.h"
 
 #include "timeout.h"
 #include "poll.h"
@@ -329,6 +331,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
                            sizeof(struct async_poll));
        io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
                            sizeof(struct io_async_msghdr));
+       io_futex_cache_init(ctx);
        init_completion(&ctx->ref_comp);
        xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
        mutex_init(&ctx->uring_lock);
@@ -338,7 +341,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
        spin_lock_init(&ctx->completion_lock);
        spin_lock_init(&ctx->timeout_lock);
        INIT_WQ_LIST(&ctx->iopoll_list);
-       INIT_LIST_HEAD(&ctx->io_buffers_pages);
        INIT_LIST_HEAD(&ctx->io_buffers_comp);
        INIT_LIST_HEAD(&ctx->defer_list);
        INIT_LIST_HEAD(&ctx->timeout_list);
@@ -348,8 +350,13 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
        INIT_LIST_HEAD(&ctx->tctx_list);
        ctx->submit_state.free_list.next = NULL;
        INIT_WQ_LIST(&ctx->locked_free_list);
+       INIT_HLIST_HEAD(&ctx->waitid_list);
+#ifdef CONFIG_FUTEX
+       INIT_HLIST_HEAD(&ctx->futex_list);
+#endif
        INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
        INIT_WQ_LIST(&ctx->submit_state.compl_reqs);
+       INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd);
        return ctx;
 err:
        kfree(ctx->cancel_table.hbs);
@@ -2912,6 +2919,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
        io_eventfd_unregister(ctx);
        io_alloc_cache_free(&ctx->apoll_cache, io_apoll_cache_free);
        io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);
+       io_futex_cache_free(ctx);
        io_destroy_buffers(ctx);
        mutex_unlock(&ctx->uring_lock);
        if (ctx->sq_creds)
@@ -3276,6 +3284,37 @@ static __cold bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
        return ret;
 }
 
+static bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
+               struct task_struct *task, bool cancel_all)
+{
+       struct hlist_node *tmp;
+       struct io_kiocb *req;
+       bool ret = false;
+
+       lockdep_assert_held(&ctx->uring_lock);
+
+       hlist_for_each_entry_safe(req, tmp, &ctx->cancelable_uring_cmd,
+                       hash_node) {
+               struct io_uring_cmd *cmd = io_kiocb_to_cmd(req,
+                               struct io_uring_cmd);
+               struct file *file = req->file;
+
+               if (!cancel_all && req->task != task)
+                       continue;
+
+               if (cmd->flags & IORING_URING_CMD_CANCELABLE) {
+                       /* ->sqe isn't available if no async data */
+                       if (!req_has_async_data(req))
+                               cmd->sqe = NULL;
+                       file->f_op->uring_cmd(cmd, IO_URING_F_CANCEL);
+                       ret = true;
+               }
+       }
+       io_submit_flush_completions(ctx);
+
+       return ret;
+}
+
 static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
                                                struct task_struct *task,
                                                bool cancel_all)
@@ -3323,6 +3362,9 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
        ret |= io_cancel_defer_files(ctx, task, cancel_all);
        mutex_lock(&ctx->uring_lock);
        ret |= io_poll_remove_all(ctx, task, cancel_all);
+       ret |= io_waitid_remove_all(ctx, task, cancel_all);
+       ret |= io_futex_remove_all(ctx, task, cancel_all);
+       ret |= io_uring_try_cancel_uring_cmd(ctx, task, cancel_all);
        mutex_unlock(&ctx->uring_lock);
        ret |= io_kill_timeouts(ctx, task, cancel_all);
        if (task)
@@ -4686,6 +4728,9 @@ static int __init io_uring_init(void)
 
        BUILD_BUG_ON(sizeof(atomic_t) != sizeof(u32));
 
+       /* top 8bits are for internal use */
+       BUILD_BUG_ON((IORING_URING_CMD_MASK & 0xff000000) != 0);
+
        io_uring_optable_init();
 
        /*
@@ -4701,6 +4746,9 @@ static int __init io_uring_init(void)
                                SLAB_ACCOUNT | SLAB_TYPESAFE_BY_RCU,
                                offsetof(struct io_kiocb, cmd.data),
                                sizeof_field(struct io_kiocb, cmd.data), NULL);
+       io_buf_cachep = kmem_cache_create("io_buffer", sizeof(struct io_buffer), 0,
+                                         SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT,
+                                         NULL);
 
 #ifdef CONFIG_SYSCTL
        register_sysctl_init("kernel", kernel_io_uring_disabled_table);