From: Greg Kroah-Hartman Date: Wed, 4 Jan 2023 10:44:04 +0000 (+0100) Subject: move 5.10 queue back in place X-Git-Tag: v6.1.4~80 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=71c8d2a85f200bbd0ac8e906455b58c4c1d27182;p=thirdparty%2Fkernel%2Fstable-queue.git move 5.10 queue back in place --- diff --git a/next-5.10/alpha-add-support-for-tif_notify_signal.patch b/next-5.10/alpha-add-support-for-tif_notify_signal.patch deleted file mode 100644 index 3de3f334327..00000000000 --- a/next-5.10/alpha-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,60 +0,0 @@ -From f85a5fbc85e3edbfa86d83f163307c9ef597eece Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Thu, 8 Oct 2020 09:11:42 -0600 -Subject: alpha: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 5a9a8897c253a075805401d38d987ec1ac1824b6 ] - -Wire up TIF_NOTIFY_SIGNAL handling for alpha. - -Cc: linux-alpha@vger.kernel.org -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/alpha/include/asm/thread_info.h | 2 ++ - arch/alpha/kernel/entry.S | 2 +- - arch/alpha/kernel/signal.c | 2 +- - 3 files changed, 4 insertions(+), 2 deletions(-) - ---- a/arch/alpha/include/asm/thread_info.h -+++ b/arch/alpha/include/asm/thread_info.h -@@ -62,6 +62,7 @@ register struct thread_info *__current_t - #define TIF_SIGPENDING 2 /* signal pending */ - #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ - #define TIF_SYSCALL_AUDIT 4 /* syscall audit active */ -+#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ - #define TIF_DIE_IF_KERNEL 9 /* dik recursion lock */ - #define TIF_MEMDIE 13 /* is terminating due to OOM killer */ - #define TIF_POLLING_NRFLAG 14 /* idle is polling for TIF_NEED_RESCHED */ -@@ -71,6 +72,7 @@ register struct thread_info *__current_t - #define _TIF_NEED_RESCHED (1< -Date: Sat, 18 Sep 2021 18:08:56 -0400 -Subject: alpha: fix TIF_NOTIFY_SIGNAL handling - -From: Al Viro - -[ Upstream commit e2c7554cc6d85f95e3c6635f270ec839ab9fe05e ] - -it needs to be added to _TIF_WORK_MASK, or we might not reach -do_work_pending() in the first place... - -Fixes: 5a9a8897c253a "alpha: add support for TIF_NOTIFY_SIGNAL" -Signed-off-by: Al Viro -Signed-off-by: Greg Kroah-Hartman ---- - arch/alpha/include/asm/thread_info.h | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - ---- a/arch/alpha/include/asm/thread_info.h -+++ b/arch/alpha/include/asm/thread_info.h -@@ -77,7 +77,7 @@ register struct thread_info *__current_t - - /* Work to do on interrupt/exception return. */ - #define _TIF_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \ -- _TIF_NOTIFY_RESUME) -+ _TIF_NOTIFY_RESUME | _TIF_NOTIFY_SIGNAL) - - /* Work to do on any return to userspace. */ - #define _TIF_ALLWORK_MASK (_TIF_WORK_MASK \ diff --git a/next-5.10/arc-add-support-for-tif_notify_signal.patch b/next-5.10/arc-add-support-for-tif_notify_signal.patch deleted file mode 100644 index a1ad6fd2889..00000000000 --- a/next-5.10/arc-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,68 +0,0 @@ -From b71f35bd26a9b588d59894e542c48b4360be52fc Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Mon, 12 Oct 2020 07:15:37 -0600 -Subject: arc: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 53855e12588743ea128ee31f913d1c6e2f1d32c8 ] - -Wire up TIF_NOTIFY_SIGNAL handling for arc. - -Cc: linux-snps-arc@lists.infradead.org -Acked-by: Vineet Gupta -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/arc/include/asm/thread_info.h | 4 +++- - arch/arc/kernel/entry.S | 3 ++- - arch/arc/kernel/signal.c | 2 +- - 3 files changed, 6 insertions(+), 3 deletions(-) - ---- a/arch/arc/include/asm/thread_info.h -+++ b/arch/arc/include/asm/thread_info.h -@@ -79,6 +79,7 @@ static inline __attribute_const__ struct - #define TIF_SIGPENDING 2 /* signal pending */ - #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ - #define TIF_SYSCALL_AUDIT 4 /* syscall auditing active */ -+#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ - #define TIF_SYSCALL_TRACE 15 /* syscall trace active */ - - /* true if poll_idle() is polling TIF_NEED_RESCHED */ -@@ -89,11 +90,12 @@ static inline __attribute_const__ struct - #define _TIF_SIGPENDING (1< -Date: Wed, 6 Jan 2021 12:34:36 -0800 -Subject: ARC: unbork 5.11 bootup: fix snafu in _TIF_NOTIFY_SIGNAL handling - -From: Vineet Gupta - -[ Upstream commit bb12433bf56e76789c6b08b36c546f745a6aa6e1 ] - -Linux 5.11.rcX was failing to boot on ARC HSDK board. Turns out we have -a couple of issues, this being the first one, and I'm to blame as I -didn't pay attention during review. - -TIF_NOTIFY_SIGNAL support requires checking multiple TIF_* bits in -kernel return code path. Old code only needed to check a single bit so -BBIT0 worked. New code needs to check multiple bits so -AND instruction. So needs to use bit mask variant _TIF_SIGPENDING - -Cc: Jens Axboe -Fixes: 53855e12588743ea128 ("arc: add support for TIF_NOTIFY_SIGNAL") -Link: https://github.com/foss-for-synopsys-dwc-arc-processors/linux/issues/34 -Signed-off-by: Vineet Gupta -Signed-off-by: Greg Kroah-Hartman ---- - arch/arc/kernel/entry.S | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - ---- a/arch/arc/kernel/entry.S -+++ b/arch/arc/kernel/entry.S -@@ -308,7 +308,7 @@ resume_user_mode_begin: - mov r0, sp ; pt_regs for arg to do_signal()/do_notify_resume() - - GET_CURR_THR_INFO_FLAGS r9 -- and.f 0, r9, TIF_SIGPENDING|TIF_NOTIFY_SIGNAL -+ and.f 0, r9, _TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL - bz .Lchk_notify_resume - - ; Normal Trap/IRQ entry only saves Scratch (caller-saved) regs diff --git a/next-5.10/arch-ensure-parisc-powerpc-handle-pf_io_worker-in-copy_thread.patch b/next-5.10/arch-ensure-parisc-powerpc-handle-pf_io_worker-in-copy_thread.patch deleted file mode 100644 index dabbd7698bf..00000000000 --- a/next-5.10/arch-ensure-parisc-powerpc-handle-pf_io_worker-in-copy_thread.patch +++ /dev/null @@ -1,44 +0,0 @@ -From b9634adab82019ff70e445c09a7c26a209f6d010 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Tue, 23 Feb 2021 11:57:20 -0700 -Subject: arch: ensure parisc/powerpc handle PF_IO_WORKER in copy_thread() - -From: Jens Axboe - -[ Upstream commit 0100e6bbdbb79404e56939313662b42737026574 ] - -In the arch addition of PF_IO_WORKER, I missed parisc and powerpc for -some reason. Fix that up, ensuring they handle PF_IO_WORKER like they do -PF_KTHREAD in copy_thread(). - -Reported-by: Bruno Goncalves -Fixes: 4727dc20e042 ("arch: setup PF_IO_WORKER threads like PF_KTHREAD") -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/parisc/kernel/process.c | 2 +- - arch/powerpc/kernel/process.c | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - ---- a/arch/parisc/kernel/process.c -+++ b/arch/parisc/kernel/process.c -@@ -200,7 +200,7 @@ copy_thread(unsigned long clone_flags, u - extern void * const ret_from_kernel_thread; - extern void * const child_return; - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - /* kernel thread */ - memset(cregs, 0, sizeof(struct pt_regs)); - if (!usp) /* idle thread */ ---- a/arch/powerpc/kernel/process.c -+++ b/arch/powerpc/kernel/process.c -@@ -1684,7 +1684,7 @@ int copy_thread(unsigned long clone_flag - /* Copy registers */ - sp -= sizeof(struct pt_regs); - childregs = (struct pt_regs *) sp; -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - /* kernel thread */ - memset(childregs, 0, sizeof(struct pt_regs)); - childregs->gpr[1] = sp + sizeof(struct pt_regs); diff --git a/next-5.10/arch-setup-pf_io_worker-threads-like-pf_kthread.patch b/next-5.10/arch-setup-pf_io_worker-threads-like-pf_kthread.patch deleted file mode 100644 index ca53757b9ac..00000000000 --- a/next-5.10/arch-setup-pf_io_worker-threads-like-pf_kthread.patch +++ /dev/null @@ -1,283 +0,0 @@ -From 50b21feb5a0272a0a8cb620d27d4c5e698cc1d21 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Wed, 17 Feb 2021 08:48:00 -0700 -Subject: arch: setup PF_IO_WORKER threads like PF_KTHREAD - -From: Jens Axboe - -[ Upstream commit 4727dc20e0422211a0e0c72b1ace4ed6096df8a6 ] - -PF_IO_WORKER are kernel threads too, but they aren't PF_KTHREAD in the -sense that we don't assign ->set_child_tid with our own structure. Just -ensure that every arch sets up the PF_IO_WORKER threads like kthreads -in the arch implementation of copy_thread(). - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/alpha/kernel/process.c | 2 +- - arch/arc/kernel/process.c | 2 +- - arch/arm/kernel/process.c | 2 +- - arch/arm64/kernel/process.c | 2 +- - arch/csky/kernel/process.c | 2 +- - arch/h8300/kernel/process.c | 2 +- - arch/hexagon/kernel/process.c | 2 +- - arch/ia64/kernel/process.c | 2 +- - arch/m68k/kernel/process.c | 2 +- - arch/microblaze/kernel/process.c | 2 +- - arch/mips/kernel/process.c | 2 +- - arch/nds32/kernel/process.c | 2 +- - arch/nios2/kernel/process.c | 2 +- - arch/openrisc/kernel/process.c | 2 +- - arch/riscv/kernel/process.c | 2 +- - arch/s390/kernel/process.c | 2 +- - arch/sh/kernel/process_32.c | 2 +- - arch/sparc/kernel/process_32.c | 2 +- - arch/sparc/kernel/process_64.c | 2 +- - arch/um/kernel/process.c | 2 +- - arch/x86/kernel/process.c | 2 +- - arch/xtensa/kernel/process.c | 2 +- - 22 files changed, 22 insertions(+), 22 deletions(-) - ---- a/arch/alpha/kernel/process.c -+++ b/arch/alpha/kernel/process.c -@@ -249,7 +249,7 @@ int copy_thread(unsigned long clone_flag - childti->pcb.ksp = (unsigned long) childstack; - childti->pcb.flags = 1; /* set FEN, clear everything else */ - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - /* kernel thread */ - memset(childstack, 0, - sizeof(struct switch_stack) + sizeof(struct pt_regs)); ---- a/arch/arc/kernel/process.c -+++ b/arch/arc/kernel/process.c -@@ -191,7 +191,7 @@ int copy_thread(unsigned long clone_flag - childksp[0] = 0; /* fp */ - childksp[1] = (unsigned long)ret_from_fork; /* blink */ - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - memset(c_regs, 0, sizeof(struct pt_regs)); - - c_callee->r13 = kthread_arg; ---- a/arch/arm/kernel/process.c -+++ b/arch/arm/kernel/process.c -@@ -243,7 +243,7 @@ int copy_thread(unsigned long clone_flag - thread->cpu_domain = get_domain(); - #endif - -- if (likely(!(p->flags & PF_KTHREAD))) { -+ if (likely(!(p->flags & (PF_KTHREAD | PF_IO_WORKER)))) { - *childregs = *current_pt_regs(); - childregs->ARM_r0 = 0; - if (stack_start) ---- a/arch/arm64/kernel/process.c -+++ b/arch/arm64/kernel/process.c -@@ -398,7 +398,7 @@ int copy_thread(unsigned long clone_flag - - ptrauth_thread_init_kernel(p); - -- if (likely(!(p->flags & PF_KTHREAD))) { -+ if (likely(!(p->flags & (PF_KTHREAD | PF_IO_WORKER)))) { - *childregs = *current_pt_regs(); - childregs->regs[0] = 0; - ---- a/arch/csky/kernel/process.c -+++ b/arch/csky/kernel/process.c -@@ -49,7 +49,7 @@ int copy_thread(unsigned long clone_flag - /* setup thread.sp for switch_to !!! */ - p->thread.sp = (unsigned long)childstack; - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - memset(childregs, 0, sizeof(struct pt_regs)); - childstack->r15 = (unsigned long) ret_from_kernel_thread; - childstack->r10 = kthread_arg; ---- a/arch/h8300/kernel/process.c -+++ b/arch/h8300/kernel/process.c -@@ -112,7 +112,7 @@ int copy_thread(unsigned long clone_flag - - childregs = (struct pt_regs *) (THREAD_SIZE + task_stack_page(p)) - 1; - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - memset(childregs, 0, sizeof(struct pt_regs)); - childregs->retpc = (unsigned long) ret_from_kernel_thread; - childregs->er4 = topstk; /* arg */ ---- a/arch/hexagon/kernel/process.c -+++ b/arch/hexagon/kernel/process.c -@@ -73,7 +73,7 @@ int copy_thread(unsigned long clone_flag - sizeof(*ss)); - ss->lr = (unsigned long)ret_from_fork; - p->thread.switch_sp = ss; -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - memset(childregs, 0, sizeof(struct pt_regs)); - /* r24 <- fn, r25 <- arg */ - ss->r24 = usp; ---- a/arch/ia64/kernel/process.c -+++ b/arch/ia64/kernel/process.c -@@ -338,7 +338,7 @@ copy_thread(unsigned long clone_flags, u - - ia64_drop_fpu(p); /* don't pick up stale state from a CPU's fph */ - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - if (unlikely(!user_stack_base)) { - /* fork_idle() called us */ - return 0; ---- a/arch/m68k/kernel/process.c -+++ b/arch/m68k/kernel/process.c -@@ -157,7 +157,7 @@ int copy_thread(unsigned long clone_flag - */ - p->thread.fs = get_fs().seg; - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - /* kernel thread */ - memset(frame, 0, sizeof(struct fork_frame)); - frame->regs.sr = PS_S; ---- a/arch/microblaze/kernel/process.c -+++ b/arch/microblaze/kernel/process.c -@@ -59,7 +59,7 @@ int copy_thread(unsigned long clone_flag - struct pt_regs *childregs = task_pt_regs(p); - struct thread_info *ti = task_thread_info(p); - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - /* if we're creating a new kernel thread then just zeroing all - * the registers. That's OK for a brand new thread.*/ - memset(childregs, 0, sizeof(struct pt_regs)); ---- a/arch/mips/kernel/process.c -+++ b/arch/mips/kernel/process.c -@@ -135,7 +135,7 @@ int copy_thread(unsigned long clone_flag - /* Put the stack after the struct pt_regs. */ - childksp = (unsigned long) childregs; - p->thread.cp0_status = (read_c0_status() & ~(ST0_CU2|ST0_CU1)) | ST0_KERNEL_CUMASK; -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - /* kernel thread */ - unsigned long status = p->thread.cp0_status; - memset(childregs, 0, sizeof(struct pt_regs)); ---- a/arch/nds32/kernel/process.c -+++ b/arch/nds32/kernel/process.c -@@ -156,7 +156,7 @@ int copy_thread(unsigned long clone_flag - - memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - memset(childregs, 0, sizeof(struct pt_regs)); - /* kernel thread fn */ - p->thread.cpu_context.r6 = stack_start; ---- a/arch/nios2/kernel/process.c -+++ b/arch/nios2/kernel/process.c -@@ -109,7 +109,7 @@ int copy_thread(unsigned long clone_flag - struct switch_stack *childstack = - ((struct switch_stack *)childregs) - 1; - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - memset(childstack, 0, - sizeof(struct switch_stack) + sizeof(struct pt_regs)); - ---- a/arch/openrisc/kernel/process.c -+++ b/arch/openrisc/kernel/process.c -@@ -167,7 +167,7 @@ copy_thread(unsigned long clone_flags, u - sp -= sizeof(struct pt_regs); - kregs = (struct pt_regs *)sp; - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - memset(kregs, 0, sizeof(struct pt_regs)); - kregs->gpr[20] = usp; /* fn, kernel thread */ - kregs->gpr[22] = arg; ---- a/arch/riscv/kernel/process.c -+++ b/arch/riscv/kernel/process.c -@@ -114,7 +114,7 @@ int copy_thread(unsigned long clone_flag - memset(&p->thread.s, 0, sizeof(p->thread.s)); - - /* p->thread holds context to be restored by __switch_to() */ -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - /* Kernel thread */ - memset(childregs, 0, sizeof(struct pt_regs)); - childregs->gp = gp_in_global; ---- a/arch/s390/kernel/process.c -+++ b/arch/s390/kernel/process.c -@@ -127,7 +127,7 @@ int copy_thread(unsigned long clone_flag - frame->sf.gprs[9] = (unsigned long) frame; - - /* Store access registers to kernel stack of new process. */ -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - /* kernel thread */ - memset(&frame->childregs, 0, sizeof(struct pt_regs)); - frame->childregs.psw.mask = PSW_KERNEL_BITS | PSW_MASK_DAT | ---- a/arch/sh/kernel/process_32.c -+++ b/arch/sh/kernel/process_32.c -@@ -114,7 +114,7 @@ int copy_thread(unsigned long clone_flag - - childregs = task_pt_regs(p); - p->thread.sp = (unsigned long) childregs; -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - memset(childregs, 0, sizeof(struct pt_regs)); - p->thread.pc = (unsigned long) ret_from_kernel_thread; - childregs->regs[4] = arg; ---- a/arch/sparc/kernel/process_32.c -+++ b/arch/sparc/kernel/process_32.c -@@ -309,7 +309,7 @@ int copy_thread(unsigned long clone_flag - ti->ksp = (unsigned long) new_stack; - p->thread.kregs = childregs; - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - extern int nwindows; - unsigned long psr; - memset(new_stack, 0, STACKFRAME_SZ + TRACEREG_SZ); ---- a/arch/sparc/kernel/process_64.c -+++ b/arch/sparc/kernel/process_64.c -@@ -597,7 +597,7 @@ int copy_thread(unsigned long clone_flag - sizeof(struct sparc_stackf)); - t->fpsaved[0] = 0; - -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - memset(child_trap_frame, 0, child_stack_sz); - __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] = - (current_pt_regs()->tstate + 1) & TSTATE_CWP; ---- a/arch/um/kernel/process.c -+++ b/arch/um/kernel/process.c -@@ -157,7 +157,7 @@ int copy_thread(unsigned long clone_flag - unsigned long arg, struct task_struct * p, unsigned long tls) - { - void (*handler)(void); -- int kthread = current->flags & PF_KTHREAD; -+ int kthread = current->flags & (PF_KTHREAD | PF_IO_WORKER); - int ret = 0; - - p->thread = (struct thread_struct) INIT_THREAD; ---- a/arch/x86/kernel/process.c -+++ b/arch/x86/kernel/process.c -@@ -162,7 +162,7 @@ int copy_thread(unsigned long clone_flag - #endif - - /* Kernel thread ? */ -- if (unlikely(p->flags & PF_KTHREAD)) { -+ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - memset(childregs, 0, sizeof(struct pt_regs)); - kthread_frame_init(frame, sp, arg); - return 0; ---- a/arch/xtensa/kernel/process.c -+++ b/arch/xtensa/kernel/process.c -@@ -217,7 +217,7 @@ int copy_thread(unsigned long clone_flag - - p->thread.sp = (unsigned long)childregs; - -- if (!(p->flags & PF_KTHREAD)) { -+ if (!(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { - struct pt_regs *regs = current_pt_regs(); - unsigned long usp = usp_thread_fn ? - usp_thread_fn : regs->areg[1]; diff --git a/next-5.10/arm-add-support-for-tif_notify_signal.patch b/next-5.10/arm-add-support-for-tif_notify_signal.patch deleted file mode 100644 index c6cadc79fad..00000000000 --- a/next-5.10/arm-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,110 +0,0 @@ -From 4548cba5ff2a7b81495b7c109b48972655715142 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 16:00:49 -0600 -Subject: arm: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 32d59773da38cd83e497a70eb9754d4bbae3aeae ] - -Wire up TIF_NOTIFY_SIGNAL handling for arm. - -Cc: linux-arm-kernel@lists.infradead.org -Acked-by: Russell King -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/arm/include/asm/thread_info.h | 7 ++++++- - arch/arm/kernel/entry-common.S | 6 +++--- - arch/arm/kernel/entry-v7m.S | 2 +- - arch/arm/kernel/signal.c | 2 +- - 4 files changed, 11 insertions(+), 6 deletions(-) - ---- a/arch/arm/include/asm/thread_info.h -+++ b/arch/arm/include/asm/thread_info.h -@@ -126,6 +126,8 @@ extern int vfp_restore_user_hwstate(stru - * thread information flags: - * TIF_USEDFPU - FPU was used by this task this quantum (SMP) - * TIF_POLLING_NRFLAG - true if poll_idle() is polling TIF_NEED_RESCHED -+ * -+ * Any bit in the range of 0..15 will cause do_work_pending() to be invoked. - */ - #define TIF_SIGPENDING 0 /* signal pending */ - #define TIF_NEED_RESCHED 1 /* rescheduling necessary */ -@@ -135,6 +137,7 @@ extern int vfp_restore_user_hwstate(stru - #define TIF_SYSCALL_AUDIT 5 /* syscall auditing active */ - #define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */ - #define TIF_SECCOMP 7 /* seccomp syscall filtering active */ -+#define TIF_NOTIFY_SIGNAL 8 /* signal notifications exist */ - - #define TIF_USING_IWMMXT 17 - #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ -@@ -148,6 +151,7 @@ extern int vfp_restore_user_hwstate(stru - #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) - #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) - #define _TIF_SECCOMP (1 << TIF_SECCOMP) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - #define _TIF_USING_IWMMXT (1 << TIF_USING_IWMMXT) - - /* Checks for any syscall work in entry-common.S */ -@@ -158,7 +162,8 @@ extern int vfp_restore_user_hwstate(stru - * Change these and you break ASM code in entry-common.S - */ - #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ -- _TIF_NOTIFY_RESUME | _TIF_UPROBE) -+ _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ -+ _TIF_NOTIFY_SIGNAL) - - #endif /* __KERNEL__ */ - #endif /* __ASM_ARM_THREAD_INFO_H */ ---- a/arch/arm/kernel/entry-common.S -+++ b/arch/arm/kernel/entry-common.S -@@ -53,7 +53,7 @@ __ret_fast_syscall: - cmp r2, #TASK_SIZE - blne addr_limit_check_failed - ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing -- tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK -+ movs r1, r1, lsl #16 - bne fast_work_pending - - -@@ -90,7 +90,7 @@ __ret_fast_syscall: - cmp r2, #TASK_SIZE - blne addr_limit_check_failed - ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing -- tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK -+ movs r1, r1, lsl #16 - beq no_work_pending - UNWIND(.fnend ) - ENDPROC(ret_fast_syscall) -@@ -131,7 +131,7 @@ ENTRY(ret_to_user_from_irq) - cmp r2, #TASK_SIZE - blne addr_limit_check_failed - ldr r1, [tsk, #TI_FLAGS] -- tst r1, #_TIF_WORK_MASK -+ movs r1, r1, lsl #16 - bne slow_work_pending - no_work_pending: - asm_trace_hardirqs_on save = 0 ---- a/arch/arm/kernel/entry-v7m.S -+++ b/arch/arm/kernel/entry-v7m.S -@@ -59,7 +59,7 @@ __irq_entry: - - get_thread_info tsk - ldr r2, [tsk, #TI_FLAGS] -- tst r2, #_TIF_WORK_MASK -+ movs r2, r2, lsl #16 - beq 2f @ no work pending - mov r0, #V7M_SCB_ICSR_PENDSVSET - str r0, [r1, V7M_SCB_ICSR] @ raise PendSV ---- a/arch/arm/kernel/signal.c -+++ b/arch/arm/kernel/signal.c -@@ -655,7 +655,7 @@ do_work_pending(struct pt_regs *regs, un - if (unlikely(!user_mode(regs))) - return 0; - local_irq_enable(); -- if (thread_flags & _TIF_SIGPENDING) { -+ if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) { - int restart = do_signal(regs, syscall); - if (unlikely(restart)) { - /* diff --git a/next-5.10/arm64-add-support-for-tif_notify_signal.patch b/next-5.10/arm64-add-support-for-tif_notify_signal.patch deleted file mode 100644 index f90f46d1908..00000000000 --- a/next-5.10/arm64-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,56 +0,0 @@ -From eaa379f10805fdf0f096bfd32d3697f7af0dc185 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Thu, 22 Oct 2020 20:09:23 -0600 -Subject: arm64: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 192caabd4dd92c98d23ed4334d7596af05af2fb4 ] - -Wire up TIF_NOTIFY_SIGNAL handling for arm64. - -Cc: linux-arm-kernel@lists.infradead.org -Acked-by: Will Deacon -Acked-by: Catalin Marinas -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/arm64/include/asm/thread_info.h | 5 ++++- - arch/arm64/kernel/signal.c | 2 +- - 2 files changed, 5 insertions(+), 2 deletions(-) - ---- a/arch/arm64/include/asm/thread_info.h -+++ b/arch/arm64/include/asm/thread_info.h -@@ -68,6 +68,7 @@ void arch_release_task_struct(struct tas - #define TIF_UPROBE 4 /* uprobe breakpoint or singlestep */ - #define TIF_FSCHECK 5 /* Check FS is USER_DS on return */ - #define TIF_MTE_ASYNC_FAULT 6 /* MTE Asynchronous Tag Check Fault */ -+#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ - #define TIF_SYSCALL_TRACE 8 /* syscall trace active */ - #define TIF_SYSCALL_AUDIT 9 /* syscall auditing */ - #define TIF_SYSCALL_TRACEPOINT 10 /* syscall tracepoint for ftrace */ -@@ -98,10 +99,12 @@ void arch_release_task_struct(struct tas - #define _TIF_32BIT (1 << TIF_32BIT) - #define _TIF_SVE (1 << TIF_SVE) - #define _TIF_MTE_ASYNC_FAULT (1 << TIF_MTE_ASYNC_FAULT) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - - #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ - _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \ -- _TIF_UPROBE | _TIF_FSCHECK | _TIF_MTE_ASYNC_FAULT) -+ _TIF_UPROBE | _TIF_FSCHECK | _TIF_MTE_ASYNC_FAULT | \ -+ _TIF_NOTIFY_SIGNAL) - - #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ - _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \ ---- a/arch/arm64/kernel/signal.c -+++ b/arch/arm64/kernel/signal.c -@@ -938,7 +938,7 @@ asmlinkage void do_notify_resume(struct - (void __user *)NULL, current); - } - -- if (thread_flags & _TIF_SIGPENDING) -+ if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) - do_signal(regs); - - if (thread_flags & _TIF_NOTIFY_RESUME) { diff --git a/next-5.10/c6x-add-support-for-tif_notify_signal.patch b/next-5.10/c6x-add-support-for-tif_notify_signal.patch deleted file mode 100644 index a43c5259484..00000000000 --- a/next-5.10/c6x-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,59 +0,0 @@ -From bc746e497c6c8bcda6bb30d58a2502db4eaaf600 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 14:35:34 -0600 -Subject: c6x: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 6d665a4d8b4264def0fbb72da3a500d9904ffe3e ] - -Wire up TIF_NOTIFY_SIGNAL handling for c6x. - -Cc: linux-c6x-dev@linux-c6x.org -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/c6x/include/asm/thread_info.h | 1 + - arch/c6x/kernel/asm-offsets.c | 1 + - arch/c6x/kernel/signal.c | 3 ++- - 3 files changed, 4 insertions(+), 1 deletion(-) - ---- a/arch/c6x/include/asm/thread_info.h -+++ b/arch/c6x/include/asm/thread_info.h -@@ -82,6 +82,7 @@ struct thread_info *current_thread_info( - #define TIF_SIGPENDING 2 /* signal pending */ - #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ - #define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */ -+#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ - - #define TIF_MEMDIE 17 /* OOM killer killed process */ - ---- a/arch/c6x/kernel/asm-offsets.c -+++ b/arch/c6x/kernel/asm-offsets.c -@@ -116,6 +116,7 @@ void foo(void) - DEFINE(_TIF_NOTIFY_RESUME, (1< - #include - -+#include - #include - #include - -@@ -313,7 +314,7 @@ asmlinkage void do_notify_resume(struct - int syscall) - { - /* deal with pending signal delivery */ -- if (thread_info_flags & (1 << TIF_SIGPENDING)) -+ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) - do_signal(regs, syscall); - - if (thread_info_flags & (1 << TIF_NOTIFY_RESUME)) diff --git a/next-5.10/coredump-limit-what-can-interrupt-coredumps.patch b/next-5.10/coredump-limit-what-can-interrupt-coredumps.patch deleted file mode 100644 index 0d91f001463..00000000000 --- a/next-5.10/coredump-limit-what-can-interrupt-coredumps.patch +++ /dev/null @@ -1,61 +0,0 @@ -From 428760d101b98af1e7675882337dcad7c6786d87 Mon Sep 17 00:00:00 2001 -From: "Eric W. Biederman" -Date: Thu, 10 Jun 2021 15:11:11 -0500 -Subject: coredump: Limit what can interrupt coredumps - -From: "Eric W. Biederman" - -[ Upstream commit 06af8679449d4ed282df13191fc52d5ba28ec536 ] - -Olivier Langlois has been struggling with coredumps being incompletely written in -processes using io_uring. - -Olivier Langlois writes: -> io_uring is a big user of task_work and any event that io_uring made a -> task waiting for that occurs during the core dump generation will -> generate a TIF_NOTIFY_SIGNAL. -> -> Here are the detailed steps of the problem: -> 1. io_uring calls vfs_poll() to install a task to a file wait queue -> with io_async_wake() as the wakeup function cb from io_arm_poll_handler() -> 2. wakeup function ends up calling task_work_add() with TWA_SIGNAL -> 3. task_work_add() sets the TIF_NOTIFY_SIGNAL bit by calling -> set_notify_signal() - -The coredump code deliberately supports being interrupted by SIGKILL, -and depends upon prepare_signal to filter out all other signals. Now -that signal_pending includes wake ups for TIF_NOTIFY_SIGNAL this hack -in dump_emitted by the coredump code no longer works. - -Make the coredump code more robust by explicitly testing for all of -the wakeup conditions the coredump code supports. This prevents -new wakeup conditions from breaking the coredump code, as well -as fixing the current issue. - -The filesystem code that the coredump code uses already limits -itself to only aborting on fatal_signal_pending. So it should -not develop surprising wake-up reasons either. - -v2: Don't remove the now unnecessary code in prepare_signal. - -Cc: stable@vger.kernel.org -Fixes: 12db8b690010 ("entry: Add support for TIF_NOTIFY_SIGNAL") -Reported-by: Olivier Langlois -Signed-off-by: "Eric W. Biederman" -Signed-off-by: Linus Torvalds -Signed-off-by: Greg Kroah-Hartman ---- - fs/coredump.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - ---- a/fs/coredump.c -+++ b/fs/coredump.c -@@ -523,7 +523,7 @@ static bool dump_interrupted(void) - * but then we need to teach dump_write() to restart and clear - * TIF_SIGPENDING. - */ -- return signal_pending(current); -+ return fatal_signal_pending(current) || freezing(current); - } - - static void wait_for_dump_helpers(struct file *file) diff --git a/next-5.10/csky-add-support-for-tif_notify_signal.patch b/next-5.10/csky-add-support-for-tif_notify_signal.patch deleted file mode 100644 index 8d158001c28..00000000000 --- a/next-5.10/csky-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,59 +0,0 @@ -From 9f7678180bf6ff85fbd32bd83ff287a7eb4a6602 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 14:39:00 -0600 -Subject: csky: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit f3924d763c8af4c5d441b666c01f4de03ac9449e ] - -Wire up TIF_NOTIFY_SIGNAL handling for csky. - -Cc: linux-csky@vger.kernel.org -Acked-by: Guo Ren -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/csky/include/asm/thread_info.h | 5 ++++- - arch/csky/kernel/signal.c | 2 +- - 2 files changed, 5 insertions(+), 2 deletions(-) - ---- a/arch/csky/include/asm/thread_info.h -+++ b/arch/csky/include/asm/thread_info.h -@@ -64,6 +64,7 @@ static inline struct thread_info *curren - #define TIF_SYSCALL_TRACE 4 /* syscall trace active */ - #define TIF_SYSCALL_TRACEPOINT 5 /* syscall tracepoint instrumentation */ - #define TIF_SYSCALL_AUDIT 6 /* syscall auditing */ -+#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ - #define TIF_POLLING_NRFLAG 16 /* poll_idle() is TIF_NEED_RESCHED */ - #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ - #define TIF_RESTORE_SIGMASK 20 /* restore signal mask in do_signal() */ -@@ -75,6 +76,7 @@ static inline struct thread_info *curren - #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) - #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) - #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - #define _TIF_UPROBE (1 << TIF_UPROBE) - #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) - #define _TIF_MEMDIE (1 << TIF_MEMDIE) -@@ -82,7 +84,8 @@ static inline struct thread_info *curren - #define _TIF_SECCOMP (1 << TIF_SECCOMP) - - #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ -- _TIF_NOTIFY_RESUME | _TIF_UPROBE) -+ _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ -+ _TIF_NOTIFY_SIGNAL) - - #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ - _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP) ---- a/arch/csky/kernel/signal.c -+++ b/arch/csky/kernel/signal.c -@@ -261,7 +261,7 @@ asmlinkage void do_notify_resume(struct - uprobe_notify_resume(regs); - - /* Handle pending signal delivery */ -- if (thread_info_flags & _TIF_SIGPENDING) -+ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) - do_signal(regs); - - if (thread_info_flags & _TIF_NOTIFY_RESUME) { diff --git a/next-5.10/entry-add-support-for-tif_notify_signal.patch b/next-5.10/entry-add-support-for-tif_notify_signal.patch deleted file mode 100644 index f234acc3e72..00000000000 --- a/next-5.10/entry-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,234 +0,0 @@ -From d02e3fa856542e39531f7f0910389b334cd9b1f3 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Mon, 26 Oct 2020 14:32:28 -0600 -Subject: entry: Add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 12db8b690010ccfadf9d0b49a1e1798e47dbbe1a ] - -Add TIF_NOTIFY_SIGNAL handling in the generic entry code, which if set, -will return true if signal_pending() is used in a wait loop. That causes an -exit of the loop so that notify_signal tracehooks can be run. If the wait -loop is currently inside a system call, the system call is restarted once -task_work has been processed. - -In preparation for only having arch_do_signal() handle syscall restarts if -_TIF_SIGPENDING isn't set, rename it to arch_do_signal_or_restart(). Pass -in a boolean that tells the architecture specific signal handler if it -should attempt to get a signal, or just process a potential syscall -restart. - -For !CONFIG_GENERIC_ENTRY archs, add the TIF_NOTIFY_SIGNAL handling to -get_signal(). This is done to minimize the needed architecture changes to -support this feature. - -Signed-off-by: Jens Axboe -Signed-off-by: Thomas Gleixner -Reviewed-by: Oleg Nesterov -Link: https://lore.kernel.org/r/20201026203230.386348-3-axboe@kernel.dk -Signed-off-by: Greg Kroah-Hartman ---- - arch/x86/kernel/signal.c | 4 ++-- - include/linux/entry-common.h | 11 ++++++++--- - include/linux/entry-kvm.h | 4 ++-- - include/linux/sched/signal.h | 11 ++++++++++- - include/linux/tracehook.h | 27 +++++++++++++++++++++++++++ - kernel/entry/common.c | 14 +++++++++++--- - kernel/entry/kvm.c | 3 +++ - kernel/signal.c | 14 ++++++++++++++ - 8 files changed, 77 insertions(+), 11 deletions(-) - ---- a/arch/x86/kernel/signal.c -+++ b/arch/x86/kernel/signal.c -@@ -798,11 +798,11 @@ static inline unsigned long get_nr_resta - * want to handle. Thus you cannot kill init even with a SIGKILL even by - * mistake. - */ --void arch_do_signal(struct pt_regs *regs) -+void arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal) - { - struct ksignal ksig; - -- if (get_signal(&ksig)) { -+ if (has_signal && get_signal(&ksig)) { - /* Whee! Actually deliver the signal. */ - handle_signal(&ksig, regs); - return; ---- a/include/linux/entry-common.h -+++ b/include/linux/entry-common.h -@@ -37,6 +37,10 @@ - # define _TIF_UPROBE (0) - #endif - -+#ifndef _TIF_NOTIFY_SIGNAL -+# define _TIF_NOTIFY_SIGNAL (0) -+#endif -+ - /* - * TIF flags handled in syscall_enter_from_user_mode() - */ -@@ -69,7 +73,7 @@ - - #define EXIT_TO_USER_MODE_WORK \ - (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ -- _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | \ -+ _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | \ - ARCH_EXIT_TO_USER_MODE_WORK) - - /** -@@ -259,12 +263,13 @@ static __always_inline void arch_exit_to - #endif - - /** -- * arch_do_signal - Architecture specific signal delivery function -+ * arch_do_signal_or_restart - Architecture specific signal delivery function - * @regs: Pointer to currents pt_regs -+ * @has_signal: actual signal to handle - * - * Invoked from exit_to_user_mode_loop(). - */ --void arch_do_signal(struct pt_regs *regs); -+void arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal); - - /** - * arch_syscall_exit_tracehook - Wrapper around tracehook_report_syscall_exit() ---- a/include/linux/entry-kvm.h -+++ b/include/linux/entry-kvm.h -@@ -11,8 +11,8 @@ - # define ARCH_XFER_TO_GUEST_MODE_WORK (0) - #endif - --#define XFER_TO_GUEST_MODE_WORK \ -- (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ -+#define XFER_TO_GUEST_MODE_WORK \ -+ (_TIF_NEED_RESCHED | _TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL | \ - _TIF_NOTIFY_RESUME | ARCH_XFER_TO_GUEST_MODE_WORK) - - struct kvm_vcpu; ---- a/include/linux/sched/signal.h -+++ b/include/linux/sched/signal.h -@@ -361,6 +361,15 @@ static inline int task_sigpending(struct - - static inline int signal_pending(struct task_struct *p) - { -+#if defined(TIF_NOTIFY_SIGNAL) -+ /* -+ * TIF_NOTIFY_SIGNAL isn't really a signal, but it requires the same -+ * behavior in terms of ensuring that we break out of wait loops -+ * so that notify signal callbacks can be processed. -+ */ -+ if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL))) -+ return 1; -+#endif - return task_sigpending(p); - } - -@@ -508,7 +517,7 @@ extern int set_user_sigmask(const sigset - static inline void restore_saved_sigmask_unless(bool interrupted) - { - if (interrupted) -- WARN_ON(!test_thread_flag(TIF_SIGPENDING)); -+ WARN_ON(!signal_pending(current)); - else - restore_saved_sigmask(); - } ---- a/include/linux/tracehook.h -+++ b/include/linux/tracehook.h -@@ -198,4 +198,31 @@ static inline void tracehook_notify_resu - blkcg_maybe_throttle_current(); - } - -+/* -+ * called by exit_to_user_mode_loop() if ti_work & _TIF_NOTIFY_SIGNAL. This -+ * is currently used by TWA_SIGNAL based task_work, which requires breaking -+ * wait loops to ensure that task_work is noticed and run. -+ */ -+static inline void tracehook_notify_signal(void) -+{ -+#if defined(TIF_NOTIFY_SIGNAL) -+ clear_thread_flag(TIF_NOTIFY_SIGNAL); -+ smp_mb__after_atomic(); -+ if (current->task_works) -+ task_work_run(); -+#endif -+} -+ -+/* -+ * Called when we have work to process from exit_to_user_mode_loop() -+ */ -+static inline void set_notify_signal(struct task_struct *task) -+{ -+#if defined(TIF_NOTIFY_SIGNAL) -+ if (!test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) && -+ !wake_up_state(task, TASK_INTERRUPTIBLE)) -+ kick_process(task); -+#endif -+} -+ - #endif /* */ ---- a/kernel/entry/common.c -+++ b/kernel/entry/common.c -@@ -135,7 +135,15 @@ static __always_inline void exit_to_user - } - - /* Workaround to allow gradual conversion of architecture code */ --void __weak arch_do_signal(struct pt_regs *regs) { } -+void __weak arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal) { } -+ -+static void handle_signal_work(struct pt_regs *regs, unsigned long ti_work) -+{ -+ if (ti_work & _TIF_NOTIFY_SIGNAL) -+ tracehook_notify_signal(); -+ -+ arch_do_signal_or_restart(regs, ti_work & _TIF_SIGPENDING); -+} - - static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, - unsigned long ti_work) -@@ -157,8 +165,8 @@ static unsigned long exit_to_user_mode_l - if (ti_work & _TIF_PATCH_PENDING) - klp_update_patch_state(current); - -- if (ti_work & _TIF_SIGPENDING) -- arch_do_signal(regs); -+ if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) -+ handle_signal_work(regs, ti_work); - - if (ti_work & _TIF_NOTIFY_RESUME) { - tracehook_notify_resume(regs); ---- a/kernel/entry/kvm.c -+++ b/kernel/entry/kvm.c -@@ -8,6 +8,9 @@ static int xfer_to_guest_mode_work(struc - do { - int ret; - -+ if (ti_work & _TIF_NOTIFY_SIGNAL) -+ tracehook_notify_signal(); -+ - if (ti_work & _TIF_SIGPENDING) { - kvm_handle_signal_exit(vcpu); - return -EINTR; ---- a/kernel/signal.c -+++ b/kernel/signal.c -@@ -2520,6 +2520,20 @@ bool get_signal(struct ksignal *ksig) - struct signal_struct *signal = current->signal; - int signr; - -+ /* -+ * For non-generic architectures, check for TIF_NOTIFY_SIGNAL so -+ * that the arch handlers don't all have to do it. If we get here -+ * without TIF_SIGPENDING, just exit after running signal work. -+ */ -+#ifdef TIF_NOTIFY_SIGNAL -+ if (!IS_ENABLED(CONFIG_GENERIC_ENTRY)) { -+ if (test_thread_flag(TIF_NOTIFY_SIGNAL)) -+ tracehook_notify_signal(); -+ if (!task_sigpending(current)) -+ return false; -+ } -+#endif -+ - if (unlikely(uprobe_deny_signal())) - return false; - diff --git a/next-5.10/entry-kvm-exit-to-user-mode-when-tif_notify_signal-is-set.patch b/next-5.10/entry-kvm-exit-to-user-mode-when-tif_notify_signal-is-set.patch deleted file mode 100644 index 7550ea6a631..00000000000 --- a/next-5.10/entry-kvm-exit-to-user-mode-when-tif_notify_signal-is-set.patch +++ /dev/null @@ -1,51 +0,0 @@ -From 963feaebdd2ec09424d8d7dda014c4859cce19c5 Mon Sep 17 00:00:00 2001 -From: Seth Forshee -Date: Wed, 4 May 2022 13:08:40 -0500 -Subject: entry/kvm: Exit to user mode when TIF_NOTIFY_SIGNAL is set - -From: Seth Forshee - -[ Upstream commit 3e684903a8574ffc9475fdf13c4780a7adb506ad ] - -A livepatch transition may stall indefinitely when a kvm vCPU is heavily -loaded. To the host, the vCPU task is a user thread which is spending a -very long time in the ioctl(KVM_RUN) syscall. During livepatch -transition, set_notify_signal() will be called on such tasks to -interrupt the syscall so that the task can be transitioned. This -interrupts guest execution, but when xfer_to_guest_mode_work() sees that -TIF_NOTIFY_SIGNAL is set but not TIF_SIGPENDING it concludes that an -exit to user mode is unnecessary, and guest execution is resumed without -transitioning the task for the livepatch. - -This handling of TIF_NOTIFY_SIGNAL is incorrect, as set_notify_signal() -is expected to break tasks out of interruptible kernel loops and cause -them to return to userspace. Change xfer_to_guest_mode_work() to handle -TIF_NOTIFY_SIGNAL the same as TIF_SIGPENDING, signaling to the vCPU run -loop that an exit to userpsace is needed. Any pending task_work will be -run when get_signal() is called from exit_to_user_mode_loop(), so there -is no longer any need to run task work from xfer_to_guest_mode_work(). - -Suggested-by: "Eric W. Biederman" -Cc: Petr Mladek -Signed-off-by: Seth Forshee -Message-Id: <20220504180840.2907296-1-sforshee@digitalocean.com> -Signed-off-by: Paolo Bonzini -Signed-off-by: Greg Kroah-Hartman ---- - kernel/entry/kvm.c | 5 +---- - 1 file changed, 1 insertion(+), 4 deletions(-) - ---- a/kernel/entry/kvm.c -+++ b/kernel/entry/kvm.c -@@ -8,10 +8,7 @@ static int xfer_to_guest_mode_work(struc - do { - int ret; - -- if (ti_work & _TIF_NOTIFY_SIGNAL) -- tracehook_notify_signal(); -- -- if (ti_work & _TIF_SIGPENDING) { -+ if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) { - kvm_handle_signal_exit(vcpu); - return -EINTR; - } diff --git a/next-5.10/eventfd-provide-a-eventfd_signal_mask-helper.patch b/next-5.10/eventfd-provide-a-eventfd_signal_mask-helper.patch deleted file mode 100644 index 10b94de1ffe..00000000000 --- a/next-5.10/eventfd-provide-a-eventfd_signal_mask-helper.patch +++ /dev/null @@ -1,113 +0,0 @@ -From 53270319f184c35499cf2ffeac8a75078b94934a Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Sun, 20 Nov 2022 10:13:44 -0700 -Subject: eventfd: provide a eventfd_signal_mask() helper - -From: Jens Axboe - -[ Upstream commit 03e02acda8e267a8183e1e0ed289ff1ef9cd7ed8 ] - -This is identical to eventfd_signal(), but it allows the caller to pass -in a mask to be used for the poll wakeup key. The use case is avoiding -repeated multishot triggers if we have a dependency between eventfd and -io_uring. - -If we setup an eventfd context and register that as the io_uring eventfd, -and at the same time queue a multishot poll request for the eventfd -context, then any CQE posted will repeatedly trigger the multishot request -until it terminates when the CQ ring overflows. - -In preparation for io_uring detecting this circular dependency, add the -mentioned helper so that io_uring can pass in EPOLL_URING as part of the -poll wakeup key. - -Cc: stable@vger.kernel.org # 6.0 -[axboe: fold in !CONFIG_EVENTFD fix from Zhang Qilong] -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - fs/eventfd.c | 37 +++++++++++++++++++++---------------- - include/linux/eventfd.h | 7 +++++++ - 2 files changed, 28 insertions(+), 16 deletions(-) - ---- a/fs/eventfd.c -+++ b/fs/eventfd.c -@@ -45,21 +45,7 @@ struct eventfd_ctx { - int id; - }; - --/** -- * eventfd_signal - Adds @n to the eventfd counter. -- * @ctx: [in] Pointer to the eventfd context. -- * @n: [in] Value of the counter to be added to the eventfd internal counter. -- * The value cannot be negative. -- * -- * This function is supposed to be called by the kernel in paths that do not -- * allow sleeping. In this function we allow the counter to reach the ULLONG_MAX -- * value, and we signal this as overflow condition by returning a EPOLLERR -- * to poll(2). -- * -- * Returns the amount by which the counter was incremented. This will be less -- * than @n if the counter has overflowed. -- */ --__u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n) -+__u64 eventfd_signal_mask(struct eventfd_ctx *ctx, __u64 n, unsigned mask) - { - unsigned long flags; - -@@ -80,12 +66,31 @@ __u64 eventfd_signal(struct eventfd_ctx - n = ULLONG_MAX - ctx->count; - ctx->count += n; - if (waitqueue_active(&ctx->wqh)) -- wake_up_locked_poll(&ctx->wqh, EPOLLIN); -+ wake_up_locked_poll(&ctx->wqh, EPOLLIN | mask); - this_cpu_dec(eventfd_wake_count); - spin_unlock_irqrestore(&ctx->wqh.lock, flags); - - return n; - } -+ -+/** -+ * eventfd_signal - Adds @n to the eventfd counter. -+ * @ctx: [in] Pointer to the eventfd context. -+ * @n: [in] Value of the counter to be added to the eventfd internal counter. -+ * The value cannot be negative. -+ * -+ * This function is supposed to be called by the kernel in paths that do not -+ * allow sleeping. In this function we allow the counter to reach the ULLONG_MAX -+ * value, and we signal this as overflow condition by returning a EPOLLERR -+ * to poll(2). -+ * -+ * Returns the amount by which the counter was incremented. This will be less -+ * than @n if the counter has overflowed. -+ */ -+__u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n) -+{ -+ return eventfd_signal_mask(ctx, n, 0); -+} - EXPORT_SYMBOL_GPL(eventfd_signal); - - static void eventfd_free_ctx(struct eventfd_ctx *ctx) ---- a/include/linux/eventfd.h -+++ b/include/linux/eventfd.h -@@ -39,6 +39,7 @@ struct file *eventfd_fget(int fd); - struct eventfd_ctx *eventfd_ctx_fdget(int fd); - struct eventfd_ctx *eventfd_ctx_fileget(struct file *file); - __u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n); -+__u64 eventfd_signal_mask(struct eventfd_ctx *ctx, __u64 n, unsigned mask); - int eventfd_ctx_remove_wait_queue(struct eventfd_ctx *ctx, wait_queue_entry_t *wait, - __u64 *cnt); - -@@ -65,6 +66,12 @@ static inline int eventfd_signal(struct - { - return -ENOSYS; - } -+ -+static inline int eventfd_signal_mask(struct eventfd_ctx *ctx, __u64 n, -+ unsigned mask) -+{ -+ return -ENOSYS; -+} - - static inline void eventfd_ctx_put(struct eventfd_ctx *ctx) - { diff --git a/next-5.10/eventpoll-add-epoll_uring_wake-poll-wakeup-flag.patch b/next-5.10/eventpoll-add-epoll_uring_wake-poll-wakeup-flag.patch deleted file mode 100644 index a0b3e63e48d..00000000000 --- a/next-5.10/eventpoll-add-epoll_uring_wake-poll-wakeup-flag.patch +++ /dev/null @@ -1,112 +0,0 @@ -From 5d0d6c3d70a751ae214e471989fc729956c6fccb Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Sun, 20 Nov 2022 10:10:53 -0700 -Subject: eventpoll: add EPOLL_URING_WAKE poll wakeup flag - -From: Jens Axboe - -[ Upstream commit caf1aeaffc3b09649a56769e559333ae2c4f1802 ] - -We can have dependencies between epoll and io_uring. Consider an epoll -context, identified by the epfd file descriptor, and an io_uring file -descriptor identified by iofd. If we add iofd to the epfd context, and -arm a multishot poll request for epfd with iofd, then the multishot -poll request will repeatedly trigger and generate events until terminated -by CQ ring overflow. This isn't a desired behavior. - -Add EPOLL_URING so that io_uring can pass it in as part of the poll wakeup -key, and io_uring can check for that to detect a potential recursive -invocation. - -Cc: stable@vger.kernel.org # 6.0 -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - fs/eventpoll.c | 18 ++++++++++-------- - include/uapi/linux/eventpoll.h | 6 ++++++ - 2 files changed, 16 insertions(+), 8 deletions(-) - ---- a/fs/eventpoll.c -+++ b/fs/eventpoll.c -@@ -548,7 +548,8 @@ out_unlock: - */ - #ifdef CONFIG_DEBUG_LOCK_ALLOC - --static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi) -+static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi, -+ unsigned pollflags) - { - struct eventpoll *ep_src; - unsigned long flags; -@@ -579,16 +580,17 @@ static void ep_poll_safewake(struct even - } - spin_lock_irqsave_nested(&ep->poll_wait.lock, flags, nests); - ep->nests = nests + 1; -- wake_up_locked_poll(&ep->poll_wait, EPOLLIN); -+ wake_up_locked_poll(&ep->poll_wait, EPOLLIN | pollflags); - ep->nests = 0; - spin_unlock_irqrestore(&ep->poll_wait.lock, flags); - } - - #else - --static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi) -+static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi, -+ unsigned pollflags) - { -- wake_up_poll(&ep->poll_wait, EPOLLIN); -+ wake_up_poll(&ep->poll_wait, EPOLLIN | pollflags); - } - - #endif -@@ -815,7 +817,7 @@ static void ep_free(struct eventpoll *ep - - /* We need to release all tasks waiting for these file */ - if (waitqueue_active(&ep->poll_wait)) -- ep_poll_safewake(ep, NULL); -+ ep_poll_safewake(ep, NULL, 0); - - /* - * We need to lock this because we could be hit by -@@ -1284,7 +1286,7 @@ out_unlock: - - /* We have to call this outside the lock */ - if (pwake) -- ep_poll_safewake(ep, epi); -+ ep_poll_safewake(ep, epi, pollflags & EPOLL_URING_WAKE); - - if (!(epi->event.events & EPOLLEXCLUSIVE)) - ewake = 1; -@@ -1589,7 +1591,7 @@ static int ep_insert(struct eventpoll *e - - /* We have to call this outside the lock */ - if (pwake) -- ep_poll_safewake(ep, NULL); -+ ep_poll_safewake(ep, NULL, 0); - - return 0; - -@@ -1692,7 +1694,7 @@ static int ep_modify(struct eventpoll *e - - /* We have to call this outside the lock */ - if (pwake) -- ep_poll_safewake(ep, NULL); -+ ep_poll_safewake(ep, NULL, 0); - - return 0; - } ---- a/include/uapi/linux/eventpoll.h -+++ b/include/uapi/linux/eventpoll.h -@@ -41,6 +41,12 @@ - #define EPOLLMSG (__force __poll_t)0x00000400 - #define EPOLLRDHUP (__force __poll_t)0x00002000 - -+/* -+ * Internal flag - wakeup generated by io_uring, used to detect recursion back -+ * into the io_uring poll handler. -+ */ -+#define EPOLL_URING_WAKE ((__force __poll_t)(1U << 27)) -+ - /* Set exclusive wakeup mode for the target file descriptor */ - #define EPOLLEXCLUSIVE ((__force __poll_t)(1U << 28)) - diff --git a/next-5.10/file-rename-__close_fd_get_file-close_fd_get_file.patch b/next-5.10/file-rename-__close_fd_get_file-close_fd_get_file.patch deleted file mode 100644 index 6ab08626cf4..00000000000 --- a/next-5.10/file-rename-__close_fd_get_file-close_fd_get_file.patch +++ /dev/null @@ -1,80 +0,0 @@ -From ba51a050af49c1df47fe79687d5f14256620d508 Mon Sep 17 00:00:00 2001 -From: "Eric W. Biederman" -Date: Fri, 20 Nov 2020 17:14:40 -0600 -Subject: file: Rename __close_fd_get_file close_fd_get_file - -From: "Eric W. Biederman" - -[ Upstream commit 9fe83c43e71cdb8e5b9520bcb98706a2b3c680c8 ] - -The function close_fd_get_file is explicitly a variant of -__close_fd[1]. Now that __close_fd has been renamed close_fd, rename -close_fd_get_file to be consistent with close_fd. - -When __alloc_fd, __close_fd and __fd_install were introduced the -double underscore indicated that the function took a struct -files_struct parameter. The function __close_fd_get_file never has so -the naming has always been inconsistent. This just cleans things up -so there are not any lingering mentions or references __close_fd left -in the code. - -[1] 80cd795630d6 ("binder: fix use-after-free due to ksys_close() during fdget()") -Link: https://lkml.kernel.org/r/20201120231441.29911-23-ebiederm@xmission.com -Signed-off-by: Eric W. Biederman -Signed-off-by: Greg Kroah-Hartman ---- - drivers/android/binder.c | 2 +- - fs/file.c | 4 ++-- - fs/io_uring.c | 2 +- - include/linux/fdtable.h | 2 +- - 4 files changed, 5 insertions(+), 5 deletions(-) - ---- a/drivers/android/binder.c -+++ b/drivers/android/binder.c -@@ -2255,7 +2255,7 @@ static void binder_deferred_fd_close(int - if (!twcb) - return; - init_task_work(&twcb->twork, binder_do_fd_close); -- __close_fd_get_file(fd, &twcb->file); -+ close_fd_get_file(fd, &twcb->file); - if (twcb->file) { - filp_close(twcb->file, current->files); - task_work_add(current, &twcb->twork, TWA_RESUME); ---- a/fs/file.c -+++ b/fs/file.c -@@ -780,11 +780,11 @@ int __close_range(unsigned fd, unsigned - } - - /* -- * variant of __close_fd that gets a ref on the file for later fput. -+ * variant of close_fd that gets a ref on the file for later fput. - * The caller must ensure that filp_close() called on the file, and then - * an fput(). - */ --int __close_fd_get_file(unsigned int fd, struct file **res) -+int close_fd_get_file(unsigned int fd, struct file **res) - { - struct files_struct *files = current->files; - struct file *file; ---- a/fs/io_uring.c -+++ b/fs/io_uring.c -@@ -4270,7 +4270,7 @@ static int io_close(struct io_kiocb *req - - /* might be already done during nonblock submission */ - if (!close->put_file) { -- ret = __close_fd_get_file(close->fd, &close->put_file); -+ ret = close_fd_get_file(close->fd, &close->put_file); - if (ret < 0) - return (ret == -ENOENT) ? -EBADF : ret; - } ---- a/include/linux/fdtable.h -+++ b/include/linux/fdtable.h -@@ -123,7 +123,7 @@ extern void __fd_install(struct files_st - extern int __close_fd(struct files_struct *files, - unsigned int fd); - extern int __close_range(unsigned int fd, unsigned int max_fd, unsigned int flags); --extern int __close_fd_get_file(unsigned int fd, struct file **res); -+extern int close_fd_get_file(unsigned int fd, struct file **res); - extern int unshare_fd(unsigned long unshare_flags, unsigned int max_fds, - struct files_struct **new_fdp); - diff --git a/next-5.10/fix-handling-of-nd-depth-on-lookup_cached-failures-in-try_to_unlazy.patch b/next-5.10/fix-handling-of-nd-depth-on-lookup_cached-failures-in-try_to_unlazy.patch deleted file mode 100644 index f576a7a414c..00000000000 --- a/next-5.10/fix-handling-of-nd-depth-on-lookup_cached-failures-in-try_to_unlazy.patch +++ /dev/null @@ -1,64 +0,0 @@ -From b93a6e389bdc72d423e666cbb5475ca7c7c4deb9 Mon Sep 17 00:00:00 2001 -From: Al Viro -Date: Mon, 15 Feb 2021 12:03:23 -0500 -Subject: fix handling of nd->depth on LOOKUP_CACHED failures in try_to_unlazy* - -From: Al Viro - -[ Upstream commit eacd9aa8cedeb412842c7b339adbaa0477fdd5ad ] - -After switching to non-RCU mode, we want nd->depth to match the number -of entries in nd->stack[] that need eventual path_put(). -legitimize_links() takes care of that on failures; unfortunately, -failure exits added for LOOKUP_CACHED do not. - -We could add the logics for that into those failure exits, both in -try_to_unlazy() and in try_to_unlazy_next(), but since both checks -are immediately followed by legitimize_links() and there's no calls -of legitimize_links() other than those two... It's easier to -move the check (and required handling of nd->depth on failure) into -legitimize_links() itself. - -[caught by Jens: ... and since we are zeroing ->depth here, we need -to do drop_links() first] - -Fixes: 6c6ec2b0a3e0 "fs: add support for LOOKUP_CACHED" -Tested-by: Jens Axboe -Signed-off-by: Al Viro -Signed-off-by: Greg Kroah-Hartman ---- - fs/namei.c | 9 +++++---- - 1 file changed, 5 insertions(+), 4 deletions(-) - ---- a/fs/namei.c -+++ b/fs/namei.c -@@ -630,6 +630,11 @@ static inline bool legitimize_path(struc - static bool legitimize_links(struct nameidata *nd) - { - int i; -+ if (unlikely(nd->flags & LOOKUP_CACHED)) { -+ drop_links(nd); -+ nd->depth = 0; -+ return false; -+ } - for (i = 0; i < nd->depth; i++) { - struct saved *last = nd->stack + i; - if (unlikely(!legitimize_path(nd, &last->link, last->seq))) { -@@ -686,8 +691,6 @@ static bool try_to_unlazy(struct nameida - BUG_ON(!(nd->flags & LOOKUP_RCU)); - - nd->flags &= ~LOOKUP_RCU; -- if (nd->flags & LOOKUP_CACHED) -- goto out1; - if (unlikely(!legitimize_links(nd))) - goto out1; - if (unlikely(!legitimize_path(nd, &nd->path, nd->seq))) -@@ -724,8 +727,6 @@ static bool try_to_unlazy_next(struct na - BUG_ON(!(nd->flags & LOOKUP_RCU)); - - nd->flags &= ~LOOKUP_RCU; -- if (nd->flags & LOOKUP_CACHED) -- goto out2; - if (unlikely(!legitimize_links(nd))) - goto out2; - if (unlikely(!legitimize_mnt(nd->path.mnt, nd->m_seq))) diff --git a/next-5.10/fs-add-support-for-lookup_cached.patch b/next-5.10/fs-add-support-for-lookup_cached.patch deleted file mode 100644 index 1c68635dfeb..00000000000 --- a/next-5.10/fs-add-support-for-lookup_cached.patch +++ /dev/null @@ -1,76 +0,0 @@ -From 53e825b10aa8f037c19c51997f5f8402d046d883 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Thu, 17 Dec 2020 09:19:09 -0700 -Subject: fs: add support for LOOKUP_CACHED - -From: Jens Axboe - -[ Upstream commit 6c6ec2b0a3e0381d886d531bd1471dfdb1509237 ] - -io_uring always punts opens to async context, since there's no control -over whether the lookup blocks or not. Add LOOKUP_CACHED to support -just doing the fast RCU based lookups, which we know will not block. If -we can do a cached path resolution of the filename, then we don't have -to always punt lookups for a worker. - -During path resolution, we always do LOOKUP_RCU first. If that fails and -we terminate LOOKUP_RCU, then fail a LOOKUP_CACHED attempt as well. - -Cc: Al Viro -Signed-off-by: Jens Axboe -Signed-off-by: Al Viro -Signed-off-by: Greg Kroah-Hartman ---- - fs/namei.c | 9 +++++++++ - include/linux/namei.h | 1 + - 2 files changed, 10 insertions(+) - ---- a/fs/namei.c -+++ b/fs/namei.c -@@ -686,6 +686,8 @@ static bool try_to_unlazy(struct nameida - BUG_ON(!(nd->flags & LOOKUP_RCU)); - - nd->flags &= ~LOOKUP_RCU; -+ if (nd->flags & LOOKUP_CACHED) -+ goto out1; - if (unlikely(!legitimize_links(nd))) - goto out1; - if (unlikely(!legitimize_path(nd, &nd->path, nd->seq))) -@@ -722,6 +724,8 @@ static bool try_to_unlazy_next(struct na - BUG_ON(!(nd->flags & LOOKUP_RCU)); - - nd->flags &= ~LOOKUP_RCU; -+ if (nd->flags & LOOKUP_CACHED) -+ goto out2; - if (unlikely(!legitimize_links(nd))) - goto out2; - if (unlikely(!legitimize_mnt(nd->path.mnt, nd->m_seq))) -@@ -792,6 +796,7 @@ static int complete_walk(struct nameidat - */ - if (!(nd->flags & (LOOKUP_ROOT | LOOKUP_IS_SCOPED))) - nd->root.mnt = NULL; -+ nd->flags &= ~LOOKUP_CACHED; - if (!try_to_unlazy(nd)) - return -ECHILD; - } -@@ -2204,6 +2209,10 @@ static const char *path_init(struct name - int error; - const char *s = nd->name->name; - -+ /* LOOKUP_CACHED requires RCU, ask caller to retry */ -+ if ((flags & (LOOKUP_RCU | LOOKUP_CACHED)) == LOOKUP_CACHED) -+ return ERR_PTR(-EAGAIN); -+ - if (!*s) - flags &= ~LOOKUP_RCU; - if (flags & LOOKUP_RCU) ---- a/include/linux/namei.h -+++ b/include/linux/namei.h -@@ -46,6 +46,7 @@ enum {LAST_NORM, LAST_ROOT, LAST_DOT, LA - #define LOOKUP_NO_XDEV 0x040000 /* No mountpoint crossing. */ - #define LOOKUP_BENEATH 0x080000 /* No escaping from starting point. */ - #define LOOKUP_IN_ROOT 0x100000 /* Treat dirfd as fs root. */ -+#define LOOKUP_CACHED 0x200000 /* Only do cached lookup */ - /* LOOKUP_* flags which do scope-related checks based on the dirfd. */ - #define LOOKUP_IS_SCOPED (LOOKUP_BENEATH | LOOKUP_IN_ROOT) - diff --git a/next-5.10/fs-expose-lookup_cached-through-openat2-resolve_cached.patch b/next-5.10/fs-expose-lookup_cached-through-openat2-resolve_cached.patch deleted file mode 100644 index e9297d32c35..00000000000 --- a/next-5.10/fs-expose-lookup_cached-through-openat2-resolve_cached.patch +++ /dev/null @@ -1,65 +0,0 @@ -From cb9294e007feacca1a5849773460dde71e50cdde Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Thu, 17 Dec 2020 09:19:10 -0700 -Subject: fs: expose LOOKUP_CACHED through openat2() RESOLVE_CACHED - -From: Jens Axboe - -[ Upstream commit 99668f618062816ca7ba639b007eb145b9d3d41e ] - -Now that we support non-blocking path resolution internally, expose it -via openat2() in the struct open_how ->resolve flags. This allows -applications using openat2() to limit path resolution to the extent that -it is already cached. - -If the lookup cannot be satisfied in a non-blocking manner, openat2(2) -will return -1/-EAGAIN. - -Cc: Al Viro -Signed-off-by: Jens Axboe -Signed-off-by: Al Viro -Signed-off-by: Greg Kroah-Hartman ---- - fs/open.c | 6 ++++++ - include/linux/fcntl.h | 2 +- - include/uapi/linux/openat2.h | 4 ++++ - 3 files changed, 11 insertions(+), 1 deletion(-) - ---- a/fs/open.c -+++ b/fs/open.c -@@ -1099,6 +1099,12 @@ inline int build_open_flags(const struct - lookup_flags |= LOOKUP_BENEATH; - if (how->resolve & RESOLVE_IN_ROOT) - lookup_flags |= LOOKUP_IN_ROOT; -+ if (how->resolve & RESOLVE_CACHED) { -+ /* Don't bother even trying for create/truncate/tmpfile open */ -+ if (flags & (O_TRUNC | O_CREAT | O_TMPFILE)) -+ return -EAGAIN; -+ lookup_flags |= LOOKUP_CACHED; -+ } - - op->lookup_flags = lookup_flags; - return 0; ---- a/include/linux/fcntl.h -+++ b/include/linux/fcntl.h -@@ -19,7 +19,7 @@ - /* List of all valid flags for the how->resolve argument: */ - #define VALID_RESOLVE_FLAGS \ - (RESOLVE_NO_XDEV | RESOLVE_NO_MAGICLINKS | RESOLVE_NO_SYMLINKS | \ -- RESOLVE_BENEATH | RESOLVE_IN_ROOT) -+ RESOLVE_BENEATH | RESOLVE_IN_ROOT | RESOLVE_CACHED) - - /* List of all open_how "versions". */ - #define OPEN_HOW_SIZE_VER0 24 /* sizeof first published struct */ ---- a/include/uapi/linux/openat2.h -+++ b/include/uapi/linux/openat2.h -@@ -35,5 +35,9 @@ struct open_how { - #define RESOLVE_IN_ROOT 0x10 /* Make all jumps to "/" and ".." - be scoped inside the dirfd - (similar to chroot(2)). */ -+#define RESOLVE_CACHED 0x20 /* Only complete if resolution can be -+ completed through cached lookup. May -+ return -EAGAIN if that's not -+ possible. */ - - #endif /* _UAPI_LINUX_OPENAT2_H */ diff --git a/next-5.10/fs-make-do_renameat2-take-struct-filename.patch b/next-5.10/fs-make-do_renameat2-take-struct-filename.patch deleted file mode 100644 index e24e860c508..00000000000 --- a/next-5.10/fs-make-do_renameat2-take-struct-filename.patch +++ /dev/null @@ -1,134 +0,0 @@ -From aa359e48cc999513b0167703009d3db363818540 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Sat, 26 Sep 2020 17:20:17 -0600 -Subject: fs: make do_renameat2() take struct filename - -From: Jens Axboe - -[ Upstream commit e886663cfd029b64a1d8da7efae7014526d884e9 ] - -Pass in the struct filename pointers instead of the user string, and -update the three callers to do the same. - -This behaves like do_unlinkat(), which also takes a filename struct and -puts it when it is done. Converting callers is then trivial. - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - fs/internal.h | 2 ++ - fs/namei.c | 40 ++++++++++++++++++++++------------------ - 2 files changed, 24 insertions(+), 18 deletions(-) - ---- a/fs/internal.h -+++ b/fs/internal.h -@@ -77,6 +77,8 @@ extern int vfs_path_lookup(struct dentry - long do_rmdir(int dfd, struct filename *name); - long do_unlinkat(int dfd, struct filename *name); - int may_linkat(struct path *link); -+int do_renameat2(int olddfd, struct filename *oldname, int newdfd, -+ struct filename *newname, unsigned int flags); - - /* - * namespace.c ---- a/fs/namei.c -+++ b/fs/namei.c -@@ -4353,8 +4353,8 @@ out: - } - EXPORT_SYMBOL(vfs_rename); - --static int do_renameat2(int olddfd, const char __user *oldname, int newdfd, -- const char __user *newname, unsigned int flags) -+int do_renameat2(int olddfd, struct filename *from, int newdfd, -+ struct filename *to, unsigned int flags) - { - struct dentry *old_dentry, *new_dentry; - struct dentry *trap; -@@ -4362,32 +4362,30 @@ static int do_renameat2(int olddfd, cons - struct qstr old_last, new_last; - int old_type, new_type; - struct inode *delegated_inode = NULL; -- struct filename *from; -- struct filename *to; - unsigned int lookup_flags = 0, target_flags = LOOKUP_RENAME_TARGET; - bool should_retry = false; -- int error; -+ int error = -EINVAL; - - if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) -- return -EINVAL; -+ goto put_both; - - if ((flags & (RENAME_NOREPLACE | RENAME_WHITEOUT)) && - (flags & RENAME_EXCHANGE)) -- return -EINVAL; -+ goto put_both; - - if (flags & RENAME_EXCHANGE) - target_flags = 0; - - retry: -- from = filename_parentat(olddfd, getname(oldname), lookup_flags, -- &old_path, &old_last, &old_type); -+ from = filename_parentat(olddfd, from, lookup_flags, &old_path, -+ &old_last, &old_type); - if (IS_ERR(from)) { - error = PTR_ERR(from); -- goto exit; -+ goto put_new; - } - -- to = filename_parentat(newdfd, getname(newname), lookup_flags, -- &new_path, &new_last, &new_type); -+ to = filename_parentat(newdfd, to, lookup_flags, &new_path, &new_last, -+ &new_type); - if (IS_ERR(to)) { - error = PTR_ERR(to); - goto exit1; -@@ -4480,34 +4478,40 @@ exit2: - if (retry_estale(error, lookup_flags)) - should_retry = true; - path_put(&new_path); -- putname(to); - exit1: - path_put(&old_path); -- putname(from); - if (should_retry) { - should_retry = false; - lookup_flags |= LOOKUP_REVAL; - goto retry; - } --exit: -+put_both: -+ if (!IS_ERR(from)) -+ putname(from); -+put_new: -+ if (!IS_ERR(to)) -+ putname(to); - return error; - } - - SYSCALL_DEFINE5(renameat2, int, olddfd, const char __user *, oldname, - int, newdfd, const char __user *, newname, unsigned int, flags) - { -- return do_renameat2(olddfd, oldname, newdfd, newname, flags); -+ return do_renameat2(olddfd, getname(oldname), newdfd, getname(newname), -+ flags); - } - - SYSCALL_DEFINE4(renameat, int, olddfd, const char __user *, oldname, - int, newdfd, const char __user *, newname) - { -- return do_renameat2(olddfd, oldname, newdfd, newname, 0); -+ return do_renameat2(olddfd, getname(oldname), newdfd, getname(newname), -+ 0); - } - - SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newname) - { -- return do_renameat2(AT_FDCWD, oldname, AT_FDCWD, newname, 0); -+ return do_renameat2(AT_FDCWD, getname(oldname), AT_FDCWD, -+ getname(newname), 0); - } - - int readlink_copy(char __user *buffer, int buflen, const char *link) diff --git a/next-5.10/fs-provide-locked-helper-variant-of-close_fd_get_file.patch b/next-5.10/fs-provide-locked-helper-variant-of-close_fd_get_file.patch deleted file mode 100644 index 5fdefdff4fe..00000000000 --- a/next-5.10/fs-provide-locked-helper-variant-of-close_fd_get_file.patch +++ /dev/null @@ -1,100 +0,0 @@ -From 369d83e272a9f1a316ce79b1236f1f3265e40efe Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Tue, 19 Jan 2021 15:41:52 -0700 -Subject: fs: provide locked helper variant of close_fd_get_file() - -From: Jens Axboe - -[ Upstream commit 53dec2ea74f2ef360e8455439be96a780baa6097 ] - -Assumes current->files->file_lock is already held on invocation. Helps -the caller check the file before removing the fd, if it needs to. - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - fs/file.c | 36 +++++++++++++++++++++++++----------- - fs/internal.h | 1 + - 2 files changed, 26 insertions(+), 11 deletions(-) - ---- a/fs/file.c -+++ b/fs/file.c -@@ -22,6 +22,8 @@ - #include - #include - -+#include "internal.h" -+ - unsigned int sysctl_nr_open __read_mostly = 1024*1024; - unsigned int sysctl_nr_open_min = BITS_PER_LONG; - /* our min() is unusable in constant expressions ;-/ */ -@@ -780,36 +782,48 @@ int __close_range(unsigned fd, unsigned - } - - /* -- * variant of close_fd that gets a ref on the file for later fput. -- * The caller must ensure that filp_close() called on the file, and then -- * an fput(). -+ * See close_fd_get_file() below, this variant assumes current->files->file_lock -+ * is held. - */ --int close_fd_get_file(unsigned int fd, struct file **res) -+int __close_fd_get_file(unsigned int fd, struct file **res) - { - struct files_struct *files = current->files; - struct file *file; - struct fdtable *fdt; - -- spin_lock(&files->file_lock); - fdt = files_fdtable(files); - if (fd >= fdt->max_fds) -- goto out_unlock; -+ goto out_err; - file = fdt->fd[fd]; - if (!file) -- goto out_unlock; -+ goto out_err; - rcu_assign_pointer(fdt->fd[fd], NULL); - __put_unused_fd(files, fd); -- spin_unlock(&files->file_lock); - get_file(file); - *res = file; - return 0; -- --out_unlock: -- spin_unlock(&files->file_lock); -+out_err: - *res = NULL; - return -ENOENT; - } - -+/* -+ * variant of close_fd that gets a ref on the file for later fput. -+ * The caller must ensure that filp_close() called on the file, and then -+ * an fput(). -+ */ -+int close_fd_get_file(unsigned int fd, struct file **res) -+{ -+ struct files_struct *files = current->files; -+ int ret; -+ -+ spin_lock(&files->file_lock); -+ ret = __close_fd_get_file(fd, res); -+ spin_unlock(&files->file_lock); -+ -+ return ret; -+} -+ - void do_close_on_exec(struct files_struct *files) - { - unsigned i; ---- a/fs/internal.h -+++ b/fs/internal.h -@@ -134,6 +134,7 @@ extern struct file *do_file_open_root(st - const char *, const struct open_flags *); - extern struct open_how build_open_how(int flags, umode_t mode); - extern int build_open_flags(const struct open_how *how, struct open_flags *op); -+extern int __close_fd_get_file(unsigned int fd, struct file **res); - - long do_sys_ftruncate(unsigned int fd, loff_t length, int small); - int chmod_common(const struct path *path, umode_t mode); diff --git a/next-5.10/h8300-add-support-for-tif_notify_signal.patch b/next-5.10/h8300-add-support-for-tif_notify_signal.patch deleted file mode 100644 index 3c866d2dcb3..00000000000 --- a/next-5.10/h8300-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,57 +0,0 @@ -From b83ee681e7dd713540f41cadb6217784083b38ec Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 14:42:33 -0600 -Subject: h8300: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 2f9799ad0111ee742ccc02dd2ea2c87646746fc1 ] - -Wire up TIF_NOTIFY_SIGNAL handling for h8300. - -Cc: uclinux-h8-devel@lists.sourceforge.jp -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/h8300/include/asm/thread_info.h | 4 +++- - arch/h8300/kernel/signal.c | 2 +- - 2 files changed, 4 insertions(+), 2 deletions(-) - ---- a/arch/h8300/include/asm/thread_info.h -+++ b/arch/h8300/include/asm/thread_info.h -@@ -73,6 +73,7 @@ static inline struct thread_info *curren - #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ - #define TIF_SYSCALL_TRACEPOINT 8 /* for ftrace syscall instrumentation */ - #define TIF_POLLING_NRFLAG 9 /* true if poll_idle() is polling TIF_NEED_RESCHED */ -+#define TIF_NOTIFY_SIGNAL 10 /* signal notifications exist */ - - /* as above, but as bit values */ - #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) -@@ -83,6 +84,7 @@ static inline struct thread_info *curren - #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) - #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) - #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - - /* work to do in syscall trace */ - #define _TIF_WORK_SYSCALL_MASK (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP | \ -@@ -92,7 +94,7 @@ static inline struct thread_info *curren - #define _TIF_ALLWORK_MASK (_TIF_SYSCALL_TRACE | _TIF_SIGPENDING | \ - _TIF_NEED_RESCHED | _TIF_SYSCALL_AUDIT | \ - _TIF_SINGLESTEP | _TIF_NOTIFY_RESUME | \ -- _TIF_SYSCALL_TRACEPOINT) -+ _TIF_SYSCALL_TRACEPOINT | _TIF_NOTIFY_SIGNAL) - - /* work to do on interrupt/exception return */ - #define _TIF_WORK_MASK (_TIF_ALLWORK_MASK & ~(_TIF_SYSCALL_TRACE | \ ---- a/arch/h8300/kernel/signal.c -+++ b/arch/h8300/kernel/signal.c -@@ -279,7 +279,7 @@ static void do_signal(struct pt_regs *re - - asmlinkage void do_notify_resume(struct pt_regs *regs, u32 thread_info_flags) - { -- if (thread_info_flags & _TIF_SIGPENDING) -+ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) - do_signal(regs); - - if (thread_info_flags & _TIF_NOTIFY_RESUME) diff --git a/next-5.10/hexagon-add-support-for-tif_notify_signal.patch b/next-5.10/hexagon-add-support-for-tif_notify_signal.patch deleted file mode 100644 index 8d1106b7038..00000000000 --- a/next-5.10/hexagon-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,49 +0,0 @@ -From ba4b24618290c7ee52180aa3706729e4fb2fd4e4 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 14:45:22 -0600 -Subject: hexagon: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit aeec8193578a71d0aee21218351849d38121ce90 ] - -Wire up TIF_NOTIFY_SIGNAL handling for hexagon. - -Cc: linux-hexagon@vger.kernel.org -Acked-by: Brian Cain -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/hexagon/include/asm/thread_info.h | 2 ++ - arch/hexagon/kernel/process.c | 2 +- - 2 files changed, 3 insertions(+), 1 deletion(-) - ---- a/arch/hexagon/include/asm/thread_info.h -+++ b/arch/hexagon/include/asm/thread_info.h -@@ -95,6 +95,7 @@ register struct thread_info *__current_t - #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ - #define TIF_SINGLESTEP 4 /* restore ss @ return to usr mode */ - #define TIF_RESTORE_SIGMASK 6 /* restore sig mask in do_signal() */ -+#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ - /* true if poll_idle() is polling TIF_NEED_RESCHED */ - #define TIF_MEMDIE 17 /* OOM killer killed process */ - -@@ -103,6 +104,7 @@ register struct thread_info *__current_t - #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) - #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) - #define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - - /* work to do on interrupt/exception return - All but TIF_SYSCALL_TRACE */ - #define _TIF_WORK_MASK (0x0000FFFF & ~_TIF_SYSCALL_TRACE) ---- a/arch/hexagon/kernel/process.c -+++ b/arch/hexagon/kernel/process.c -@@ -174,7 +174,7 @@ int do_work_pending(struct pt_regs *regs - return 1; - } - -- if (thread_info_flags & _TIF_SIGPENDING) { -+ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) { - do_signal(regs); - return 1; - } diff --git a/next-5.10/ia64-add-support-for-tif_notify_signal.patch b/next-5.10/ia64-add-support-for-tif_notify_signal.patch deleted file mode 100644 index a327b343b51..00000000000 --- a/next-5.10/ia64-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,59 +0,0 @@ -From 4732812234a886b6e508d15ebb8ba1f61cb82c87 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 14:49:43 -0600 -Subject: ia64: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit b269c229b0e89aedb7943c06673b56b6052cf5e5 ] - -Wire up TIF_NOTIFY_SIGNAL handling for ia64. - -Cc: linux-ia64@vger.kernel.org -[axboe: added fixes from Mike Rapoport ] -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/ia64/include/asm/thread_info.h | 4 +++- - arch/ia64/kernel/process.c | 3 ++- - 2 files changed, 5 insertions(+), 2 deletions(-) - ---- a/arch/ia64/include/asm/thread_info.h -+++ b/arch/ia64/include/asm/thread_info.h -@@ -103,6 +103,7 @@ struct thread_info { - #define TIF_SYSCALL_TRACE 2 /* syscall trace active */ - #define TIF_SYSCALL_AUDIT 3 /* syscall auditing active */ - #define TIF_SINGLESTEP 4 /* restore singlestep on return to user mode */ -+#define TIF_NOTIFY_SIGNAL 5 /* signal notification exist */ - #define TIF_NOTIFY_RESUME 6 /* resumption notification requested */ - #define TIF_MEMDIE 17 /* is terminating due to OOM killer */ - #define TIF_MCA_INIT 18 /* this task is processing MCA or INIT */ -@@ -115,6 +116,7 @@ struct thread_info { - #define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) - #define _TIF_SYSCALL_TRACEAUDIT (_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP) - #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) - #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) - #define _TIF_MCA_INIT (1 << TIF_MCA_INIT) -@@ -124,7 +126,7 @@ struct thread_info { - - /* "work to do on user-return" bits */ - #define TIF_ALLWORK_MASK (_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SYSCALL_AUDIT|\ -- _TIF_NEED_RESCHED|_TIF_SYSCALL_TRACE) -+ _TIF_NEED_RESCHED|_TIF_SYSCALL_TRACE|_TIF_NOTIFY_SIGNAL) - /* like TIF_ALLWORK_BITS but sans TIF_SYSCALL_TRACE or TIF_SYSCALL_AUDIT */ - #define TIF_WORK_MASK (TIF_ALLWORK_MASK&~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT)) - ---- a/arch/ia64/kernel/process.c -+++ b/arch/ia64/kernel/process.c -@@ -171,7 +171,8 @@ do_notify_resume_user(sigset_t *unused, - } - - /* deal with pending signal delivery */ -- if (test_thread_flag(TIF_SIGPENDING)) { -+ if (test_thread_flag(TIF_SIGPENDING) || -+ test_thread_flag(TIF_NOTIFY_SIGNAL)) { - local_irq_enable(); /* force interrupt enable */ - ia64_do_signal(scr, in_syscall); - } diff --git a/next-5.10/ia64-don-t-call-handle_signal-unless-there-s-actually-a-signal-queued.patch b/next-5.10/ia64-don-t-call-handle_signal-unless-there-s-actually-a-signal-queued.patch deleted file mode 100644 index c9431cb81d6..00000000000 --- a/next-5.10/ia64-don-t-call-handle_signal-unless-there-s-actually-a-signal-queued.patch +++ /dev/null @@ -1,40 +0,0 @@ -From 621649af8b9db1666b299d6b8c1bd384f01c4e23 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Tue, 2 Mar 2021 17:22:11 -0700 -Subject: ia64: don't call handle_signal() unless there's actually a signal queued - -From: Jens Axboe - -[ Upstream commit f5f4fc4649ae542b1a25670b17aaf3cbb6187acc ] - -Sergei and John both reported that ia64 failed to boot in 5.11, and it -was related to signals. Turns out the ia64 signal handling is a bit odd, -it doesn't check the return value of get_signal() for whether there's a -signal to deliver or not. With the introduction of TIF_NOTIFY_SIGNAL, -then task_work could trigger it. - -Fix it by only calling handle_signal() if we actually have a real signal -to deliver. This brings it in line with all other archs, too. - -Fixes: b269c229b0e8 ("ia64: add support for TIF_NOTIFY_SIGNAL") -Reported-by: Sergei Trofimovich -Reported-by: John Paul Adrian Glaubitz -Tested-by: Sergei Trofimovich -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/ia64/kernel/signal.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - ---- a/arch/ia64/kernel/signal.c -+++ b/arch/ia64/kernel/signal.c -@@ -341,7 +341,8 @@ ia64_do_signal (struct sigscratch *scr, - * need to push through a forced SIGSEGV. - */ - while (1) { -- get_signal(&ksig); -+ if (!get_signal(&ksig)) -+ break; - - /* - * get_signal() may have run a debugger (via notify_parent()) diff --git a/next-5.10/io_uring-import-5.15-stable-io_uring.patch b/next-5.10/io_uring-import-5.15-stable-io_uring.patch deleted file mode 100644 index f8bceb60436..00000000000 --- a/next-5.10/io_uring-import-5.15-stable-io_uring.patch +++ /dev/null @@ -1,24573 +0,0 @@ -From ab51127cad79fdc5ff972e04acb2287c0799efc4 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Thu, 22 Dec 2022 14:30:11 -0700 -Subject: io_uring: import 5.15-stable io_uring - -From: Jens Axboe - -No upstream commit exists. - -This imports the io_uring codebase from 5.15.85, wholesale. Changes -from that code base: - -- Drop IOCB_ALLOC_CACHE, we don't have that in 5.10. -- Drop MKDIRAT/SYMLINKAT/LINKAT. Would require further VFS backports, - and we don't support these in 5.10 to begin with. -- sock_from_file() old style calling convention. -- Use compat_get_bitmap() only for CONFIG_COMPAT=y - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - Makefile | 2 - fs/Makefile | 2 - fs/io-wq.c | 1242 ---- - fs/io-wq.h | 157 - fs/io_uring.c | 9971 ------------------------------------ - include/linux/io_uring.h | 46 - include/linux/sched.h | 3 - include/linux/syscalls.h | 2 - include/trace/events/io_uring.h | 121 - include/uapi/linux/io_uring.h | 115 - io_uring/Makefile | 6 - io_uring/io-wq.c | 1398 +++++ - io_uring/io-wq.h | 160 - io_uring/io_uring.c |10945 ++++++++++++++++++++++++++++++++++++++++ - kernel/exit.c | 2 - kernel/fork.c | 1 - kernel/sched/core.c | 2 - 17 files changed, 12704 insertions(+), 11471 deletions(-) - delete mode 100644 fs/io-wq.c - create mode 100644 io_uring/Makefile - create mode 100644 io_uring/io-wq.c - rename {fs => io_uring}/io-wq.h (81%) - rename {fs => io_uring}/io_uring.c (51%) - ---- a/Makefile -+++ b/Makefile -@@ -1128,7 +1128,7 @@ export MODORDER := $(extmod-prefix)modul - export MODULES_NSDEPS := $(extmod-prefix)modules.nsdeps - - ifeq ($(KBUILD_EXTMOD),) --core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/ block/ -+core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/ block/ io_uring/ - - vmlinux-dirs := $(patsubst %/,%,$(filter %/, \ - $(core-y) $(core-m) $(drivers-y) $(drivers-m) \ ---- a/fs/Makefile -+++ b/fs/Makefile -@@ -32,8 +32,6 @@ obj-$(CONFIG_TIMERFD) += timerfd.o - obj-$(CONFIG_EVENTFD) += eventfd.o - obj-$(CONFIG_USERFAULTFD) += userfaultfd.o - obj-$(CONFIG_AIO) += aio.o --obj-$(CONFIG_IO_URING) += io_uring.o --obj-$(CONFIG_IO_WQ) += io-wq.o - obj-$(CONFIG_FS_DAX) += dax.o - obj-$(CONFIG_FS_ENCRYPTION) += crypto/ - obj-$(CONFIG_FS_VERITY) += verity/ ---- a/fs/io-wq.c -+++ /dev/null -@@ -1,1242 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* -- * Basic worker thread pool for io_uring -- * -- * Copyright (C) 2019 Jens Axboe -- * -- */ --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include "../kernel/sched/sched.h" --#include "io-wq.h" -- --#define WORKER_IDLE_TIMEOUT (5 * HZ) -- --enum { -- IO_WORKER_F_UP = 1, /* up and active */ -- IO_WORKER_F_RUNNING = 2, /* account as running */ -- IO_WORKER_F_FREE = 4, /* worker on free list */ -- IO_WORKER_F_FIXED = 8, /* static idle worker */ -- IO_WORKER_F_BOUND = 16, /* is doing bounded work */ --}; -- --enum { -- IO_WQ_BIT_EXIT = 0, /* wq exiting */ -- IO_WQ_BIT_CANCEL = 1, /* cancel work on list */ -- IO_WQ_BIT_ERROR = 2, /* error on setup */ --}; -- --enum { -- IO_WQE_FLAG_STALLED = 1, /* stalled on hash */ --}; -- --/* -- * One for each thread in a wqe pool -- */ --struct io_worker { -- refcount_t ref; -- unsigned flags; -- struct hlist_nulls_node nulls_node; -- struct list_head all_list; -- struct task_struct *task; -- struct io_wqe *wqe; -- -- struct io_wq_work *cur_work; -- spinlock_t lock; -- -- struct rcu_head rcu; -- struct mm_struct *mm; --#ifdef CONFIG_BLK_CGROUP -- struct cgroup_subsys_state *blkcg_css; --#endif -- const struct cred *cur_creds; -- const struct cred *saved_creds; -- struct files_struct *restore_files; -- struct nsproxy *restore_nsproxy; -- struct fs_struct *restore_fs; --}; -- --#if BITS_PER_LONG == 64 --#define IO_WQ_HASH_ORDER 6 --#else --#define IO_WQ_HASH_ORDER 5 --#endif -- --#define IO_WQ_NR_HASH_BUCKETS (1u << IO_WQ_HASH_ORDER) -- --struct io_wqe_acct { -- unsigned nr_workers; -- unsigned max_workers; -- atomic_t nr_running; --}; -- --enum { -- IO_WQ_ACCT_BOUND, -- IO_WQ_ACCT_UNBOUND, --}; -- --/* -- * Per-node worker thread pool -- */ --struct io_wqe { -- struct { -- raw_spinlock_t lock; -- struct io_wq_work_list work_list; -- unsigned long hash_map; -- unsigned flags; -- } ____cacheline_aligned_in_smp; -- -- int node; -- struct io_wqe_acct acct[2]; -- -- struct hlist_nulls_head free_list; -- struct list_head all_list; -- -- struct io_wq *wq; -- struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS]; --}; -- --/* -- * Per io_wq state -- */ --struct io_wq { -- struct io_wqe **wqes; -- unsigned long state; -- -- free_work_fn *free_work; -- io_wq_work_fn *do_work; -- -- struct task_struct *manager; -- struct user_struct *user; -- refcount_t refs; -- struct completion done; -- -- struct hlist_node cpuhp_node; -- -- refcount_t use_refs; --}; -- --static enum cpuhp_state io_wq_online; -- --static bool io_worker_get(struct io_worker *worker) --{ -- return refcount_inc_not_zero(&worker->ref); --} -- --static void io_worker_release(struct io_worker *worker) --{ -- if (refcount_dec_and_test(&worker->ref)) -- wake_up_process(worker->task); --} -- --/* -- * Note: drops the wqe->lock if returning true! The caller must re-acquire -- * the lock in that case. Some callers need to restart handling if this -- * happens, so we can't just re-acquire the lock on behalf of the caller. -- */ --static bool __io_worker_unuse(struct io_wqe *wqe, struct io_worker *worker) --{ -- bool dropped_lock = false; -- -- if (worker->saved_creds) { -- revert_creds(worker->saved_creds); -- worker->cur_creds = worker->saved_creds = NULL; -- } -- -- if (current->files != worker->restore_files) { -- __acquire(&wqe->lock); -- raw_spin_unlock_irq(&wqe->lock); -- dropped_lock = true; -- -- task_lock(current); -- current->files = worker->restore_files; -- current->nsproxy = worker->restore_nsproxy; -- task_unlock(current); -- } -- -- if (current->fs != worker->restore_fs) -- current->fs = worker->restore_fs; -- -- /* -- * If we have an active mm, we need to drop the wq lock before unusing -- * it. If we do, return true and let the caller retry the idle loop. -- */ -- if (worker->mm) { -- if (!dropped_lock) { -- __acquire(&wqe->lock); -- raw_spin_unlock_irq(&wqe->lock); -- dropped_lock = true; -- } -- __set_current_state(TASK_RUNNING); -- kthread_unuse_mm(worker->mm); -- mmput(worker->mm); -- worker->mm = NULL; -- } -- --#ifdef CONFIG_BLK_CGROUP -- if (worker->blkcg_css) { -- kthread_associate_blkcg(NULL); -- worker->blkcg_css = NULL; -- } --#endif -- if (current->signal->rlim[RLIMIT_FSIZE].rlim_cur != RLIM_INFINITY) -- current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY; -- return dropped_lock; --} -- --static inline struct io_wqe_acct *io_work_get_acct(struct io_wqe *wqe, -- struct io_wq_work *work) --{ -- if (work->flags & IO_WQ_WORK_UNBOUND) -- return &wqe->acct[IO_WQ_ACCT_UNBOUND]; -- -- return &wqe->acct[IO_WQ_ACCT_BOUND]; --} -- --static inline struct io_wqe_acct *io_wqe_get_acct(struct io_wqe *wqe, -- struct io_worker *worker) --{ -- if (worker->flags & IO_WORKER_F_BOUND) -- return &wqe->acct[IO_WQ_ACCT_BOUND]; -- -- return &wqe->acct[IO_WQ_ACCT_UNBOUND]; --} -- --static void io_worker_exit(struct io_worker *worker) --{ -- struct io_wqe *wqe = worker->wqe; -- struct io_wqe_acct *acct = io_wqe_get_acct(wqe, worker); -- -- /* -- * If we're not at zero, someone else is holding a brief reference -- * to the worker. Wait for that to go away. -- */ -- set_current_state(TASK_INTERRUPTIBLE); -- if (!refcount_dec_and_test(&worker->ref)) -- schedule(); -- __set_current_state(TASK_RUNNING); -- -- preempt_disable(); -- current->flags &= ~PF_IO_WORKER; -- if (worker->flags & IO_WORKER_F_RUNNING) -- atomic_dec(&acct->nr_running); -- if (!(worker->flags & IO_WORKER_F_BOUND)) -- atomic_dec(&wqe->wq->user->processes); -- worker->flags = 0; -- preempt_enable(); -- -- raw_spin_lock_irq(&wqe->lock); -- hlist_nulls_del_rcu(&worker->nulls_node); -- list_del_rcu(&worker->all_list); -- if (__io_worker_unuse(wqe, worker)) { -- __release(&wqe->lock); -- raw_spin_lock_irq(&wqe->lock); -- } -- acct->nr_workers--; -- raw_spin_unlock_irq(&wqe->lock); -- -- kfree_rcu(worker, rcu); -- if (refcount_dec_and_test(&wqe->wq->refs)) -- complete(&wqe->wq->done); --} -- --static inline bool io_wqe_run_queue(struct io_wqe *wqe) -- __must_hold(wqe->lock) --{ -- if (!wq_list_empty(&wqe->work_list) && -- !(wqe->flags & IO_WQE_FLAG_STALLED)) -- return true; -- return false; --} -- --/* -- * Check head of free list for an available worker. If one isn't available, -- * caller must wake up the wq manager to create one. -- */ --static bool io_wqe_activate_free_worker(struct io_wqe *wqe) -- __must_hold(RCU) --{ -- struct hlist_nulls_node *n; -- struct io_worker *worker; -- -- n = rcu_dereference(hlist_nulls_first_rcu(&wqe->free_list)); -- if (is_a_nulls(n)) -- return false; -- -- worker = hlist_nulls_entry(n, struct io_worker, nulls_node); -- if (io_worker_get(worker)) { -- wake_up_process(worker->task); -- io_worker_release(worker); -- return true; -- } -- -- return false; --} -- --/* -- * We need a worker. If we find a free one, we're good. If not, and we're -- * below the max number of workers, wake up the manager to create one. -- */ --static void io_wqe_wake_worker(struct io_wqe *wqe, struct io_wqe_acct *acct) --{ -- bool ret; -- -- /* -- * Most likely an attempt to queue unbounded work on an io_wq that -- * wasn't setup with any unbounded workers. -- */ -- if (unlikely(!acct->max_workers)) -- pr_warn_once("io-wq is not configured for unbound workers"); -- -- rcu_read_lock(); -- ret = io_wqe_activate_free_worker(wqe); -- rcu_read_unlock(); -- -- if (!ret && acct->nr_workers < acct->max_workers) -- wake_up_process(wqe->wq->manager); --} -- --static void io_wqe_inc_running(struct io_wqe *wqe, struct io_worker *worker) --{ -- struct io_wqe_acct *acct = io_wqe_get_acct(wqe, worker); -- -- atomic_inc(&acct->nr_running); --} -- --static void io_wqe_dec_running(struct io_wqe *wqe, struct io_worker *worker) -- __must_hold(wqe->lock) --{ -- struct io_wqe_acct *acct = io_wqe_get_acct(wqe, worker); -- -- if (atomic_dec_and_test(&acct->nr_running) && io_wqe_run_queue(wqe)) -- io_wqe_wake_worker(wqe, acct); --} -- --static void io_worker_start(struct io_wqe *wqe, struct io_worker *worker) --{ -- allow_kernel_signal(SIGINT); -- -- current->flags |= PF_IO_WORKER; -- -- worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); -- worker->restore_files = current->files; -- worker->restore_nsproxy = current->nsproxy; -- worker->restore_fs = current->fs; -- io_wqe_inc_running(wqe, worker); --} -- --/* -- * Worker will start processing some work. Move it to the busy list, if -- * it's currently on the freelist -- */ --static void __io_worker_busy(struct io_wqe *wqe, struct io_worker *worker, -- struct io_wq_work *work) -- __must_hold(wqe->lock) --{ -- bool worker_bound, work_bound; -- -- if (worker->flags & IO_WORKER_F_FREE) { -- worker->flags &= ~IO_WORKER_F_FREE; -- hlist_nulls_del_init_rcu(&worker->nulls_node); -- } -- -- /* -- * If worker is moving from bound to unbound (or vice versa), then -- * ensure we update the running accounting. -- */ -- worker_bound = (worker->flags & IO_WORKER_F_BOUND) != 0; -- work_bound = (work->flags & IO_WQ_WORK_UNBOUND) == 0; -- if (worker_bound != work_bound) { -- io_wqe_dec_running(wqe, worker); -- if (work_bound) { -- worker->flags |= IO_WORKER_F_BOUND; -- wqe->acct[IO_WQ_ACCT_UNBOUND].nr_workers--; -- wqe->acct[IO_WQ_ACCT_BOUND].nr_workers++; -- atomic_dec(&wqe->wq->user->processes); -- } else { -- worker->flags &= ~IO_WORKER_F_BOUND; -- wqe->acct[IO_WQ_ACCT_UNBOUND].nr_workers++; -- wqe->acct[IO_WQ_ACCT_BOUND].nr_workers--; -- atomic_inc(&wqe->wq->user->processes); -- } -- io_wqe_inc_running(wqe, worker); -- } --} -- --/* -- * No work, worker going to sleep. Move to freelist, and unuse mm if we -- * have one attached. Dropping the mm may potentially sleep, so we drop -- * the lock in that case and return success. Since the caller has to -- * retry the loop in that case (we changed task state), we don't regrab -- * the lock if we return success. -- */ --static bool __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker) -- __must_hold(wqe->lock) --{ -- if (!(worker->flags & IO_WORKER_F_FREE)) { -- worker->flags |= IO_WORKER_F_FREE; -- hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); -- } -- -- return __io_worker_unuse(wqe, worker); --} -- --static inline unsigned int io_get_work_hash(struct io_wq_work *work) --{ -- return work->flags >> IO_WQ_HASH_SHIFT; --} -- --static struct io_wq_work *io_get_next_work(struct io_wqe *wqe) -- __must_hold(wqe->lock) --{ -- struct io_wq_work_node *node, *prev; -- struct io_wq_work *work, *tail; -- unsigned int hash; -- -- wq_list_for_each(node, prev, &wqe->work_list) { -- work = container_of(node, struct io_wq_work, list); -- -- /* not hashed, can run anytime */ -- if (!io_wq_is_hashed(work)) { -- wq_list_del(&wqe->work_list, node, prev); -- return work; -- } -- -- /* hashed, can run if not already running */ -- hash = io_get_work_hash(work); -- if (!(wqe->hash_map & BIT(hash))) { -- wqe->hash_map |= BIT(hash); -- /* all items with this hash lie in [work, tail] */ -- tail = wqe->hash_tail[hash]; -- wqe->hash_tail[hash] = NULL; -- wq_list_cut(&wqe->work_list, &tail->list, prev); -- return work; -- } -- } -- -- return NULL; --} -- --static void io_wq_switch_mm(struct io_worker *worker, struct io_wq_work *work) --{ -- if (worker->mm) { -- kthread_unuse_mm(worker->mm); -- mmput(worker->mm); -- worker->mm = NULL; -- } -- -- if (mmget_not_zero(work->identity->mm)) { -- kthread_use_mm(work->identity->mm); -- worker->mm = work->identity->mm; -- return; -- } -- -- /* failed grabbing mm, ensure work gets cancelled */ -- work->flags |= IO_WQ_WORK_CANCEL; --} -- --static inline void io_wq_switch_blkcg(struct io_worker *worker, -- struct io_wq_work *work) --{ --#ifdef CONFIG_BLK_CGROUP -- if (!(work->flags & IO_WQ_WORK_BLKCG)) -- return; -- if (work->identity->blkcg_css != worker->blkcg_css) { -- kthread_associate_blkcg(work->identity->blkcg_css); -- worker->blkcg_css = work->identity->blkcg_css; -- } --#endif --} -- --static void io_wq_switch_creds(struct io_worker *worker, -- struct io_wq_work *work) --{ -- const struct cred *old_creds = override_creds(work->identity->creds); -- -- worker->cur_creds = work->identity->creds; -- if (worker->saved_creds) -- put_cred(old_creds); /* creds set by previous switch */ -- else -- worker->saved_creds = old_creds; --} -- --static void io_impersonate_work(struct io_worker *worker, -- struct io_wq_work *work) --{ -- if ((work->flags & IO_WQ_WORK_FILES) && -- current->files != work->identity->files) { -- task_lock(current); -- current->files = work->identity->files; -- current->nsproxy = work->identity->nsproxy; -- task_unlock(current); -- if (!work->identity->files) { -- /* failed grabbing files, ensure work gets cancelled */ -- work->flags |= IO_WQ_WORK_CANCEL; -- } -- } -- if ((work->flags & IO_WQ_WORK_FS) && current->fs != work->identity->fs) -- current->fs = work->identity->fs; -- if ((work->flags & IO_WQ_WORK_MM) && work->identity->mm != worker->mm) -- io_wq_switch_mm(worker, work); -- if ((work->flags & IO_WQ_WORK_CREDS) && -- worker->cur_creds != work->identity->creds) -- io_wq_switch_creds(worker, work); -- if (work->flags & IO_WQ_WORK_FSIZE) -- current->signal->rlim[RLIMIT_FSIZE].rlim_cur = work->identity->fsize; -- else if (current->signal->rlim[RLIMIT_FSIZE].rlim_cur != RLIM_INFINITY) -- current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY; -- io_wq_switch_blkcg(worker, work); --#ifdef CONFIG_AUDIT -- current->loginuid = work->identity->loginuid; -- current->sessionid = work->identity->sessionid; --#endif --} -- --static void io_assign_current_work(struct io_worker *worker, -- struct io_wq_work *work) --{ -- if (work) { -- /* flush pending signals before assigning new work */ -- if (signal_pending(current)) -- flush_signals(current); -- cond_resched(); -- } -- --#ifdef CONFIG_AUDIT -- current->loginuid = KUIDT_INIT(AUDIT_UID_UNSET); -- current->sessionid = AUDIT_SID_UNSET; --#endif -- -- spin_lock_irq(&worker->lock); -- worker->cur_work = work; -- spin_unlock_irq(&worker->lock); --} -- --static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work); -- --static void io_worker_handle_work(struct io_worker *worker) -- __releases(wqe->lock) --{ -- struct io_wqe *wqe = worker->wqe; -- struct io_wq *wq = wqe->wq; -- -- do { -- struct io_wq_work *work; --get_next: -- /* -- * If we got some work, mark us as busy. If we didn't, but -- * the list isn't empty, it means we stalled on hashed work. -- * Mark us stalled so we don't keep looking for work when we -- * can't make progress, any work completion or insertion will -- * clear the stalled flag. -- */ -- work = io_get_next_work(wqe); -- if (work) -- __io_worker_busy(wqe, worker, work); -- else if (!wq_list_empty(&wqe->work_list)) -- wqe->flags |= IO_WQE_FLAG_STALLED; -- -- raw_spin_unlock_irq(&wqe->lock); -- if (!work) -- break; -- io_assign_current_work(worker, work); -- -- /* handle a whole dependent link */ -- do { -- struct io_wq_work *old_work, *next_hashed, *linked; -- unsigned int hash = io_get_work_hash(work); -- -- next_hashed = wq_next_work(work); -- io_impersonate_work(worker, work); -- /* -- * OK to set IO_WQ_WORK_CANCEL even for uncancellable -- * work, the worker function will do the right thing. -- */ -- if (test_bit(IO_WQ_BIT_CANCEL, &wq->state)) -- work->flags |= IO_WQ_WORK_CANCEL; -- -- old_work = work; -- linked = wq->do_work(work); -- -- work = next_hashed; -- if (!work && linked && !io_wq_is_hashed(linked)) { -- work = linked; -- linked = NULL; -- } -- io_assign_current_work(worker, work); -- wq->free_work(old_work); -- -- if (linked) -- io_wqe_enqueue(wqe, linked); -- -- if (hash != -1U && !next_hashed) { -- raw_spin_lock_irq(&wqe->lock); -- wqe->hash_map &= ~BIT_ULL(hash); -- wqe->flags &= ~IO_WQE_FLAG_STALLED; -- /* skip unnecessary unlock-lock wqe->lock */ -- if (!work) -- goto get_next; -- raw_spin_unlock_irq(&wqe->lock); -- } -- } while (work); -- -- raw_spin_lock_irq(&wqe->lock); -- } while (1); --} -- --static int io_wqe_worker(void *data) --{ -- struct io_worker *worker = data; -- struct io_wqe *wqe = worker->wqe; -- struct io_wq *wq = wqe->wq; -- -- io_worker_start(wqe, worker); -- -- while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { -- set_current_state(TASK_INTERRUPTIBLE); --loop: -- raw_spin_lock_irq(&wqe->lock); -- if (io_wqe_run_queue(wqe)) { -- __set_current_state(TASK_RUNNING); -- io_worker_handle_work(worker); -- goto loop; -- } -- /* drops the lock on success, retry */ -- if (__io_worker_idle(wqe, worker)) { -- __release(&wqe->lock); -- goto loop; -- } -- raw_spin_unlock_irq(&wqe->lock); -- if (signal_pending(current)) -- flush_signals(current); -- if (schedule_timeout(WORKER_IDLE_TIMEOUT)) -- continue; -- /* timed out, exit unless we're the fixed worker */ -- if (test_bit(IO_WQ_BIT_EXIT, &wq->state) || -- !(worker->flags & IO_WORKER_F_FIXED)) -- break; -- } -- -- if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) { -- raw_spin_lock_irq(&wqe->lock); -- if (!wq_list_empty(&wqe->work_list)) -- io_worker_handle_work(worker); -- else -- raw_spin_unlock_irq(&wqe->lock); -- } -- -- io_worker_exit(worker); -- return 0; --} -- --/* -- * Called when a worker is scheduled in. Mark us as currently running. -- */ --void io_wq_worker_running(struct task_struct *tsk) --{ -- struct io_worker *worker = kthread_data(tsk); -- struct io_wqe *wqe = worker->wqe; -- -- if (!(worker->flags & IO_WORKER_F_UP)) -- return; -- if (worker->flags & IO_WORKER_F_RUNNING) -- return; -- worker->flags |= IO_WORKER_F_RUNNING; -- io_wqe_inc_running(wqe, worker); --} -- --/* -- * Called when worker is going to sleep. If there are no workers currently -- * running and we have work pending, wake up a free one or have the manager -- * set one up. -- */ --void io_wq_worker_sleeping(struct task_struct *tsk) --{ -- struct io_worker *worker = kthread_data(tsk); -- struct io_wqe *wqe = worker->wqe; -- -- if (!(worker->flags & IO_WORKER_F_UP)) -- return; -- if (!(worker->flags & IO_WORKER_F_RUNNING)) -- return; -- -- worker->flags &= ~IO_WORKER_F_RUNNING; -- -- raw_spin_lock_irq(&wqe->lock); -- io_wqe_dec_running(wqe, worker); -- raw_spin_unlock_irq(&wqe->lock); --} -- --static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) --{ -- struct io_wqe_acct *acct = &wqe->acct[index]; -- struct io_worker *worker; -- -- worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, wqe->node); -- if (!worker) -- return false; -- -- refcount_set(&worker->ref, 1); -- worker->nulls_node.pprev = NULL; -- worker->wqe = wqe; -- spin_lock_init(&worker->lock); -- -- worker->task = kthread_create_on_node(io_wqe_worker, worker, wqe->node, -- "io_wqe_worker-%d/%d", index, wqe->node); -- if (IS_ERR(worker->task)) { -- kfree(worker); -- return false; -- } -- kthread_bind_mask(worker->task, cpumask_of_node(wqe->node)); -- -- raw_spin_lock_irq(&wqe->lock); -- hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); -- list_add_tail_rcu(&worker->all_list, &wqe->all_list); -- worker->flags |= IO_WORKER_F_FREE; -- if (index == IO_WQ_ACCT_BOUND) -- worker->flags |= IO_WORKER_F_BOUND; -- if (!acct->nr_workers && (worker->flags & IO_WORKER_F_BOUND)) -- worker->flags |= IO_WORKER_F_FIXED; -- acct->nr_workers++; -- raw_spin_unlock_irq(&wqe->lock); -- -- if (index == IO_WQ_ACCT_UNBOUND) -- atomic_inc(&wq->user->processes); -- -- refcount_inc(&wq->refs); -- wake_up_process(worker->task); -- return true; --} -- --static inline bool io_wqe_need_worker(struct io_wqe *wqe, int index) -- __must_hold(wqe->lock) --{ -- struct io_wqe_acct *acct = &wqe->acct[index]; -- -- /* if we have available workers or no work, no need */ -- if (!hlist_nulls_empty(&wqe->free_list) || !io_wqe_run_queue(wqe)) -- return false; -- return acct->nr_workers < acct->max_workers; --} -- --static bool io_wqe_worker_send_sig(struct io_worker *worker, void *data) --{ -- send_sig(SIGINT, worker->task, 1); -- return false; --} -- --/* -- * Iterate the passed in list and call the specific function for each -- * worker that isn't exiting -- */ --static bool io_wq_for_each_worker(struct io_wqe *wqe, -- bool (*func)(struct io_worker *, void *), -- void *data) --{ -- struct io_worker *worker; -- bool ret = false; -- -- list_for_each_entry_rcu(worker, &wqe->all_list, all_list) { -- if (io_worker_get(worker)) { -- /* no task if node is/was offline */ -- if (worker->task) -- ret = func(worker, data); -- io_worker_release(worker); -- if (ret) -- break; -- } -- } -- -- return ret; --} -- --static bool io_wq_worker_wake(struct io_worker *worker, void *data) --{ -- wake_up_process(worker->task); -- return false; --} -- --/* -- * Manager thread. Tasked with creating new workers, if we need them. -- */ --static int io_wq_manager(void *data) --{ -- struct io_wq *wq = data; -- int node; -- -- /* create fixed workers */ -- refcount_set(&wq->refs, 1); -- for_each_node(node) { -- if (!node_online(node)) -- continue; -- if (create_io_worker(wq, wq->wqes[node], IO_WQ_ACCT_BOUND)) -- continue; -- set_bit(IO_WQ_BIT_ERROR, &wq->state); -- set_bit(IO_WQ_BIT_EXIT, &wq->state); -- goto out; -- } -- -- complete(&wq->done); -- -- while (!kthread_should_stop()) { -- if (current->task_works) -- task_work_run(); -- -- for_each_node(node) { -- struct io_wqe *wqe = wq->wqes[node]; -- bool fork_worker[2] = { false, false }; -- -- if (!node_online(node)) -- continue; -- -- raw_spin_lock_irq(&wqe->lock); -- if (io_wqe_need_worker(wqe, IO_WQ_ACCT_BOUND)) -- fork_worker[IO_WQ_ACCT_BOUND] = true; -- if (io_wqe_need_worker(wqe, IO_WQ_ACCT_UNBOUND)) -- fork_worker[IO_WQ_ACCT_UNBOUND] = true; -- raw_spin_unlock_irq(&wqe->lock); -- if (fork_worker[IO_WQ_ACCT_BOUND]) -- create_io_worker(wq, wqe, IO_WQ_ACCT_BOUND); -- if (fork_worker[IO_WQ_ACCT_UNBOUND]) -- create_io_worker(wq, wqe, IO_WQ_ACCT_UNBOUND); -- } -- set_current_state(TASK_INTERRUPTIBLE); -- schedule_timeout(HZ); -- } -- -- if (current->task_works) -- task_work_run(); -- --out: -- if (refcount_dec_and_test(&wq->refs)) { -- complete(&wq->done); -- return 0; -- } -- /* if ERROR is set and we get here, we have workers to wake */ -- if (test_bit(IO_WQ_BIT_ERROR, &wq->state)) { -- rcu_read_lock(); -- for_each_node(node) -- io_wq_for_each_worker(wq->wqes[node], io_wq_worker_wake, NULL); -- rcu_read_unlock(); -- } -- return 0; --} -- --static bool io_wq_can_queue(struct io_wqe *wqe, struct io_wqe_acct *acct, -- struct io_wq_work *work) --{ -- bool free_worker; -- -- if (!(work->flags & IO_WQ_WORK_UNBOUND)) -- return true; -- if (atomic_read(&acct->nr_running)) -- return true; -- -- rcu_read_lock(); -- free_worker = !hlist_nulls_empty(&wqe->free_list); -- rcu_read_unlock(); -- if (free_worker) -- return true; -- -- if (atomic_read(&wqe->wq->user->processes) >= acct->max_workers && -- !(capable(CAP_SYS_RESOURCE) || capable(CAP_SYS_ADMIN))) -- return false; -- -- return true; --} -- --static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe) --{ -- struct io_wq *wq = wqe->wq; -- -- do { -- struct io_wq_work *old_work = work; -- -- work->flags |= IO_WQ_WORK_CANCEL; -- work = wq->do_work(work); -- wq->free_work(old_work); -- } while (work); --} -- --static void io_wqe_insert_work(struct io_wqe *wqe, struct io_wq_work *work) --{ -- unsigned int hash; -- struct io_wq_work *tail; -- -- if (!io_wq_is_hashed(work)) { --append: -- wq_list_add_tail(&work->list, &wqe->work_list); -- return; -- } -- -- hash = io_get_work_hash(work); -- tail = wqe->hash_tail[hash]; -- wqe->hash_tail[hash] = work; -- if (!tail) -- goto append; -- -- wq_list_add_after(&work->list, &tail->list, &wqe->work_list); --} -- --static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) --{ -- struct io_wqe_acct *acct = io_work_get_acct(wqe, work); -- bool do_wake; -- unsigned long flags; -- -- /* -- * Do early check to see if we need a new unbound worker, and if we do, -- * if we're allowed to do so. This isn't 100% accurate as there's a -- * gap between this check and incrementing the value, but that's OK. -- * It's close enough to not be an issue, fork() has the same delay. -- */ -- if (unlikely(!io_wq_can_queue(wqe, acct, work))) { -- io_run_cancel(work, wqe); -- return; -- } -- -- raw_spin_lock_irqsave(&wqe->lock, flags); -- io_wqe_insert_work(wqe, work); -- wqe->flags &= ~IO_WQE_FLAG_STALLED; -- do_wake = (work->flags & IO_WQ_WORK_CONCURRENT) || -- !atomic_read(&acct->nr_running); -- raw_spin_unlock_irqrestore(&wqe->lock, flags); -- -- if (do_wake) -- io_wqe_wake_worker(wqe, acct); --} -- --void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work) --{ -- struct io_wqe *wqe = wq->wqes[numa_node_id()]; -- -- io_wqe_enqueue(wqe, work); --} -- --/* -- * Work items that hash to the same value will not be done in parallel. -- * Used to limit concurrent writes, generally hashed by inode. -- */ --void io_wq_hash_work(struct io_wq_work *work, void *val) --{ -- unsigned int bit; -- -- bit = hash_ptr(val, IO_WQ_HASH_ORDER); -- work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT)); --} -- --void io_wq_cancel_all(struct io_wq *wq) --{ -- int node; -- -- set_bit(IO_WQ_BIT_CANCEL, &wq->state); -- -- rcu_read_lock(); -- for_each_node(node) { -- struct io_wqe *wqe = wq->wqes[node]; -- -- io_wq_for_each_worker(wqe, io_wqe_worker_send_sig, NULL); -- } -- rcu_read_unlock(); --} -- --struct io_cb_cancel_data { -- work_cancel_fn *fn; -- void *data; -- int nr_running; -- int nr_pending; -- bool cancel_all; --}; -- --static bool io_wq_worker_cancel(struct io_worker *worker, void *data) --{ -- struct io_cb_cancel_data *match = data; -- unsigned long flags; -- -- /* -- * Hold the lock to avoid ->cur_work going out of scope, caller -- * may dereference the passed in work. -- */ -- spin_lock_irqsave(&worker->lock, flags); -- if (worker->cur_work && -- !(worker->cur_work->flags & IO_WQ_WORK_NO_CANCEL) && -- match->fn(worker->cur_work, match->data)) { -- send_sig(SIGINT, worker->task, 1); -- match->nr_running++; -- } -- spin_unlock_irqrestore(&worker->lock, flags); -- -- return match->nr_running && !match->cancel_all; --} -- --static inline void io_wqe_remove_pending(struct io_wqe *wqe, -- struct io_wq_work *work, -- struct io_wq_work_node *prev) --{ -- unsigned int hash = io_get_work_hash(work); -- struct io_wq_work *prev_work = NULL; -- -- if (io_wq_is_hashed(work) && work == wqe->hash_tail[hash]) { -- if (prev) -- prev_work = container_of(prev, struct io_wq_work, list); -- if (prev_work && io_get_work_hash(prev_work) == hash) -- wqe->hash_tail[hash] = prev_work; -- else -- wqe->hash_tail[hash] = NULL; -- } -- wq_list_del(&wqe->work_list, &work->list, prev); --} -- --static void io_wqe_cancel_pending_work(struct io_wqe *wqe, -- struct io_cb_cancel_data *match) --{ -- struct io_wq_work_node *node, *prev; -- struct io_wq_work *work; -- unsigned long flags; -- --retry: -- raw_spin_lock_irqsave(&wqe->lock, flags); -- wq_list_for_each(node, prev, &wqe->work_list) { -- work = container_of(node, struct io_wq_work, list); -- if (!match->fn(work, match->data)) -- continue; -- io_wqe_remove_pending(wqe, work, prev); -- raw_spin_unlock_irqrestore(&wqe->lock, flags); -- io_run_cancel(work, wqe); -- match->nr_pending++; -- if (!match->cancel_all) -- return; -- -- /* not safe to continue after unlock */ -- goto retry; -- } -- raw_spin_unlock_irqrestore(&wqe->lock, flags); --} -- --static void io_wqe_cancel_running_work(struct io_wqe *wqe, -- struct io_cb_cancel_data *match) --{ -- rcu_read_lock(); -- io_wq_for_each_worker(wqe, io_wq_worker_cancel, match); -- rcu_read_unlock(); --} -- --enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, -- void *data, bool cancel_all) --{ -- struct io_cb_cancel_data match = { -- .fn = cancel, -- .data = data, -- .cancel_all = cancel_all, -- }; -- int node; -- -- /* -- * First check pending list, if we're lucky we can just remove it -- * from there. CANCEL_OK means that the work is returned as-new, -- * no completion will be posted for it. -- */ -- for_each_node(node) { -- struct io_wqe *wqe = wq->wqes[node]; -- -- io_wqe_cancel_pending_work(wqe, &match); -- if (match.nr_pending && !match.cancel_all) -- return IO_WQ_CANCEL_OK; -- } -- -- /* -- * Now check if a free (going busy) or busy worker has the work -- * currently running. If we find it there, we'll return CANCEL_RUNNING -- * as an indication that we attempt to signal cancellation. The -- * completion will run normally in this case. -- */ -- for_each_node(node) { -- struct io_wqe *wqe = wq->wqes[node]; -- -- io_wqe_cancel_running_work(wqe, &match); -- if (match.nr_running && !match.cancel_all) -- return IO_WQ_CANCEL_RUNNING; -- } -- -- if (match.nr_running) -- return IO_WQ_CANCEL_RUNNING; -- if (match.nr_pending) -- return IO_WQ_CANCEL_OK; -- return IO_WQ_CANCEL_NOTFOUND; --} -- --struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) --{ -- int ret = -ENOMEM, node; -- struct io_wq *wq; -- -- if (WARN_ON_ONCE(!data->free_work || !data->do_work)) -- return ERR_PTR(-EINVAL); -- if (WARN_ON_ONCE(!bounded)) -- return ERR_PTR(-EINVAL); -- -- wq = kzalloc(sizeof(*wq), GFP_KERNEL); -- if (!wq) -- return ERR_PTR(-ENOMEM); -- -- wq->wqes = kcalloc(nr_node_ids, sizeof(struct io_wqe *), GFP_KERNEL); -- if (!wq->wqes) -- goto err_wq; -- -- ret = cpuhp_state_add_instance_nocalls(io_wq_online, &wq->cpuhp_node); -- if (ret) -- goto err_wqes; -- -- wq->free_work = data->free_work; -- wq->do_work = data->do_work; -- -- /* caller must already hold a reference to this */ -- wq->user = data->user; -- -- ret = -ENOMEM; -- for_each_node(node) { -- struct io_wqe *wqe; -- int alloc_node = node; -- -- if (!node_online(alloc_node)) -- alloc_node = NUMA_NO_NODE; -- wqe = kzalloc_node(sizeof(struct io_wqe), GFP_KERNEL, alloc_node); -- if (!wqe) -- goto err; -- wq->wqes[node] = wqe; -- wqe->node = alloc_node; -- wqe->acct[IO_WQ_ACCT_BOUND].max_workers = bounded; -- atomic_set(&wqe->acct[IO_WQ_ACCT_BOUND].nr_running, 0); -- if (wq->user) { -- wqe->acct[IO_WQ_ACCT_UNBOUND].max_workers = -- task_rlimit(current, RLIMIT_NPROC); -- } -- atomic_set(&wqe->acct[IO_WQ_ACCT_UNBOUND].nr_running, 0); -- wqe->wq = wq; -- raw_spin_lock_init(&wqe->lock); -- INIT_WQ_LIST(&wqe->work_list); -- INIT_HLIST_NULLS_HEAD(&wqe->free_list, 0); -- INIT_LIST_HEAD(&wqe->all_list); -- } -- -- init_completion(&wq->done); -- -- wq->manager = kthread_create(io_wq_manager, wq, "io_wq_manager"); -- if (!IS_ERR(wq->manager)) { -- wake_up_process(wq->manager); -- wait_for_completion(&wq->done); -- if (test_bit(IO_WQ_BIT_ERROR, &wq->state)) { -- ret = -ENOMEM; -- goto err; -- } -- refcount_set(&wq->use_refs, 1); -- reinit_completion(&wq->done); -- return wq; -- } -- -- ret = PTR_ERR(wq->manager); -- complete(&wq->done); --err: -- cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); -- for_each_node(node) -- kfree(wq->wqes[node]); --err_wqes: -- kfree(wq->wqes); --err_wq: -- kfree(wq); -- return ERR_PTR(ret); --} -- --bool io_wq_get(struct io_wq *wq, struct io_wq_data *data) --{ -- if (data->free_work != wq->free_work || data->do_work != wq->do_work) -- return false; -- -- return refcount_inc_not_zero(&wq->use_refs); --} -- --static void __io_wq_destroy(struct io_wq *wq) --{ -- int node; -- -- cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); -- -- set_bit(IO_WQ_BIT_EXIT, &wq->state); -- if (wq->manager) -- kthread_stop(wq->manager); -- -- rcu_read_lock(); -- for_each_node(node) -- io_wq_for_each_worker(wq->wqes[node], io_wq_worker_wake, NULL); -- rcu_read_unlock(); -- -- wait_for_completion(&wq->done); -- -- for_each_node(node) -- kfree(wq->wqes[node]); -- kfree(wq->wqes); -- kfree(wq); --} -- --void io_wq_destroy(struct io_wq *wq) --{ -- if (refcount_dec_and_test(&wq->use_refs)) -- __io_wq_destroy(wq); --} -- --struct task_struct *io_wq_get_task(struct io_wq *wq) --{ -- return wq->manager; --} -- --static bool io_wq_worker_affinity(struct io_worker *worker, void *data) --{ -- struct task_struct *task = worker->task; -- struct rq_flags rf; -- struct rq *rq; -- -- rq = task_rq_lock(task, &rf); -- do_set_cpus_allowed(task, cpumask_of_node(worker->wqe->node)); -- task->flags |= PF_NO_SETAFFINITY; -- task_rq_unlock(rq, task, &rf); -- return false; --} -- --static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node) --{ -- struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); -- int i; -- -- rcu_read_lock(); -- for_each_node(i) -- io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, NULL); -- rcu_read_unlock(); -- return 0; --} -- --static __init int io_wq_init(void) --{ -- int ret; -- -- ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online", -- io_wq_cpu_online, NULL); -- if (ret < 0) -- return ret; -- io_wq_online = ret; -- return 0; --} --subsys_initcall(io_wq_init); ---- a/fs/io-wq.h -+++ /dev/null -@@ -1,157 +0,0 @@ --#ifndef INTERNAL_IO_WQ_H --#define INTERNAL_IO_WQ_H -- --#include -- --struct io_wq; -- --enum { -- IO_WQ_WORK_CANCEL = 1, -- IO_WQ_WORK_HASHED = 2, -- IO_WQ_WORK_UNBOUND = 4, -- IO_WQ_WORK_NO_CANCEL = 8, -- IO_WQ_WORK_CONCURRENT = 16, -- -- IO_WQ_WORK_FILES = 32, -- IO_WQ_WORK_FS = 64, -- IO_WQ_WORK_MM = 128, -- IO_WQ_WORK_CREDS = 256, -- IO_WQ_WORK_BLKCG = 512, -- IO_WQ_WORK_FSIZE = 1024, -- -- IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */ --}; -- --enum io_wq_cancel { -- IO_WQ_CANCEL_OK, /* cancelled before started */ -- IO_WQ_CANCEL_RUNNING, /* found, running, and attempted cancelled */ -- IO_WQ_CANCEL_NOTFOUND, /* work not found */ --}; -- --struct io_wq_work_node { -- struct io_wq_work_node *next; --}; -- --struct io_wq_work_list { -- struct io_wq_work_node *first; -- struct io_wq_work_node *last; --}; -- --static inline void wq_list_add_after(struct io_wq_work_node *node, -- struct io_wq_work_node *pos, -- struct io_wq_work_list *list) --{ -- struct io_wq_work_node *next = pos->next; -- -- pos->next = node; -- node->next = next; -- if (!next) -- list->last = node; --} -- --static inline void wq_list_add_tail(struct io_wq_work_node *node, -- struct io_wq_work_list *list) --{ -- if (!list->first) { -- list->last = node; -- WRITE_ONCE(list->first, node); -- } else { -- list->last->next = node; -- list->last = node; -- } -- node->next = NULL; --} -- --static inline void wq_list_cut(struct io_wq_work_list *list, -- struct io_wq_work_node *last, -- struct io_wq_work_node *prev) --{ -- /* first in the list, if prev==NULL */ -- if (!prev) -- WRITE_ONCE(list->first, last->next); -- else -- prev->next = last->next; -- -- if (last == list->last) -- list->last = prev; -- last->next = NULL; --} -- --static inline void wq_list_del(struct io_wq_work_list *list, -- struct io_wq_work_node *node, -- struct io_wq_work_node *prev) --{ -- wq_list_cut(list, node, prev); --} -- --#define wq_list_for_each(pos, prv, head) \ -- for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next) -- --#define wq_list_empty(list) (READ_ONCE((list)->first) == NULL) --#define INIT_WQ_LIST(list) do { \ -- (list)->first = NULL; \ -- (list)->last = NULL; \ --} while (0) -- --struct io_wq_work { -- struct io_wq_work_node list; -- struct io_identity *identity; -- unsigned flags; --}; -- --static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) --{ -- if (!work->list.next) -- return NULL; -- -- return container_of(work->list.next, struct io_wq_work, list); --} -- --typedef void (free_work_fn)(struct io_wq_work *); --typedef struct io_wq_work *(io_wq_work_fn)(struct io_wq_work *); -- --struct io_wq_data { -- struct user_struct *user; -- -- io_wq_work_fn *do_work; -- free_work_fn *free_work; --}; -- --struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); --bool io_wq_get(struct io_wq *wq, struct io_wq_data *data); --void io_wq_destroy(struct io_wq *wq); -- --void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); --void io_wq_hash_work(struct io_wq_work *work, void *val); -- --static inline bool io_wq_is_hashed(struct io_wq_work *work) --{ -- return work->flags & IO_WQ_WORK_HASHED; --} -- --void io_wq_cancel_all(struct io_wq *wq); -- --typedef bool (work_cancel_fn)(struct io_wq_work *, void *); -- --enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, -- void *data, bool cancel_all); -- --struct task_struct *io_wq_get_task(struct io_wq *wq); -- --#if defined(CONFIG_IO_WQ) --extern void io_wq_worker_sleeping(struct task_struct *); --extern void io_wq_worker_running(struct task_struct *); --#else --static inline void io_wq_worker_sleeping(struct task_struct *tsk) --{ --} --static inline void io_wq_worker_running(struct task_struct *tsk) --{ --} --#endif -- --static inline bool io_wq_current_is_worker(void) --{ -- return in_task() && (current->flags & PF_IO_WORKER); --} --#endif ---- a/fs/io_uring.c -+++ /dev/null -@@ -1,9971 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* -- * Shared application/kernel submission and completion ring pairs, for -- * supporting fast/efficient IO. -- * -- * A note on the read/write ordering memory barriers that are matched between -- * the application and kernel side. -- * -- * After the application reads the CQ ring tail, it must use an -- * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses -- * before writing the tail (using smp_load_acquire to read the tail will -- * do). It also needs a smp_mb() before updating CQ head (ordering the -- * entry load(s) with the head store), pairing with an implicit barrier -- * through a control-dependency in io_get_cqring (smp_store_release to -- * store head will do). Failure to do so could lead to reading invalid -- * CQ entries. -- * -- * Likewise, the application must use an appropriate smp_wmb() before -- * writing the SQ tail (ordering SQ entry stores with the tail store), -- * which pairs with smp_load_acquire in io_get_sqring (smp_store_release -- * to store the tail will do). And it needs a barrier ordering the SQ -- * head load before writing new SQ entries (smp_load_acquire to read -- * head will do). -- * -- * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application -- * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after* -- * updating the SQ tail; a full memory barrier smp_mb() is needed -- * between. -- * -- * Also see the examples in the liburing library: -- * -- * git://git.kernel.dk/liburing -- * -- * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens -- * from data shared between the kernel and application. This is done both -- * for ordering purposes, but also to ensure that once a value is loaded from -- * data that the application could potentially modify, it remains stable. -- * -- * Copyright (C) 2018-2019 Jens Axboe -- * Copyright (c) 2018-2019 Christoph Hellwig -- */ --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#define CREATE_TRACE_POINTS --#include -- --#include -- --#include "internal.h" --#include "io-wq.h" -- --#define IORING_MAX_ENTRIES 32768 --#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) -- --/* -- * Shift of 9 is 512 entries, or exactly one page on 64-bit archs -- */ --#define IORING_FILE_TABLE_SHIFT 9 --#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT) --#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1) --#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE) --#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ -- IORING_REGISTER_LAST + IORING_OP_LAST) -- --struct io_uring { -- u32 head ____cacheline_aligned_in_smp; -- u32 tail ____cacheline_aligned_in_smp; --}; -- --/* -- * This data is shared with the application through the mmap at offsets -- * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING. -- * -- * The offsets to the member fields are published through struct -- * io_sqring_offsets when calling io_uring_setup. -- */ --struct io_rings { -- /* -- * Head and tail offsets into the ring; the offsets need to be -- * masked to get valid indices. -- * -- * The kernel controls head of the sq ring and the tail of the cq ring, -- * and the application controls tail of the sq ring and the head of the -- * cq ring. -- */ -- struct io_uring sq, cq; -- /* -- * Bitmasks to apply to head and tail offsets (constant, equals -- * ring_entries - 1) -- */ -- u32 sq_ring_mask, cq_ring_mask; -- /* Ring sizes (constant, power of 2) */ -- u32 sq_ring_entries, cq_ring_entries; -- /* -- * Number of invalid entries dropped by the kernel due to -- * invalid index stored in array -- * -- * Written by the kernel, shouldn't be modified by the -- * application (i.e. get number of "new events" by comparing to -- * cached value). -- * -- * After a new SQ head value was read by the application this -- * counter includes all submissions that were dropped reaching -- * the new SQ head (and possibly more). -- */ -- u32 sq_dropped; -- /* -- * Runtime SQ flags -- * -- * Written by the kernel, shouldn't be modified by the -- * application. -- * -- * The application needs a full memory barrier before checking -- * for IORING_SQ_NEED_WAKEUP after updating the sq tail. -- */ -- u32 sq_flags; -- /* -- * Runtime CQ flags -- * -- * Written by the application, shouldn't be modified by the -- * kernel. -- */ -- u32 cq_flags; -- /* -- * Number of completion events lost because the queue was full; -- * this should be avoided by the application by making sure -- * there are not more requests pending than there is space in -- * the completion queue. -- * -- * Written by the kernel, shouldn't be modified by the -- * application (i.e. get number of "new events" by comparing to -- * cached value). -- * -- * As completion events come in out of order this counter is not -- * ordered with any other data. -- */ -- u32 cq_overflow; -- /* -- * Ring buffer of completion events. -- * -- * The kernel writes completion events fresh every time they are -- * produced, so the application is allowed to modify pending -- * entries. -- */ -- struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp; --}; -- --struct io_mapped_ubuf { -- u64 ubuf; -- size_t len; -- struct bio_vec *bvec; -- unsigned int nr_bvecs; -- unsigned long acct_pages; --}; -- --struct fixed_file_table { -- struct file **files; --}; -- --struct fixed_file_ref_node { -- struct percpu_ref refs; -- struct list_head node; -- struct list_head file_list; -- struct fixed_file_data *file_data; -- struct llist_node llist; -- bool done; --}; -- --struct fixed_file_data { -- struct fixed_file_table *table; -- struct io_ring_ctx *ctx; -- -- struct fixed_file_ref_node *node; -- struct percpu_ref refs; -- struct completion done; -- struct list_head ref_list; -- spinlock_t lock; -- bool quiesce; --}; -- --struct io_buffer { -- struct list_head list; -- __u64 addr; -- __u32 len; -- __u16 bid; --}; -- --struct io_restriction { -- DECLARE_BITMAP(register_op, IORING_REGISTER_LAST); -- DECLARE_BITMAP(sqe_op, IORING_OP_LAST); -- u8 sqe_flags_allowed; -- u8 sqe_flags_required; -- bool registered; --}; -- --struct io_sq_data { -- refcount_t refs; -- struct mutex lock; -- -- /* ctx's that are using this sqd */ -- struct list_head ctx_list; -- struct list_head ctx_new_list; -- struct mutex ctx_lock; -- -- struct task_struct *thread; -- struct wait_queue_head wait; --}; -- --struct io_ring_ctx { -- struct { -- struct percpu_ref refs; -- } ____cacheline_aligned_in_smp; -- -- struct { -- unsigned int flags; -- unsigned int compat: 1; -- unsigned int limit_mem: 1; -- unsigned int cq_overflow_flushed: 1; -- unsigned int drain_next: 1; -- unsigned int eventfd_async: 1; -- unsigned int restricted: 1; -- unsigned int sqo_dead: 1; -- -- /* -- * Ring buffer of indices into array of io_uring_sqe, which is -- * mmapped by the application using the IORING_OFF_SQES offset. -- * -- * This indirection could e.g. be used to assign fixed -- * io_uring_sqe entries to operations and only submit them to -- * the queue when needed. -- * -- * The kernel modifies neither the indices array nor the entries -- * array. -- */ -- u32 *sq_array; -- unsigned cached_sq_head; -- unsigned sq_entries; -- unsigned sq_mask; -- unsigned sq_thread_idle; -- unsigned cached_sq_dropped; -- unsigned cached_cq_overflow; -- unsigned long sq_check_overflow; -- -- struct list_head defer_list; -- struct list_head timeout_list; -- struct list_head cq_overflow_list; -- -- struct io_uring_sqe *sq_sqes; -- } ____cacheline_aligned_in_smp; -- -- struct io_rings *rings; -- -- /* IO offload */ -- struct io_wq *io_wq; -- -- /* -- * For SQPOLL usage - we hold a reference to the parent task, so we -- * have access to the ->files -- */ -- struct task_struct *sqo_task; -- -- /* Only used for accounting purposes */ -- struct mm_struct *mm_account; -- --#ifdef CONFIG_BLK_CGROUP -- struct cgroup_subsys_state *sqo_blkcg_css; --#endif -- -- struct io_sq_data *sq_data; /* if using sq thread polling */ -- -- struct wait_queue_head sqo_sq_wait; -- struct wait_queue_entry sqo_wait_entry; -- struct list_head sqd_list; -- -- /* -- * If used, fixed file set. Writers must ensure that ->refs is dead, -- * readers must ensure that ->refs is alive as long as the file* is -- * used. Only updated through io_uring_register(2). -- */ -- struct fixed_file_data *file_data; -- unsigned nr_user_files; -- -- /* if used, fixed mapped user buffers */ -- unsigned nr_user_bufs; -- struct io_mapped_ubuf *user_bufs; -- -- struct user_struct *user; -- -- const struct cred *creds; -- --#ifdef CONFIG_AUDIT -- kuid_t loginuid; -- unsigned int sessionid; --#endif -- -- struct completion ref_comp; -- struct completion sq_thread_comp; -- -- /* if all else fails... */ -- struct io_kiocb *fallback_req; -- --#if defined(CONFIG_UNIX) -- struct socket *ring_sock; --#endif -- -- struct xarray io_buffers; -- -- struct xarray personalities; -- u32 pers_next; -- -- struct { -- unsigned cached_cq_tail; -- unsigned cq_entries; -- unsigned cq_mask; -- atomic_t cq_timeouts; -- unsigned cq_last_tm_flush; -- unsigned long cq_check_overflow; -- struct wait_queue_head cq_wait; -- struct fasync_struct *cq_fasync; -- struct eventfd_ctx *cq_ev_fd; -- } ____cacheline_aligned_in_smp; -- -- struct { -- struct mutex uring_lock; -- wait_queue_head_t wait; -- } ____cacheline_aligned_in_smp; -- -- struct { -- spinlock_t completion_lock; -- -- /* -- * ->iopoll_list is protected by the ctx->uring_lock for -- * io_uring instances that don't use IORING_SETUP_SQPOLL. -- * For SQPOLL, only the single threaded io_sq_thread() will -- * manipulate the list, hence no extra locking is needed there. -- */ -- struct list_head iopoll_list; -- struct hlist_head *cancel_hash; -- unsigned cancel_hash_bits; -- bool poll_multi_file; -- -- spinlock_t inflight_lock; -- struct list_head inflight_list; -- } ____cacheline_aligned_in_smp; -- -- struct delayed_work file_put_work; -- struct llist_head file_put_llist; -- -- struct work_struct exit_work; -- struct io_restriction restrictions; --}; -- --/* -- * First field must be the file pointer in all the -- * iocb unions! See also 'struct kiocb' in -- */ --struct io_poll_iocb { -- struct file *file; -- union { -- struct wait_queue_head *head; -- u64 addr; -- }; -- __poll_t events; -- bool done; -- bool canceled; -- struct wait_queue_entry wait; --}; -- --struct io_close { -- struct file *file; -- struct file *put_file; -- int fd; --}; -- --struct io_timeout_data { -- struct io_kiocb *req; -- struct hrtimer timer; -- struct timespec64 ts; -- enum hrtimer_mode mode; --}; -- --struct io_accept { -- struct file *file; -- struct sockaddr __user *addr; -- int __user *addr_len; -- int flags; -- unsigned long nofile; --}; -- --struct io_sync { -- struct file *file; -- loff_t len; -- loff_t off; -- int flags; -- int mode; --}; -- --struct io_cancel { -- struct file *file; -- u64 addr; --}; -- --struct io_timeout { -- struct file *file; -- u32 off; -- u32 target_seq; -- struct list_head list; --}; -- --struct io_timeout_rem { -- struct file *file; -- u64 addr; --}; -- --struct io_rw { -- /* NOTE: kiocb has the file as the first member, so don't do it here */ -- struct kiocb kiocb; -- u64 addr; -- u64 len; --}; -- --struct io_connect { -- struct file *file; -- struct sockaddr __user *addr; -- int addr_len; --}; -- --struct io_sr_msg { -- struct file *file; -- union { -- struct user_msghdr __user *umsg; -- void __user *buf; -- }; -- int msg_flags; -- int bgid; -- size_t len; -- struct io_buffer *kbuf; --}; -- --struct io_open { -- struct file *file; -- int dfd; -- bool ignore_nonblock; -- struct filename *filename; -- struct open_how how; -- unsigned long nofile; --}; -- --struct io_files_update { -- struct file *file; -- u64 arg; -- u32 nr_args; -- u32 offset; --}; -- --struct io_fadvise { -- struct file *file; -- u64 offset; -- u32 len; -- u32 advice; --}; -- --struct io_madvise { -- struct file *file; -- u64 addr; -- u32 len; -- u32 advice; --}; -- --struct io_epoll { -- struct file *file; -- int epfd; -- int op; -- int fd; -- struct epoll_event event; --}; -- --struct io_splice { -- struct file *file_out; -- struct file *file_in; -- loff_t off_out; -- loff_t off_in; -- u64 len; -- unsigned int flags; --}; -- --struct io_provide_buf { -- struct file *file; -- __u64 addr; -- __u32 len; -- __u32 bgid; -- __u16 nbufs; -- __u16 bid; --}; -- --struct io_statx { -- struct file *file; -- int dfd; -- unsigned int mask; -- unsigned int flags; -- const char __user *filename; -- struct statx __user *buffer; --}; -- --struct io_completion { -- struct file *file; -- struct list_head list; -- u32 cflags; --}; -- --struct io_async_connect { -- struct sockaddr_storage address; --}; -- --struct io_async_msghdr { -- struct iovec fast_iov[UIO_FASTIOV]; -- struct iovec *iov; -- struct sockaddr __user *uaddr; -- struct msghdr msg; -- struct sockaddr_storage addr; --}; -- --struct io_async_rw { -- struct iovec fast_iov[UIO_FASTIOV]; -- const struct iovec *free_iovec; -- struct iov_iter iter; -- size_t bytes_done; -- struct wait_page_queue wpq; --}; -- --enum { -- REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT, -- REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT, -- REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT, -- REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT, -- REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT, -- REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT, -- -- REQ_F_LINK_HEAD_BIT, -- REQ_F_FAIL_LINK_BIT, -- REQ_F_INFLIGHT_BIT, -- REQ_F_CUR_POS_BIT, -- REQ_F_NOWAIT_BIT, -- REQ_F_LINK_TIMEOUT_BIT, -- REQ_F_ISREG_BIT, -- REQ_F_NEED_CLEANUP_BIT, -- REQ_F_POLLED_BIT, -- REQ_F_BUFFER_SELECTED_BIT, -- REQ_F_NO_FILE_TABLE_BIT, -- REQ_F_WORK_INITIALIZED_BIT, -- REQ_F_LTIMEOUT_ACTIVE_BIT, -- -- /* not a real bit, just to check we're not overflowing the space */ -- __REQ_F_LAST_BIT, --}; -- --enum { -- /* ctx owns file */ -- REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT), -- /* drain existing IO first */ -- REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT), -- /* linked sqes */ -- REQ_F_LINK = BIT(REQ_F_LINK_BIT), -- /* doesn't sever on completion < 0 */ -- REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT), -- /* IOSQE_ASYNC */ -- REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT), -- /* IOSQE_BUFFER_SELECT */ -- REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT), -- -- /* head of a link */ -- REQ_F_LINK_HEAD = BIT(REQ_F_LINK_HEAD_BIT), -- /* fail rest of links */ -- REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT), -- /* on inflight list */ -- REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT), -- /* read/write uses file position */ -- REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT), -- /* must not punt to workers */ -- REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT), -- /* has or had linked timeout */ -- REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT), -- /* regular file */ -- REQ_F_ISREG = BIT(REQ_F_ISREG_BIT), -- /* needs cleanup */ -- REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT), -- /* already went through poll handler */ -- REQ_F_POLLED = BIT(REQ_F_POLLED_BIT), -- /* buffer already selected */ -- REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT), -- /* doesn't need file table for this request */ -- REQ_F_NO_FILE_TABLE = BIT(REQ_F_NO_FILE_TABLE_BIT), -- /* io_wq_work is initialized */ -- REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT), -- /* linked timeout is active, i.e. prepared by link's head */ -- REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT), --}; -- --struct async_poll { -- struct io_poll_iocb poll; -- struct io_poll_iocb *double_poll; --}; -- --/* -- * NOTE! Each of the iocb union members has the file pointer -- * as the first entry in their struct definition. So you can -- * access the file pointer through any of the sub-structs, -- * or directly as just 'ki_filp' in this struct. -- */ --struct io_kiocb { -- union { -- struct file *file; -- struct io_rw rw; -- struct io_poll_iocb poll; -- struct io_accept accept; -- struct io_sync sync; -- struct io_cancel cancel; -- struct io_timeout timeout; -- struct io_timeout_rem timeout_rem; -- struct io_connect connect; -- struct io_sr_msg sr_msg; -- struct io_open open; -- struct io_close close; -- struct io_files_update files_update; -- struct io_fadvise fadvise; -- struct io_madvise madvise; -- struct io_epoll epoll; -- struct io_splice splice; -- struct io_provide_buf pbuf; -- struct io_statx statx; -- /* use only after cleaning per-op data, see io_clean_op() */ -- struct io_completion compl; -- }; -- -- /* opcode allocated if it needs to store data for async defer */ -- void *async_data; -- u8 opcode; -- /* polled IO has completed */ -- u8 iopoll_completed; -- -- u16 buf_index; -- u32 result; -- -- struct io_ring_ctx *ctx; -- unsigned int flags; -- refcount_t refs; -- struct task_struct *task; -- u64 user_data; -- -- struct list_head link_list; -- -- /* -- * 1. used with ctx->iopoll_list with reads/writes -- * 2. to track reqs with ->files (see io_op_def::file_table) -- */ -- struct list_head inflight_entry; -- -- struct list_head iopoll_entry; -- -- struct percpu_ref *fixed_file_refs; -- struct callback_head task_work; -- /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ -- struct hlist_node hash_node; -- struct async_poll *apoll; -- struct io_wq_work work; --}; -- --struct io_defer_entry { -- struct list_head list; -- struct io_kiocb *req; -- u32 seq; --}; -- --#define IO_IOPOLL_BATCH 8 -- --struct io_comp_state { -- unsigned int nr; -- struct list_head list; -- struct io_ring_ctx *ctx; --}; -- --struct io_submit_state { -- struct blk_plug plug; -- -- /* -- * io_kiocb alloc cache -- */ -- void *reqs[IO_IOPOLL_BATCH]; -- unsigned int free_reqs; -- -- /* -- * Batch completion logic -- */ -- struct io_comp_state comp; -- -- /* -- * File reference cache -- */ -- struct file *file; -- unsigned int fd; -- unsigned int has_refs; -- unsigned int ios_left; --}; -- --struct io_op_def { -- /* needs req->file assigned */ -- unsigned needs_file : 1; -- /* don't fail if file grab fails */ -- unsigned needs_file_no_error : 1; -- /* hash wq insertion if file is a regular file */ -- unsigned hash_reg_file : 1; -- /* unbound wq insertion if file is a non-regular file */ -- unsigned unbound_nonreg_file : 1; -- /* opcode is not supported by this kernel */ -- unsigned not_supported : 1; -- /* set if opcode supports polled "wait" */ -- unsigned pollin : 1; -- unsigned pollout : 1; -- /* op supports buffer selection */ -- unsigned buffer_select : 1; -- /* must always have async data allocated */ -- unsigned needs_async_data : 1; -- /* size of async data needed, if any */ -- unsigned short async_size; -- unsigned work_flags; --}; -- --static const struct io_op_def io_op_defs[] = { -- [IORING_OP_NOP] = {}, -- [IORING_OP_READV] = { -- .needs_file = 1, -- .unbound_nonreg_file = 1, -- .pollin = 1, -- .buffer_select = 1, -- .needs_async_data = 1, -- .async_size = sizeof(struct io_async_rw), -- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | -- IO_WQ_WORK_FILES, -- }, -- [IORING_OP_WRITEV] = { -- .needs_file = 1, -- .hash_reg_file = 1, -- .unbound_nonreg_file = 1, -- .pollout = 1, -- .needs_async_data = 1, -- .async_size = sizeof(struct io_async_rw), -- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | -- IO_WQ_WORK_FSIZE | IO_WQ_WORK_FILES, -- }, -- [IORING_OP_FSYNC] = { -- .needs_file = 1, -- .work_flags = IO_WQ_WORK_BLKCG, -- }, -- [IORING_OP_READ_FIXED] = { -- .needs_file = 1, -- .unbound_nonreg_file = 1, -- .pollin = 1, -- .async_size = sizeof(struct io_async_rw), -- .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_MM | -- IO_WQ_WORK_FILES, -- }, -- [IORING_OP_WRITE_FIXED] = { -- .needs_file = 1, -- .hash_reg_file = 1, -- .unbound_nonreg_file = 1, -- .pollout = 1, -- .async_size = sizeof(struct io_async_rw), -- .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE | -- IO_WQ_WORK_MM | IO_WQ_WORK_FILES, -- }, -- [IORING_OP_POLL_ADD] = { -- .needs_file = 1, -- .unbound_nonreg_file = 1, -- }, -- [IORING_OP_POLL_REMOVE] = {}, -- [IORING_OP_SYNC_FILE_RANGE] = { -- .needs_file = 1, -- .work_flags = IO_WQ_WORK_BLKCG, -- }, -- [IORING_OP_SENDMSG] = { -- .needs_file = 1, -- .unbound_nonreg_file = 1, -- .pollout = 1, -- .needs_async_data = 1, -- .async_size = sizeof(struct io_async_msghdr), -- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | -- IO_WQ_WORK_FS, -- }, -- [IORING_OP_RECVMSG] = { -- .needs_file = 1, -- .unbound_nonreg_file = 1, -- .pollin = 1, -- .buffer_select = 1, -- .needs_async_data = 1, -- .async_size = sizeof(struct io_async_msghdr), -- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | -- IO_WQ_WORK_FS, -- }, -- [IORING_OP_TIMEOUT] = { -- .needs_async_data = 1, -- .async_size = sizeof(struct io_timeout_data), -- .work_flags = IO_WQ_WORK_MM, -- }, -- [IORING_OP_TIMEOUT_REMOVE] = {}, -- [IORING_OP_ACCEPT] = { -- .needs_file = 1, -- .unbound_nonreg_file = 1, -- .pollin = 1, -- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_FILES, -- }, -- [IORING_OP_ASYNC_CANCEL] = {}, -- [IORING_OP_LINK_TIMEOUT] = { -- .needs_async_data = 1, -- .async_size = sizeof(struct io_timeout_data), -- .work_flags = IO_WQ_WORK_MM, -- }, -- [IORING_OP_CONNECT] = { -- .needs_file = 1, -- .unbound_nonreg_file = 1, -- .pollout = 1, -- .needs_async_data = 1, -- .async_size = sizeof(struct io_async_connect), -- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_FS, -- }, -- [IORING_OP_FALLOCATE] = { -- .needs_file = 1, -- .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE, -- }, -- [IORING_OP_OPENAT] = { -- .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_BLKCG | -- IO_WQ_WORK_FS, -- }, -- [IORING_OP_CLOSE] = { -- .needs_file = 1, -- .needs_file_no_error = 1, -- .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_BLKCG, -- }, -- [IORING_OP_FILES_UPDATE] = { -- .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_MM, -- }, -- [IORING_OP_STATX] = { -- .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_MM | -- IO_WQ_WORK_FS | IO_WQ_WORK_BLKCG, -- }, -- [IORING_OP_READ] = { -- .needs_file = 1, -- .unbound_nonreg_file = 1, -- .pollin = 1, -- .buffer_select = 1, -- .async_size = sizeof(struct io_async_rw), -- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | -- IO_WQ_WORK_FILES, -- }, -- [IORING_OP_WRITE] = { -- .needs_file = 1, -- .hash_reg_file = 1, -- .unbound_nonreg_file = 1, -- .pollout = 1, -- .async_size = sizeof(struct io_async_rw), -- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | -- IO_WQ_WORK_FSIZE | IO_WQ_WORK_FILES, -- }, -- [IORING_OP_FADVISE] = { -- .needs_file = 1, -- .work_flags = IO_WQ_WORK_BLKCG, -- }, -- [IORING_OP_MADVISE] = { -- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG, -- }, -- [IORING_OP_SEND] = { -- .needs_file = 1, -- .unbound_nonreg_file = 1, -- .pollout = 1, -- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | -- IO_WQ_WORK_FS, -- }, -- [IORING_OP_RECV] = { -- .needs_file = 1, -- .unbound_nonreg_file = 1, -- .pollin = 1, -- .buffer_select = 1, -- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | -- IO_WQ_WORK_FS, -- }, -- [IORING_OP_OPENAT2] = { -- .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_FS | -- IO_WQ_WORK_BLKCG, -- }, -- [IORING_OP_EPOLL_CTL] = { -- .unbound_nonreg_file = 1, -- .work_flags = IO_WQ_WORK_FILES, -- }, -- [IORING_OP_SPLICE] = { -- .needs_file = 1, -- .hash_reg_file = 1, -- .unbound_nonreg_file = 1, -- .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FILES, -- }, -- [IORING_OP_PROVIDE_BUFFERS] = {}, -- [IORING_OP_REMOVE_BUFFERS] = {}, -- [IORING_OP_TEE] = { -- .needs_file = 1, -- .hash_reg_file = 1, -- .unbound_nonreg_file = 1, -- }, --}; -- --enum io_mem_account { -- ACCT_LOCKED, -- ACCT_PINNED, --}; -- --static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node); --static struct fixed_file_ref_node *alloc_fixed_file_ref_node( -- struct io_ring_ctx *ctx); -- --static void __io_complete_rw(struct io_kiocb *req, long res, long res2, -- struct io_comp_state *cs); --static void io_cqring_fill_event(struct io_kiocb *req, long res); --static void io_put_req(struct io_kiocb *req); --static void io_put_req_deferred(struct io_kiocb *req, int nr); --static void io_double_put_req(struct io_kiocb *req); --static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req); --static void __io_queue_linked_timeout(struct io_kiocb *req); --static void io_queue_linked_timeout(struct io_kiocb *req); --static int __io_sqe_files_update(struct io_ring_ctx *ctx, -- struct io_uring_files_update *ip, -- unsigned nr_args); --static void __io_clean_op(struct io_kiocb *req); --static struct file *io_file_get(struct io_submit_state *state, -- struct io_kiocb *req, int fd, bool fixed); --static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs); --static void io_file_put_work(struct work_struct *work); -- --static ssize_t io_import_iovec(int rw, struct io_kiocb *req, -- struct iovec **iovec, struct iov_iter *iter, -- bool needs_lock); --static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, -- const struct iovec *fast_iov, -- struct iov_iter *iter, bool force); --static void io_req_drop_files(struct io_kiocb *req); --static void io_req_task_queue(struct io_kiocb *req); -- --static struct kmem_cache *req_cachep; -- --static const struct file_operations io_uring_fops; -- --struct sock *io_uring_get_socket(struct file *file) --{ --#if defined(CONFIG_UNIX) -- if (file->f_op == &io_uring_fops) { -- struct io_ring_ctx *ctx = file->private_data; -- -- return ctx->ring_sock->sk; -- } --#endif -- return NULL; --} --EXPORT_SYMBOL(io_uring_get_socket); -- --static inline void io_clean_op(struct io_kiocb *req) --{ -- if (req->flags & (REQ_F_NEED_CLEANUP | REQ_F_BUFFER_SELECTED)) -- __io_clean_op(req); --} -- --static inline bool __io_match_files(struct io_kiocb *req, -- struct files_struct *files) --{ -- if (req->file && req->file->f_op == &io_uring_fops) -- return true; -- -- return ((req->flags & REQ_F_WORK_INITIALIZED) && -- (req->work.flags & IO_WQ_WORK_FILES)) && -- req->work.identity->files == files; --} -- --static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl) --{ -- bool got = percpu_ref_tryget(ref); -- -- /* already at zero, wait for ->release() */ -- if (!got) -- wait_for_completion(compl); -- percpu_ref_resurrect(ref); -- if (got) -- percpu_ref_put(ref); --} -- --static bool io_match_task(struct io_kiocb *head, -- struct task_struct *task, -- struct files_struct *files) --{ -- struct io_kiocb *link; -- -- if (task && head->task != task) { -- /* in terms of cancelation, always match if req task is dead */ -- if (head->task->flags & PF_EXITING) -- return true; -- return false; -- } -- if (!files) -- return true; -- if (__io_match_files(head, files)) -- return true; -- if (head->flags & REQ_F_LINK_HEAD) { -- list_for_each_entry(link, &head->link_list, link_list) { -- if (__io_match_files(link, files)) -- return true; -- } -- } -- return false; --} -- -- --static void io_sq_thread_drop_mm(void) --{ -- struct mm_struct *mm = current->mm; -- -- if (mm) { -- kthread_unuse_mm(mm); -- mmput(mm); -- current->mm = NULL; -- } --} -- --static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx) --{ -- struct mm_struct *mm; -- -- if (current->flags & PF_EXITING) -- return -EFAULT; -- if (current->mm) -- return 0; -- -- /* Should never happen */ -- if (unlikely(!(ctx->flags & IORING_SETUP_SQPOLL))) -- return -EFAULT; -- -- task_lock(ctx->sqo_task); -- mm = ctx->sqo_task->mm; -- if (unlikely(!mm || !mmget_not_zero(mm))) -- mm = NULL; -- task_unlock(ctx->sqo_task); -- -- if (mm) { -- kthread_use_mm(mm); -- return 0; -- } -- -- return -EFAULT; --} -- --static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx, -- struct io_kiocb *req) --{ -- if (!(io_op_defs[req->opcode].work_flags & IO_WQ_WORK_MM)) -- return 0; -- return __io_sq_thread_acquire_mm(ctx); --} -- --static void io_sq_thread_associate_blkcg(struct io_ring_ctx *ctx, -- struct cgroup_subsys_state **cur_css) -- --{ --#ifdef CONFIG_BLK_CGROUP -- /* puts the old one when swapping */ -- if (*cur_css != ctx->sqo_blkcg_css) { -- kthread_associate_blkcg(ctx->sqo_blkcg_css); -- *cur_css = ctx->sqo_blkcg_css; -- } --#endif --} -- --static void io_sq_thread_unassociate_blkcg(void) --{ --#ifdef CONFIG_BLK_CGROUP -- kthread_associate_blkcg(NULL); --#endif --} -- --static inline void req_set_fail_links(struct io_kiocb *req) --{ -- if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK) -- req->flags |= REQ_F_FAIL_LINK; --} -- --/* -- * None of these are dereferenced, they are simply used to check if any of -- * them have changed. If we're under current and check they are still the -- * same, we're fine to grab references to them for actual out-of-line use. -- */ --static void io_init_identity(struct io_identity *id) --{ -- id->files = current->files; -- id->mm = current->mm; --#ifdef CONFIG_BLK_CGROUP -- rcu_read_lock(); -- id->blkcg_css = blkcg_css(); -- rcu_read_unlock(); --#endif -- id->creds = current_cred(); -- id->nsproxy = current->nsproxy; -- id->fs = current->fs; -- id->fsize = rlimit(RLIMIT_FSIZE); --#ifdef CONFIG_AUDIT -- id->loginuid = current->loginuid; -- id->sessionid = current->sessionid; --#endif -- refcount_set(&id->count, 1); --} -- --static inline void __io_req_init_async(struct io_kiocb *req) --{ -- memset(&req->work, 0, sizeof(req->work)); -- req->flags |= REQ_F_WORK_INITIALIZED; --} -- --/* -- * Note: must call io_req_init_async() for the first time you -- * touch any members of io_wq_work. -- */ --static inline void io_req_init_async(struct io_kiocb *req) --{ -- struct io_uring_task *tctx = req->task->io_uring; -- -- if (req->flags & REQ_F_WORK_INITIALIZED) -- return; -- -- __io_req_init_async(req); -- -- /* Grab a ref if this isn't our static identity */ -- req->work.identity = tctx->identity; -- if (tctx->identity != &tctx->__identity) -- refcount_inc(&req->work.identity->count); --} -- --static inline bool io_async_submit(struct io_ring_ctx *ctx) --{ -- return ctx->flags & IORING_SETUP_SQPOLL; --} -- --static void io_ring_ctx_ref_free(struct percpu_ref *ref) --{ -- struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); -- -- complete(&ctx->ref_comp); --} -- --static inline bool io_is_timeout_noseq(struct io_kiocb *req) --{ -- return !req->timeout.off; --} -- --static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) --{ -- struct io_ring_ctx *ctx; -- int hash_bits; -- -- ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); -- if (!ctx) -- return NULL; -- -- ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL); -- if (!ctx->fallback_req) -- goto err; -- -- /* -- * Use 5 bits less than the max cq entries, that should give us around -- * 32 entries per hash list if totally full and uniformly spread. -- */ -- hash_bits = ilog2(p->cq_entries); -- hash_bits -= 5; -- if (hash_bits <= 0) -- hash_bits = 1; -- ctx->cancel_hash_bits = hash_bits; -- ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head), -- GFP_KERNEL); -- if (!ctx->cancel_hash) -- goto err; -- __hash_init(ctx->cancel_hash, 1U << hash_bits); -- -- if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free, -- PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) -- goto err; -- -- ctx->flags = p->flags; -- init_waitqueue_head(&ctx->sqo_sq_wait); -- INIT_LIST_HEAD(&ctx->sqd_list); -- init_waitqueue_head(&ctx->cq_wait); -- INIT_LIST_HEAD(&ctx->cq_overflow_list); -- init_completion(&ctx->ref_comp); -- init_completion(&ctx->sq_thread_comp); -- xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1); -- xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1); -- mutex_init(&ctx->uring_lock); -- init_waitqueue_head(&ctx->wait); -- spin_lock_init(&ctx->completion_lock); -- INIT_LIST_HEAD(&ctx->iopoll_list); -- INIT_LIST_HEAD(&ctx->defer_list); -- INIT_LIST_HEAD(&ctx->timeout_list); -- spin_lock_init(&ctx->inflight_lock); -- INIT_LIST_HEAD(&ctx->inflight_list); -- INIT_DELAYED_WORK(&ctx->file_put_work, io_file_put_work); -- init_llist_head(&ctx->file_put_llist); -- return ctx; --err: -- if (ctx->fallback_req) -- kmem_cache_free(req_cachep, ctx->fallback_req); -- kfree(ctx->cancel_hash); -- kfree(ctx); -- return NULL; --} -- --static bool req_need_defer(struct io_kiocb *req, u32 seq) --{ -- if (unlikely(req->flags & REQ_F_IO_DRAIN)) { -- struct io_ring_ctx *ctx = req->ctx; -- -- return seq != ctx->cached_cq_tail -- + READ_ONCE(ctx->cached_cq_overflow); -- } -- -- return false; --} -- --static void __io_commit_cqring(struct io_ring_ctx *ctx) --{ -- struct io_rings *rings = ctx->rings; -- -- /* order cqe stores with ring update */ -- smp_store_release(&rings->cq.tail, ctx->cached_cq_tail); --} -- --static void io_put_identity(struct io_uring_task *tctx, struct io_kiocb *req) --{ -- if (req->work.identity == &tctx->__identity) -- return; -- if (refcount_dec_and_test(&req->work.identity->count)) -- kfree(req->work.identity); --} -- --static void io_req_clean_work(struct io_kiocb *req) --{ -- if (!(req->flags & REQ_F_WORK_INITIALIZED)) -- return; -- -- req->flags &= ~REQ_F_WORK_INITIALIZED; -- -- if (req->work.flags & IO_WQ_WORK_MM) { -- mmdrop(req->work.identity->mm); -- req->work.flags &= ~IO_WQ_WORK_MM; -- } --#ifdef CONFIG_BLK_CGROUP -- if (req->work.flags & IO_WQ_WORK_BLKCG) { -- css_put(req->work.identity->blkcg_css); -- req->work.flags &= ~IO_WQ_WORK_BLKCG; -- } --#endif -- if (req->work.flags & IO_WQ_WORK_CREDS) { -- put_cred(req->work.identity->creds); -- req->work.flags &= ~IO_WQ_WORK_CREDS; -- } -- if (req->work.flags & IO_WQ_WORK_FS) { -- struct fs_struct *fs = req->work.identity->fs; -- -- spin_lock(&req->work.identity->fs->lock); -- if (--fs->users) -- fs = NULL; -- spin_unlock(&req->work.identity->fs->lock); -- if (fs) -- free_fs_struct(fs); -- req->work.flags &= ~IO_WQ_WORK_FS; -- } -- if (req->flags & REQ_F_INFLIGHT) -- io_req_drop_files(req); -- -- io_put_identity(req->task->io_uring, req); --} -- --/* -- * Create a private copy of io_identity, since some fields don't match -- * the current context. -- */ --static bool io_identity_cow(struct io_kiocb *req) --{ -- struct io_uring_task *tctx = req->task->io_uring; -- const struct cred *creds = NULL; -- struct io_identity *id; -- -- if (req->work.flags & IO_WQ_WORK_CREDS) -- creds = req->work.identity->creds; -- -- id = kmemdup(req->work.identity, sizeof(*id), GFP_KERNEL); -- if (unlikely(!id)) { -- req->work.flags |= IO_WQ_WORK_CANCEL; -- return false; -- } -- -- /* -- * We can safely just re-init the creds we copied Either the field -- * matches the current one, or we haven't grabbed it yet. The only -- * exception is ->creds, through registered personalities, so handle -- * that one separately. -- */ -- io_init_identity(id); -- if (creds) -- id->creds = creds; -- -- /* add one for this request */ -- refcount_inc(&id->count); -- -- /* drop tctx and req identity references, if needed */ -- if (tctx->identity != &tctx->__identity && -- refcount_dec_and_test(&tctx->identity->count)) -- kfree(tctx->identity); -- if (req->work.identity != &tctx->__identity && -- refcount_dec_and_test(&req->work.identity->count)) -- kfree(req->work.identity); -- -- req->work.identity = id; -- tctx->identity = id; -- return true; --} -- --static bool io_grab_identity(struct io_kiocb *req) --{ -- const struct io_op_def *def = &io_op_defs[req->opcode]; -- struct io_identity *id = req->work.identity; -- struct io_ring_ctx *ctx = req->ctx; -- -- if (def->work_flags & IO_WQ_WORK_FSIZE) { -- if (id->fsize != rlimit(RLIMIT_FSIZE)) -- return false; -- req->work.flags |= IO_WQ_WORK_FSIZE; -- } --#ifdef CONFIG_BLK_CGROUP -- if (!(req->work.flags & IO_WQ_WORK_BLKCG) && -- (def->work_flags & IO_WQ_WORK_BLKCG)) { -- rcu_read_lock(); -- if (id->blkcg_css != blkcg_css()) { -- rcu_read_unlock(); -- return false; -- } -- /* -- * This should be rare, either the cgroup is dying or the task -- * is moving cgroups. Just punt to root for the handful of ios. -- */ -- if (css_tryget_online(id->blkcg_css)) -- req->work.flags |= IO_WQ_WORK_BLKCG; -- rcu_read_unlock(); -- } --#endif -- if (!(req->work.flags & IO_WQ_WORK_CREDS)) { -- if (id->creds != current_cred()) -- return false; -- get_cred(id->creds); -- req->work.flags |= IO_WQ_WORK_CREDS; -- } --#ifdef CONFIG_AUDIT -- if (!uid_eq(current->loginuid, id->loginuid) || -- current->sessionid != id->sessionid) -- return false; --#endif -- if (!(req->work.flags & IO_WQ_WORK_FS) && -- (def->work_flags & IO_WQ_WORK_FS)) { -- if (current->fs != id->fs) -- return false; -- spin_lock(&id->fs->lock); -- if (!id->fs->in_exec) { -- id->fs->users++; -- req->work.flags |= IO_WQ_WORK_FS; -- } else { -- req->work.flags |= IO_WQ_WORK_CANCEL; -- } -- spin_unlock(¤t->fs->lock); -- } -- if (!(req->work.flags & IO_WQ_WORK_FILES) && -- (def->work_flags & IO_WQ_WORK_FILES) && -- !(req->flags & REQ_F_NO_FILE_TABLE)) { -- if (id->files != current->files || -- id->nsproxy != current->nsproxy) -- return false; -- atomic_inc(&id->files->count); -- get_nsproxy(id->nsproxy); -- -- if (!(req->flags & REQ_F_INFLIGHT)) { -- req->flags |= REQ_F_INFLIGHT; -- -- spin_lock_irq(&ctx->inflight_lock); -- list_add(&req->inflight_entry, &ctx->inflight_list); -- spin_unlock_irq(&ctx->inflight_lock); -- } -- req->work.flags |= IO_WQ_WORK_FILES; -- } -- if (!(req->work.flags & IO_WQ_WORK_MM) && -- (def->work_flags & IO_WQ_WORK_MM)) { -- if (id->mm != current->mm) -- return false; -- mmgrab(id->mm); -- req->work.flags |= IO_WQ_WORK_MM; -- } -- -- return true; --} -- --static void io_prep_async_work(struct io_kiocb *req) --{ -- const struct io_op_def *def = &io_op_defs[req->opcode]; -- struct io_ring_ctx *ctx = req->ctx; -- struct io_identity *id; -- -- io_req_init_async(req); -- id = req->work.identity; -- -- if (req->flags & REQ_F_FORCE_ASYNC) -- req->work.flags |= IO_WQ_WORK_CONCURRENT; -- -- if (req->flags & REQ_F_ISREG) { -- if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL)) -- io_wq_hash_work(&req->work, file_inode(req->file)); -- } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) { -- if (def->unbound_nonreg_file) -- req->work.flags |= IO_WQ_WORK_UNBOUND; -- } -- -- /* if we fail grabbing identity, we must COW, regrab, and retry */ -- if (io_grab_identity(req)) -- return; -- -- if (!io_identity_cow(req)) -- return; -- -- /* can't fail at this point */ -- if (!io_grab_identity(req)) -- WARN_ON(1); --} -- --static void io_prep_async_link(struct io_kiocb *req) --{ -- struct io_kiocb *cur; -- -- io_prep_async_work(req); -- if (req->flags & REQ_F_LINK_HEAD) -- list_for_each_entry(cur, &req->link_list, link_list) -- io_prep_async_work(cur); --} -- --static struct io_kiocb *__io_queue_async_work(struct io_kiocb *req) --{ -- struct io_ring_ctx *ctx = req->ctx; -- struct io_kiocb *link = io_prep_linked_timeout(req); -- -- trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req, -- &req->work, req->flags); -- io_wq_enqueue(ctx->io_wq, &req->work); -- return link; --} -- --static void io_queue_async_work(struct io_kiocb *req) --{ -- struct io_kiocb *link; -- -- /* init ->work of the whole link before punting */ -- io_prep_async_link(req); -- link = __io_queue_async_work(req); -- -- if (link) -- io_queue_linked_timeout(link); --} -- --static void io_kill_timeout(struct io_kiocb *req, int status) --{ -- struct io_timeout_data *io = req->async_data; -- int ret; -- -- ret = hrtimer_try_to_cancel(&io->timer); -- if (ret != -1) { -- if (status) -- req_set_fail_links(req); -- atomic_set(&req->ctx->cq_timeouts, -- atomic_read(&req->ctx->cq_timeouts) + 1); -- list_del_init(&req->timeout.list); -- io_cqring_fill_event(req, status); -- io_put_req_deferred(req, 1); -- } --} -- --/* -- * Returns true if we found and killed one or more timeouts -- */ --static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, -- struct files_struct *files) --{ -- struct io_kiocb *req, *tmp; -- int canceled = 0; -- -- spin_lock_irq(&ctx->completion_lock); -- list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { -- if (io_match_task(req, tsk, files)) { -- io_kill_timeout(req, -ECANCELED); -- canceled++; -- } -- } -- spin_unlock_irq(&ctx->completion_lock); -- return canceled != 0; --} -- --static void __io_queue_deferred(struct io_ring_ctx *ctx) --{ -- do { -- struct io_defer_entry *de = list_first_entry(&ctx->defer_list, -- struct io_defer_entry, list); -- -- if (req_need_defer(de->req, de->seq)) -- break; -- list_del_init(&de->list); -- io_req_task_queue(de->req); -- kfree(de); -- } while (!list_empty(&ctx->defer_list)); --} -- --static void io_flush_timeouts(struct io_ring_ctx *ctx) --{ -- struct io_kiocb *req, *tmp; -- u32 seq; -- -- if (list_empty(&ctx->timeout_list)) -- return; -- -- seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); -- -- list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { -- u32 events_needed, events_got; -- -- if (io_is_timeout_noseq(req)) -- break; -- -- /* -- * Since seq can easily wrap around over time, subtract -- * the last seq at which timeouts were flushed before comparing. -- * Assuming not more than 2^31-1 events have happened since, -- * these subtractions won't have wrapped, so we can check if -- * target is in [last_seq, current_seq] by comparing the two. -- */ -- events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush; -- events_got = seq - ctx->cq_last_tm_flush; -- if (events_got < events_needed) -- break; -- -- io_kill_timeout(req, 0); -- } -- -- ctx->cq_last_tm_flush = seq; --} -- --static void io_commit_cqring(struct io_ring_ctx *ctx) --{ -- io_flush_timeouts(ctx); -- __io_commit_cqring(ctx); -- -- if (unlikely(!list_empty(&ctx->defer_list))) -- __io_queue_deferred(ctx); --} -- --static inline bool io_sqring_full(struct io_ring_ctx *ctx) --{ -- struct io_rings *r = ctx->rings; -- -- return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == r->sq_ring_entries; --} -- --static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx) --{ -- struct io_rings *rings = ctx->rings; -- unsigned tail; -- -- tail = ctx->cached_cq_tail; -- /* -- * writes to the cq entry need to come after reading head; the -- * control dependency is enough as we're using WRITE_ONCE to -- * fill the cq entry -- */ -- if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries) -- return NULL; -- -- ctx->cached_cq_tail++; -- return &rings->cqes[tail & ctx->cq_mask]; --} -- --static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx) --{ -- if (!ctx->cq_ev_fd) -- return false; -- if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) -- return false; -- if (!ctx->eventfd_async) -- return true; -- return io_wq_current_is_worker(); --} -- --static void io_cqring_ev_posted(struct io_ring_ctx *ctx) --{ -- if (wq_has_sleeper(&ctx->cq_wait)) { -- wake_up_interruptible(&ctx->cq_wait); -- kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN); -- } -- if (waitqueue_active(&ctx->wait)) -- wake_up(&ctx->wait); -- if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait)) -- wake_up(&ctx->sq_data->wait); -- if (io_should_trigger_evfd(ctx)) -- eventfd_signal(ctx->cq_ev_fd, 1); --} -- --static void io_cqring_mark_overflow(struct io_ring_ctx *ctx) --{ -- if (list_empty(&ctx->cq_overflow_list)) { -- clear_bit(0, &ctx->sq_check_overflow); -- clear_bit(0, &ctx->cq_check_overflow); -- ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW; -- } --} -- --/* Returns true if there are no backlogged entries after the flush */ --static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, -- struct task_struct *tsk, -- struct files_struct *files) --{ -- struct io_rings *rings = ctx->rings; -- struct io_kiocb *req, *tmp; -- struct io_uring_cqe *cqe; -- unsigned long flags; -- LIST_HEAD(list); -- -- if (!force) { -- if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) == -- rings->cq_ring_entries)) -- return false; -- } -- -- spin_lock_irqsave(&ctx->completion_lock, flags); -- -- cqe = NULL; -- list_for_each_entry_safe(req, tmp, &ctx->cq_overflow_list, compl.list) { -- if (!io_match_task(req, tsk, files)) -- continue; -- -- cqe = io_get_cqring(ctx); -- if (!cqe && !force) -- break; -- -- list_move(&req->compl.list, &list); -- if (cqe) { -- WRITE_ONCE(cqe->user_data, req->user_data); -- WRITE_ONCE(cqe->res, req->result); -- WRITE_ONCE(cqe->flags, req->compl.cflags); -- } else { -- ctx->cached_cq_overflow++; -- WRITE_ONCE(ctx->rings->cq_overflow, -- ctx->cached_cq_overflow); -- } -- } -- -- io_commit_cqring(ctx); -- io_cqring_mark_overflow(ctx); -- -- spin_unlock_irqrestore(&ctx->completion_lock, flags); -- io_cqring_ev_posted(ctx); -- -- while (!list_empty(&list)) { -- req = list_first_entry(&list, struct io_kiocb, compl.list); -- list_del(&req->compl.list); -- io_put_req(req); -- } -- -- return cqe != NULL; --} -- --static void io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, -- struct task_struct *tsk, -- struct files_struct *files) --{ -- if (test_bit(0, &ctx->cq_check_overflow)) { -- /* iopoll syncs against uring_lock, not completion_lock */ -- if (ctx->flags & IORING_SETUP_IOPOLL) -- mutex_lock(&ctx->uring_lock); -- __io_cqring_overflow_flush(ctx, force, tsk, files); -- if (ctx->flags & IORING_SETUP_IOPOLL) -- mutex_unlock(&ctx->uring_lock); -- } --} -- --static void __io_cqring_fill_event(struct io_kiocb *req, long res, -- unsigned int cflags) --{ -- struct io_ring_ctx *ctx = req->ctx; -- struct io_uring_cqe *cqe; -- -- trace_io_uring_complete(ctx, req->user_data, res); -- -- /* -- * If we can't get a cq entry, userspace overflowed the -- * submission (by quite a lot). Increment the overflow count in -- * the ring. -- */ -- cqe = io_get_cqring(ctx); -- if (likely(cqe)) { -- WRITE_ONCE(cqe->user_data, req->user_data); -- WRITE_ONCE(cqe->res, res); -- WRITE_ONCE(cqe->flags, cflags); -- } else if (ctx->cq_overflow_flushed || -- atomic_read(&req->task->io_uring->in_idle)) { -- /* -- * If we're in ring overflow flush mode, or in task cancel mode, -- * then we cannot store the request for later flushing, we need -- * to drop it on the floor. -- */ -- ctx->cached_cq_overflow++; -- WRITE_ONCE(ctx->rings->cq_overflow, ctx->cached_cq_overflow); -- } else { -- if (list_empty(&ctx->cq_overflow_list)) { -- set_bit(0, &ctx->sq_check_overflow); -- set_bit(0, &ctx->cq_check_overflow); -- ctx->rings->sq_flags |= IORING_SQ_CQ_OVERFLOW; -- } -- io_clean_op(req); -- req->result = res; -- req->compl.cflags = cflags; -- refcount_inc(&req->refs); -- list_add_tail(&req->compl.list, &ctx->cq_overflow_list); -- } --} -- --static void io_cqring_fill_event(struct io_kiocb *req, long res) --{ -- __io_cqring_fill_event(req, res, 0); --} -- --static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags) --{ -- struct io_ring_ctx *ctx = req->ctx; -- unsigned long flags; -- -- spin_lock_irqsave(&ctx->completion_lock, flags); -- __io_cqring_fill_event(req, res, cflags); -- io_commit_cqring(ctx); -- spin_unlock_irqrestore(&ctx->completion_lock, flags); -- -- io_cqring_ev_posted(ctx); --} -- --static void io_submit_flush_completions(struct io_comp_state *cs) --{ -- struct io_ring_ctx *ctx = cs->ctx; -- -- spin_lock_irq(&ctx->completion_lock); -- while (!list_empty(&cs->list)) { -- struct io_kiocb *req; -- -- req = list_first_entry(&cs->list, struct io_kiocb, compl.list); -- list_del(&req->compl.list); -- __io_cqring_fill_event(req, req->result, req->compl.cflags); -- -- /* -- * io_free_req() doesn't care about completion_lock unless one -- * of these flags is set. REQ_F_WORK_INITIALIZED is in the list -- * because of a potential deadlock with req->work.fs->lock -- */ -- if (req->flags & (REQ_F_FAIL_LINK|REQ_F_LINK_TIMEOUT -- |REQ_F_WORK_INITIALIZED)) { -- spin_unlock_irq(&ctx->completion_lock); -- io_put_req(req); -- spin_lock_irq(&ctx->completion_lock); -- } else { -- io_put_req(req); -- } -- } -- io_commit_cqring(ctx); -- spin_unlock_irq(&ctx->completion_lock); -- -- io_cqring_ev_posted(ctx); -- cs->nr = 0; --} -- --static void __io_req_complete(struct io_kiocb *req, long res, unsigned cflags, -- struct io_comp_state *cs) --{ -- if (!cs) { -- io_cqring_add_event(req, res, cflags); -- io_put_req(req); -- } else { -- io_clean_op(req); -- req->result = res; -- req->compl.cflags = cflags; -- list_add_tail(&req->compl.list, &cs->list); -- if (++cs->nr >= 32) -- io_submit_flush_completions(cs); -- } --} -- --static void io_req_complete(struct io_kiocb *req, long res) --{ -- __io_req_complete(req, res, 0, NULL); --} -- --static inline bool io_is_fallback_req(struct io_kiocb *req) --{ -- return req == (struct io_kiocb *) -- ((unsigned long) req->ctx->fallback_req & ~1UL); --} -- --static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx) --{ -- struct io_kiocb *req; -- -- req = ctx->fallback_req; -- if (!test_and_set_bit_lock(0, (unsigned long *) &ctx->fallback_req)) -- return req; -- -- return NULL; --} -- --static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx, -- struct io_submit_state *state) --{ -- if (!state->free_reqs) { -- gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; -- size_t sz; -- int ret; -- -- sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs)); -- ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs); -- -- /* -- * Bulk alloc is all-or-nothing. If we fail to get a batch, -- * retry single alloc to be on the safe side. -- */ -- if (unlikely(ret <= 0)) { -- state->reqs[0] = kmem_cache_alloc(req_cachep, gfp); -- if (!state->reqs[0]) -- goto fallback; -- ret = 1; -- } -- state->free_reqs = ret; -- } -- -- state->free_reqs--; -- return state->reqs[state->free_reqs]; --fallback: -- return io_get_fallback_req(ctx); --} -- --static inline void io_put_file(struct io_kiocb *req, struct file *file, -- bool fixed) --{ -- if (fixed) -- percpu_ref_put(req->fixed_file_refs); -- else -- fput(file); --} -- --static void io_dismantle_req(struct io_kiocb *req) --{ -- io_clean_op(req); -- -- if (req->async_data) -- kfree(req->async_data); -- if (req->file) -- io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE)); -- -- io_req_clean_work(req); --} -- --static void __io_free_req(struct io_kiocb *req) --{ -- struct io_uring_task *tctx = req->task->io_uring; -- struct io_ring_ctx *ctx = req->ctx; -- -- io_dismantle_req(req); -- -- percpu_counter_dec(&tctx->inflight); -- if (atomic_read(&tctx->in_idle)) -- wake_up(&tctx->wait); -- put_task_struct(req->task); -- -- if (likely(!io_is_fallback_req(req))) -- kmem_cache_free(req_cachep, req); -- else -- clear_bit_unlock(0, (unsigned long *) &ctx->fallback_req); -- percpu_ref_put(&ctx->refs); --} -- --static void io_kill_linked_timeout(struct io_kiocb *req) --{ -- struct io_ring_ctx *ctx = req->ctx; -- struct io_kiocb *link; -- bool cancelled = false; -- unsigned long flags; -- -- spin_lock_irqsave(&ctx->completion_lock, flags); -- link = list_first_entry_or_null(&req->link_list, struct io_kiocb, -- link_list); -- /* -- * Can happen if a linked timeout fired and link had been like -- * req -> link t-out -> link t-out [-> ...] -- */ -- if (link && (link->flags & REQ_F_LTIMEOUT_ACTIVE)) { -- struct io_timeout_data *io = link->async_data; -- int ret; -- -- list_del_init(&link->link_list); -- ret = hrtimer_try_to_cancel(&io->timer); -- if (ret != -1) { -- io_cqring_fill_event(link, -ECANCELED); -- io_commit_cqring(ctx); -- cancelled = true; -- } -- } -- req->flags &= ~REQ_F_LINK_TIMEOUT; -- spin_unlock_irqrestore(&ctx->completion_lock, flags); -- -- if (cancelled) { -- io_cqring_ev_posted(ctx); -- io_put_req(link); -- } --} -- --static struct io_kiocb *io_req_link_next(struct io_kiocb *req) --{ -- struct io_kiocb *nxt; -- -- /* -- * The list should never be empty when we are called here. But could -- * potentially happen if the chain is messed up, check to be on the -- * safe side. -- */ -- if (unlikely(list_empty(&req->link_list))) -- return NULL; -- -- nxt = list_first_entry(&req->link_list, struct io_kiocb, link_list); -- list_del_init(&req->link_list); -- if (!list_empty(&nxt->link_list)) -- nxt->flags |= REQ_F_LINK_HEAD; -- return nxt; --} -- --/* -- * Called if REQ_F_LINK_HEAD is set, and we fail the head request -- */ --static void io_fail_links(struct io_kiocb *req) --{ -- struct io_ring_ctx *ctx = req->ctx; -- unsigned long flags; -- -- spin_lock_irqsave(&ctx->completion_lock, flags); -- while (!list_empty(&req->link_list)) { -- struct io_kiocb *link = list_first_entry(&req->link_list, -- struct io_kiocb, link_list); -- -- list_del_init(&link->link_list); -- trace_io_uring_fail_link(req, link); -- -- io_cqring_fill_event(link, -ECANCELED); -- -- /* -- * It's ok to free under spinlock as they're not linked anymore, -- * but avoid REQ_F_WORK_INITIALIZED because it may deadlock on -- * work.fs->lock. -- */ -- if (link->flags & REQ_F_WORK_INITIALIZED) -- io_put_req_deferred(link, 2); -- else -- io_double_put_req(link); -- } -- -- io_commit_cqring(ctx); -- spin_unlock_irqrestore(&ctx->completion_lock, flags); -- -- io_cqring_ev_posted(ctx); --} -- --static struct io_kiocb *__io_req_find_next(struct io_kiocb *req) --{ -- req->flags &= ~REQ_F_LINK_HEAD; -- if (req->flags & REQ_F_LINK_TIMEOUT) -- io_kill_linked_timeout(req); -- -- /* -- * If LINK is set, we have dependent requests in this chain. If we -- * didn't fail this request, queue the first one up, moving any other -- * dependencies to the next request. In case of failure, fail the rest -- * of the chain. -- */ -- if (likely(!(req->flags & REQ_F_FAIL_LINK))) -- return io_req_link_next(req); -- io_fail_links(req); -- return NULL; --} -- --static struct io_kiocb *io_req_find_next(struct io_kiocb *req) --{ -- if (likely(!(req->flags & REQ_F_LINK_HEAD))) -- return NULL; -- return __io_req_find_next(req); --} -- --static int io_req_task_work_add(struct io_kiocb *req, bool twa_signal_ok) --{ -- struct task_struct *tsk = req->task; -- struct io_ring_ctx *ctx = req->ctx; -- enum task_work_notify_mode notify; -- int ret; -- -- if (tsk->flags & PF_EXITING) -- return -ESRCH; -- -- /* -- * SQPOLL kernel thread doesn't need notification, just a wakeup. For -- * all other cases, use TWA_SIGNAL unconditionally to ensure we're -- * processing task_work. There's no reliable way to tell if TWA_RESUME -- * will do the job. -- */ -- notify = TWA_NONE; -- if (!(ctx->flags & IORING_SETUP_SQPOLL) && twa_signal_ok) -- notify = TWA_SIGNAL; -- -- ret = task_work_add(tsk, &req->task_work, notify); -- if (!ret) -- wake_up_process(tsk); -- -- return ret; --} -- --static void __io_req_task_cancel(struct io_kiocb *req, int error) --{ -- struct io_ring_ctx *ctx = req->ctx; -- -- spin_lock_irq(&ctx->completion_lock); -- io_cqring_fill_event(req, error); -- io_commit_cqring(ctx); -- spin_unlock_irq(&ctx->completion_lock); -- -- io_cqring_ev_posted(ctx); -- req_set_fail_links(req); -- io_double_put_req(req); --} -- --static void io_req_task_cancel(struct callback_head *cb) --{ -- struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); -- struct io_ring_ctx *ctx = req->ctx; -- -- mutex_lock(&ctx->uring_lock); -- __io_req_task_cancel(req, -ECANCELED); -- mutex_unlock(&ctx->uring_lock); -- percpu_ref_put(&ctx->refs); --} -- --static void __io_req_task_submit(struct io_kiocb *req) --{ -- struct io_ring_ctx *ctx = req->ctx; -- -- mutex_lock(&ctx->uring_lock); -- if (!ctx->sqo_dead && !__io_sq_thread_acquire_mm(ctx)) -- __io_queue_sqe(req, NULL); -- else -- __io_req_task_cancel(req, -EFAULT); -- mutex_unlock(&ctx->uring_lock); -- -- if (ctx->flags & IORING_SETUP_SQPOLL) -- io_sq_thread_drop_mm(); --} -- --static void io_req_task_submit(struct callback_head *cb) --{ -- struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); -- struct io_ring_ctx *ctx = req->ctx; -- -- __io_req_task_submit(req); -- percpu_ref_put(&ctx->refs); --} -- --static void io_req_task_queue(struct io_kiocb *req) --{ -- int ret; -- -- init_task_work(&req->task_work, io_req_task_submit); -- percpu_ref_get(&req->ctx->refs); -- -- ret = io_req_task_work_add(req, true); -- if (unlikely(ret)) { -- struct task_struct *tsk; -- -- init_task_work(&req->task_work, io_req_task_cancel); -- tsk = io_wq_get_task(req->ctx->io_wq); -- task_work_add(tsk, &req->task_work, TWA_NONE); -- wake_up_process(tsk); -- } --} -- --static void io_queue_next(struct io_kiocb *req) --{ -- struct io_kiocb *nxt = io_req_find_next(req); -- -- if (nxt) -- io_req_task_queue(nxt); --} -- --static void io_free_req(struct io_kiocb *req) --{ -- io_queue_next(req); -- __io_free_req(req); --} -- --struct req_batch { -- void *reqs[IO_IOPOLL_BATCH]; -- int to_free; -- -- struct task_struct *task; -- int task_refs; --}; -- --static inline void io_init_req_batch(struct req_batch *rb) --{ -- rb->to_free = 0; -- rb->task_refs = 0; -- rb->task = NULL; --} -- --static void __io_req_free_batch_flush(struct io_ring_ctx *ctx, -- struct req_batch *rb) --{ -- kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs); -- percpu_ref_put_many(&ctx->refs, rb->to_free); -- rb->to_free = 0; --} -- --static void io_req_free_batch_finish(struct io_ring_ctx *ctx, -- struct req_batch *rb) --{ -- if (rb->to_free) -- __io_req_free_batch_flush(ctx, rb); -- if (rb->task) { -- struct io_uring_task *tctx = rb->task->io_uring; -- -- percpu_counter_sub(&tctx->inflight, rb->task_refs); -- if (atomic_read(&tctx->in_idle)) -- wake_up(&tctx->wait); -- put_task_struct_many(rb->task, rb->task_refs); -- rb->task = NULL; -- } --} -- --static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) --{ -- if (unlikely(io_is_fallback_req(req))) { -- io_free_req(req); -- return; -- } -- if (req->flags & REQ_F_LINK_HEAD) -- io_queue_next(req); -- -- if (req->task != rb->task) { -- if (rb->task) { -- struct io_uring_task *tctx = rb->task->io_uring; -- -- percpu_counter_sub(&tctx->inflight, rb->task_refs); -- if (atomic_read(&tctx->in_idle)) -- wake_up(&tctx->wait); -- put_task_struct_many(rb->task, rb->task_refs); -- } -- rb->task = req->task; -- rb->task_refs = 0; -- } -- rb->task_refs++; -- -- io_dismantle_req(req); -- rb->reqs[rb->to_free++] = req; -- if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs))) -- __io_req_free_batch_flush(req->ctx, rb); --} -- --/* -- * Drop reference to request, return next in chain (if there is one) if this -- * was the last reference to this request. -- */ --static struct io_kiocb *io_put_req_find_next(struct io_kiocb *req) --{ -- struct io_kiocb *nxt = NULL; -- -- if (refcount_dec_and_test(&req->refs)) { -- nxt = io_req_find_next(req); -- __io_free_req(req); -- } -- return nxt; --} -- --static void io_put_req(struct io_kiocb *req) --{ -- if (refcount_dec_and_test(&req->refs)) -- io_free_req(req); --} -- --static void io_put_req_deferred_cb(struct callback_head *cb) --{ -- struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); -- -- io_free_req(req); --} -- --static void io_free_req_deferred(struct io_kiocb *req) --{ -- int ret; -- -- init_task_work(&req->task_work, io_put_req_deferred_cb); -- ret = io_req_task_work_add(req, true); -- if (unlikely(ret)) { -- struct task_struct *tsk; -- -- tsk = io_wq_get_task(req->ctx->io_wq); -- task_work_add(tsk, &req->task_work, TWA_NONE); -- wake_up_process(tsk); -- } --} -- --static inline void io_put_req_deferred(struct io_kiocb *req, int refs) --{ -- if (refcount_sub_and_test(refs, &req->refs)) -- io_free_req_deferred(req); --} -- --static struct io_wq_work *io_steal_work(struct io_kiocb *req) --{ -- struct io_kiocb *nxt; -- -- /* -- * A ref is owned by io-wq in which context we're. So, if that's the -- * last one, it's safe to steal next work. False negatives are Ok, -- * it just will be re-punted async in io_put_work() -- */ -- if (refcount_read(&req->refs) != 1) -- return NULL; -- -- nxt = io_req_find_next(req); -- return nxt ? &nxt->work : NULL; --} -- --static void io_double_put_req(struct io_kiocb *req) --{ -- /* drop both submit and complete references */ -- if (refcount_sub_and_test(2, &req->refs)) -- io_free_req(req); --} -- --static unsigned io_cqring_events(struct io_ring_ctx *ctx) --{ -- struct io_rings *rings = ctx->rings; -- -- /* See comment at the top of this file */ -- smp_rmb(); -- return ctx->cached_cq_tail - READ_ONCE(rings->cq.head); --} -- --static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) --{ -- struct io_rings *rings = ctx->rings; -- -- /* make sure SQ entry isn't read before tail */ -- return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; --} -- --static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf) --{ -- unsigned int cflags; -- -- cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT; -- cflags |= IORING_CQE_F_BUFFER; -- req->flags &= ~REQ_F_BUFFER_SELECTED; -- kfree(kbuf); -- return cflags; --} -- --static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req) --{ -- struct io_buffer *kbuf; -- -- kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; -- return io_put_kbuf(req, kbuf); --} -- --static inline bool io_run_task_work(void) --{ -- /* -- * Not safe to run on exiting task, and the task_work handling will -- * not add work to such a task. -- */ -- if (unlikely(current->flags & PF_EXITING)) -- return false; -- if (current->task_works) { -- __set_current_state(TASK_RUNNING); -- task_work_run(); -- return true; -- } -- -- return false; --} -- --static void io_iopoll_queue(struct list_head *again) --{ -- struct io_kiocb *req; -- -- do { -- req = list_first_entry(again, struct io_kiocb, iopoll_entry); -- list_del(&req->iopoll_entry); -- __io_complete_rw(req, -EAGAIN, 0, NULL); -- } while (!list_empty(again)); --} -- --/* -- * Find and free completed poll iocbs -- */ --static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, -- struct list_head *done) --{ -- struct req_batch rb; -- struct io_kiocb *req; -- LIST_HEAD(again); -- -- /* order with ->result store in io_complete_rw_iopoll() */ -- smp_rmb(); -- -- io_init_req_batch(&rb); -- while (!list_empty(done)) { -- int cflags = 0; -- -- req = list_first_entry(done, struct io_kiocb, iopoll_entry); -- if (READ_ONCE(req->result) == -EAGAIN) { -- req->result = 0; -- req->iopoll_completed = 0; -- list_move_tail(&req->iopoll_entry, &again); -- continue; -- } -- list_del(&req->iopoll_entry); -- -- if (req->flags & REQ_F_BUFFER_SELECTED) -- cflags = io_put_rw_kbuf(req); -- -- __io_cqring_fill_event(req, req->result, cflags); -- (*nr_events)++; -- -- if (refcount_dec_and_test(&req->refs)) -- io_req_free_batch(&rb, req); -- } -- -- io_commit_cqring(ctx); -- if (ctx->flags & IORING_SETUP_SQPOLL) -- io_cqring_ev_posted(ctx); -- io_req_free_batch_finish(ctx, &rb); -- -- if (!list_empty(&again)) -- io_iopoll_queue(&again); --} -- --static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, -- long min) --{ -- struct io_kiocb *req, *tmp; -- LIST_HEAD(done); -- bool spin; -- int ret; -- -- /* -- * Only spin for completions if we don't have multiple devices hanging -- * off our complete list, and we're under the requested amount. -- */ -- spin = !ctx->poll_multi_file && *nr_events < min; -- -- ret = 0; -- list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, iopoll_entry) { -- struct kiocb *kiocb = &req->rw.kiocb; -- -- /* -- * Move completed and retryable entries to our local lists. -- * If we find a request that requires polling, break out -- * and complete those lists first, if we have entries there. -- */ -- if (READ_ONCE(req->iopoll_completed)) { -- list_move_tail(&req->iopoll_entry, &done); -- continue; -- } -- if (!list_empty(&done)) -- break; -- -- ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin); -- if (ret < 0) -- break; -- -- /* iopoll may have completed current req */ -- if (READ_ONCE(req->iopoll_completed)) -- list_move_tail(&req->iopoll_entry, &done); -- -- if (ret && spin) -- spin = false; -- ret = 0; -- } -- -- if (!list_empty(&done)) -- io_iopoll_complete(ctx, nr_events, &done); -- -- return ret; --} -- --/* -- * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a -- * non-spinning poll check - we'll still enter the driver poll loop, but only -- * as a non-spinning completion check. -- */ --static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events, -- long min) --{ -- while (!list_empty(&ctx->iopoll_list) && !need_resched()) { -- int ret; -- -- ret = io_do_iopoll(ctx, nr_events, min); -- if (ret < 0) -- return ret; -- if (*nr_events >= min) -- return 0; -- } -- -- return 1; --} -- --/* -- * We can't just wait for polled events to come to us, we have to actively -- * find and complete them. -- */ --static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx) --{ -- if (!(ctx->flags & IORING_SETUP_IOPOLL)) -- return; -- -- mutex_lock(&ctx->uring_lock); -- while (!list_empty(&ctx->iopoll_list)) { -- unsigned int nr_events = 0; -- -- io_do_iopoll(ctx, &nr_events, 0); -- -- /* let it sleep and repeat later if can't complete a request */ -- if (nr_events == 0) -- break; -- /* -- * Ensure we allow local-to-the-cpu processing to take place, -- * in this case we need to ensure that we reap all events. -- * Also let task_work, etc. to progress by releasing the mutex -- */ -- if (need_resched()) { -- mutex_unlock(&ctx->uring_lock); -- cond_resched(); -- mutex_lock(&ctx->uring_lock); -- } -- } -- mutex_unlock(&ctx->uring_lock); --} -- --static int io_iopoll_check(struct io_ring_ctx *ctx, long min) --{ -- unsigned int nr_events = 0; -- int iters = 0, ret = 0; -- -- /* -- * We disallow the app entering submit/complete with polling, but we -- * still need to lock the ring to prevent racing with polled issue -- * that got punted to a workqueue. -- */ -- mutex_lock(&ctx->uring_lock); -- do { -- /* -- * Don't enter poll loop if we already have events pending. -- * If we do, we can potentially be spinning for commands that -- * already triggered a CQE (eg in error). -- */ -- if (test_bit(0, &ctx->cq_check_overflow)) -- __io_cqring_overflow_flush(ctx, false, NULL, NULL); -- if (io_cqring_events(ctx)) -- break; -- -- /* -- * If a submit got punted to a workqueue, we can have the -- * application entering polling for a command before it gets -- * issued. That app will hold the uring_lock for the duration -- * of the poll right here, so we need to take a breather every -- * now and then to ensure that the issue has a chance to add -- * the poll to the issued list. Otherwise we can spin here -- * forever, while the workqueue is stuck trying to acquire the -- * very same mutex. -- */ -- if (!(++iters & 7)) { -- mutex_unlock(&ctx->uring_lock); -- io_run_task_work(); -- mutex_lock(&ctx->uring_lock); -- } -- -- ret = io_iopoll_getevents(ctx, &nr_events, min); -- if (ret <= 0) -- break; -- ret = 0; -- } while (min && !nr_events && !need_resched()); -- -- mutex_unlock(&ctx->uring_lock); -- return ret; --} -- --static void kiocb_end_write(struct io_kiocb *req) --{ -- /* -- * Tell lockdep we inherited freeze protection from submission -- * thread. -- */ -- if (req->flags & REQ_F_ISREG) { -- struct inode *inode = file_inode(req->file); -- -- __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE); -- } -- file_end_write(req->file); --} -- --static void io_complete_rw_common(struct kiocb *kiocb, long res, -- struct io_comp_state *cs) --{ -- struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); -- int cflags = 0; -- -- if (kiocb->ki_flags & IOCB_WRITE) -- kiocb_end_write(req); -- -- if (res != req->result) -- req_set_fail_links(req); -- if (req->flags & REQ_F_BUFFER_SELECTED) -- cflags = io_put_rw_kbuf(req); -- __io_req_complete(req, res, cflags, cs); --} -- --#ifdef CONFIG_BLOCK --static bool io_resubmit_prep(struct io_kiocb *req, int error) --{ -- req_set_fail_links(req); -- return false; --} --#endif -- --static bool io_rw_reissue(struct io_kiocb *req, long res) --{ --#ifdef CONFIG_BLOCK -- umode_t mode = file_inode(req->file)->i_mode; -- int ret; -- -- if (!S_ISBLK(mode) && !S_ISREG(mode)) -- return false; -- if ((res != -EAGAIN && res != -EOPNOTSUPP) || io_wq_current_is_worker()) -- return false; -- /* -- * If ref is dying, we might be running poll reap from the exit work. -- * Don't attempt to reissue from that path, just let it fail with -- * -EAGAIN. -- */ -- if (percpu_ref_is_dying(&req->ctx->refs)) -- return false; -- -- ret = io_sq_thread_acquire_mm(req->ctx, req); -- -- if (io_resubmit_prep(req, ret)) { -- refcount_inc(&req->refs); -- io_queue_async_work(req); -- return true; -- } -- --#endif -- return false; --} -- --static void __io_complete_rw(struct io_kiocb *req, long res, long res2, -- struct io_comp_state *cs) --{ -- if (!io_rw_reissue(req, res)) -- io_complete_rw_common(&req->rw.kiocb, res, cs); --} -- --static void io_complete_rw(struct kiocb *kiocb, long res, long res2) --{ -- struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); -- -- __io_complete_rw(req, res, res2, NULL); --} -- --static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) --{ -- struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); -- -- if (kiocb->ki_flags & IOCB_WRITE) -- kiocb_end_write(req); -- -- if (res != -EAGAIN && res != req->result) -- req_set_fail_links(req); -- -- WRITE_ONCE(req->result, res); -- /* order with io_poll_complete() checking ->result */ -- smp_wmb(); -- WRITE_ONCE(req->iopoll_completed, 1); --} -- --/* -- * After the iocb has been issued, it's safe to be found on the poll list. -- * Adding the kiocb to the list AFTER submission ensures that we don't -- * find it from a io_iopoll_getevents() thread before the issuer is done -- * accessing the kiocb cookie. -- */ --static void io_iopoll_req_issued(struct io_kiocb *req) --{ -- struct io_ring_ctx *ctx = req->ctx; -- -- /* -- * Track whether we have multiple files in our lists. This will impact -- * how we do polling eventually, not spinning if we're on potentially -- * different devices. -- */ -- if (list_empty(&ctx->iopoll_list)) { -- ctx->poll_multi_file = false; -- } else if (!ctx->poll_multi_file) { -- struct io_kiocb *list_req; -- -- list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb, -- iopoll_entry); -- if (list_req->file != req->file) -- ctx->poll_multi_file = true; -- } -- -- /* -- * For fast devices, IO may have already completed. If it has, add -- * it to the front so we find it first. -- */ -- if (READ_ONCE(req->iopoll_completed)) -- list_add(&req->iopoll_entry, &ctx->iopoll_list); -- else -- list_add_tail(&req->iopoll_entry, &ctx->iopoll_list); -- -- if ((ctx->flags & IORING_SETUP_SQPOLL) && -- wq_has_sleeper(&ctx->sq_data->wait)) -- wake_up(&ctx->sq_data->wait); --} -- --static void __io_state_file_put(struct io_submit_state *state) --{ -- if (state->has_refs) -- fput_many(state->file, state->has_refs); -- state->file = NULL; --} -- --static inline void io_state_file_put(struct io_submit_state *state) --{ -- if (state->file) -- __io_state_file_put(state); --} -- --/* -- * Get as many references to a file as we have IOs left in this submission, -- * assuming most submissions are for one file, or at least that each file -- * has more than one submission. -- */ --static struct file *__io_file_get(struct io_submit_state *state, int fd) --{ -- if (!state) -- return fget(fd); -- -- if (state->file) { -- if (state->fd == fd) { -- state->has_refs--; -- return state->file; -- } -- __io_state_file_put(state); -- } -- state->file = fget_many(fd, state->ios_left); -- if (!state->file) -- return NULL; -- -- state->fd = fd; -- state->has_refs = state->ios_left - 1; -- return state->file; --} -- --static bool io_bdev_nowait(struct block_device *bdev) --{ --#ifdef CONFIG_BLOCK -- return !bdev || blk_queue_nowait(bdev_get_queue(bdev)); --#else -- return true; --#endif --} -- --/* -- * If we tracked the file through the SCM inflight mechanism, we could support -- * any file. For now, just ensure that anything potentially problematic is done -- * inline. -- */ --static bool io_file_supports_async(struct file *file, int rw) --{ -- umode_t mode = file_inode(file)->i_mode; -- -- if (S_ISBLK(mode)) { -- if (io_bdev_nowait(file->f_inode->i_bdev)) -- return true; -- return false; -- } -- if (S_ISSOCK(mode)) -- return true; -- if (S_ISREG(mode)) { -- if (io_bdev_nowait(file->f_inode->i_sb->s_bdev) && -- file->f_op != &io_uring_fops) -- return true; -- return false; -- } -- -- /* any ->read/write should understand O_NONBLOCK */ -- if (file->f_flags & O_NONBLOCK) -- return true; -- -- if (!(file->f_mode & FMODE_NOWAIT)) -- return false; -- -- if (rw == READ) -- return file->f_op->read_iter != NULL; -- -- return file->f_op->write_iter != NULL; --} -- --static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- struct io_ring_ctx *ctx = req->ctx; -- struct kiocb *kiocb = &req->rw.kiocb; -- unsigned ioprio; -- int ret; -- -- if (S_ISREG(file_inode(req->file)->i_mode)) -- req->flags |= REQ_F_ISREG; -- -- kiocb->ki_pos = READ_ONCE(sqe->off); -- if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) { -- req->flags |= REQ_F_CUR_POS; -- kiocb->ki_pos = req->file->f_pos; -- } -- kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp)); -- kiocb->ki_flags = iocb_flags(kiocb->ki_filp); -- ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags)); -- if (unlikely(ret)) -- return ret; -- -- ioprio = READ_ONCE(sqe->ioprio); -- if (ioprio) { -- ret = ioprio_check_cap(ioprio); -- if (ret) -- return ret; -- -- kiocb->ki_ioprio = ioprio; -- } else -- kiocb->ki_ioprio = get_current_ioprio(); -- -- /* don't allow async punt if RWF_NOWAIT was requested */ -- if (kiocb->ki_flags & IOCB_NOWAIT) -- req->flags |= REQ_F_NOWAIT; -- -- if (ctx->flags & IORING_SETUP_IOPOLL) { -- if (!(kiocb->ki_flags & IOCB_DIRECT) || -- !kiocb->ki_filp->f_op->iopoll) -- return -EOPNOTSUPP; -- -- kiocb->ki_flags |= IOCB_HIPRI; -- kiocb->ki_complete = io_complete_rw_iopoll; -- req->iopoll_completed = 0; -- } else { -- if (kiocb->ki_flags & IOCB_HIPRI) -- return -EINVAL; -- kiocb->ki_complete = io_complete_rw; -- } -- -- req->rw.addr = READ_ONCE(sqe->addr); -- req->rw.len = READ_ONCE(sqe->len); -- req->buf_index = READ_ONCE(sqe->buf_index); -- return 0; --} -- --static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) --{ -- switch (ret) { -- case -EIOCBQUEUED: -- break; -- case -ERESTARTSYS: -- case -ERESTARTNOINTR: -- case -ERESTARTNOHAND: -- case -ERESTART_RESTARTBLOCK: -- /* -- * We can't just restart the syscall, since previously -- * submitted sqes may already be in progress. Just fail this -- * IO with EINTR. -- */ -- ret = -EINTR; -- fallthrough; -- default: -- kiocb->ki_complete(kiocb, ret, 0); -- } --} -- --static void kiocb_done(struct kiocb *kiocb, ssize_t ret, -- struct io_comp_state *cs) --{ -- struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); -- struct io_async_rw *io = req->async_data; -- -- /* add previously done IO, if any */ -- if (io && io->bytes_done > 0) { -- if (ret < 0) -- ret = io->bytes_done; -- else -- ret += io->bytes_done; -- } -- -- if (req->flags & REQ_F_CUR_POS) -- req->file->f_pos = kiocb->ki_pos; -- if (ret >= 0 && kiocb->ki_complete == io_complete_rw) -- __io_complete_rw(req, ret, 0, cs); -- else -- io_rw_done(kiocb, ret); --} -- --static ssize_t io_import_fixed(struct io_kiocb *req, int rw, -- struct iov_iter *iter) --{ -- struct io_ring_ctx *ctx = req->ctx; -- size_t len = req->rw.len; -- struct io_mapped_ubuf *imu; -- u16 index, buf_index = req->buf_index; -- size_t offset; -- u64 buf_addr; -- -- if (unlikely(buf_index >= ctx->nr_user_bufs)) -- return -EFAULT; -- index = array_index_nospec(buf_index, ctx->nr_user_bufs); -- imu = &ctx->user_bufs[index]; -- buf_addr = req->rw.addr; -- -- /* overflow */ -- if (buf_addr + len < buf_addr) -- return -EFAULT; -- /* not inside the mapped region */ -- if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len) -- return -EFAULT; -- -- /* -- * May not be a start of buffer, set size appropriately -- * and advance us to the beginning. -- */ -- offset = buf_addr - imu->ubuf; -- iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len); -- -- if (offset) { -- /* -- * Don't use iov_iter_advance() here, as it's really slow for -- * using the latter parts of a big fixed buffer - it iterates -- * over each segment manually. We can cheat a bit here, because -- * we know that: -- * -- * 1) it's a BVEC iter, we set it up -- * 2) all bvecs are PAGE_SIZE in size, except potentially the -- * first and last bvec -- * -- * So just find our index, and adjust the iterator afterwards. -- * If the offset is within the first bvec (or the whole first -- * bvec, just use iov_iter_advance(). This makes it easier -- * since we can just skip the first segment, which may not -- * be PAGE_SIZE aligned. -- */ -- const struct bio_vec *bvec = imu->bvec; -- -- if (offset <= bvec->bv_len) { -- iov_iter_advance(iter, offset); -- } else { -- unsigned long seg_skip; -- -- /* skip first vec */ -- offset -= bvec->bv_len; -- seg_skip = 1 + (offset >> PAGE_SHIFT); -- -- iter->bvec = bvec + seg_skip; -- iter->nr_segs -= seg_skip; -- iter->count -= bvec->bv_len + offset; -- iter->iov_offset = offset & ~PAGE_MASK; -- } -- } -- -- return len; --} -- --static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock) --{ -- if (needs_lock) -- mutex_unlock(&ctx->uring_lock); --} -- --static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock) --{ -- /* -- * "Normal" inline submissions always hold the uring_lock, since we -- * grab it from the system call. Same is true for the SQPOLL offload. -- * The only exception is when we've detached the request and issue it -- * from an async worker thread, grab the lock for that case. -- */ -- if (needs_lock) -- mutex_lock(&ctx->uring_lock); --} -- --static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len, -- int bgid, struct io_buffer *kbuf, -- bool needs_lock) --{ -- struct io_buffer *head; -- -- if (req->flags & REQ_F_BUFFER_SELECTED) -- return kbuf; -- -- io_ring_submit_lock(req->ctx, needs_lock); -- -- lockdep_assert_held(&req->ctx->uring_lock); -- -- head = xa_load(&req->ctx->io_buffers, bgid); -- if (head) { -- if (!list_empty(&head->list)) { -- kbuf = list_last_entry(&head->list, struct io_buffer, -- list); -- list_del(&kbuf->list); -- } else { -- kbuf = head; -- xa_erase(&req->ctx->io_buffers, bgid); -- } -- if (*len > kbuf->len) -- *len = kbuf->len; -- } else { -- kbuf = ERR_PTR(-ENOBUFS); -- } -- -- io_ring_submit_unlock(req->ctx, needs_lock); -- -- return kbuf; --} -- --static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len, -- bool needs_lock) --{ -- struct io_buffer *kbuf; -- u16 bgid; -- -- kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; -- bgid = req->buf_index; -- kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock); -- if (IS_ERR(kbuf)) -- return kbuf; -- req->rw.addr = (u64) (unsigned long) kbuf; -- req->flags |= REQ_F_BUFFER_SELECTED; -- return u64_to_user_ptr(kbuf->addr); --} -- --#ifdef CONFIG_COMPAT --static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, -- bool needs_lock) --{ -- struct compat_iovec __user *uiov; -- compat_ssize_t clen; -- void __user *buf; -- ssize_t len; -- -- uiov = u64_to_user_ptr(req->rw.addr); -- if (!access_ok(uiov, sizeof(*uiov))) -- return -EFAULT; -- if (__get_user(clen, &uiov->iov_len)) -- return -EFAULT; -- if (clen < 0) -- return -EINVAL; -- -- len = clen; -- buf = io_rw_buffer_select(req, &len, needs_lock); -- if (IS_ERR(buf)) -- return PTR_ERR(buf); -- iov[0].iov_base = buf; -- iov[0].iov_len = (compat_size_t) len; -- return 0; --} --#endif -- --static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, -- bool needs_lock) --{ -- struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr); -- void __user *buf; -- ssize_t len; -- -- if (copy_from_user(iov, uiov, sizeof(*uiov))) -- return -EFAULT; -- -- len = iov[0].iov_len; -- if (len < 0) -- return -EINVAL; -- buf = io_rw_buffer_select(req, &len, needs_lock); -- if (IS_ERR(buf)) -- return PTR_ERR(buf); -- iov[0].iov_base = buf; -- iov[0].iov_len = len; -- return 0; --} -- --static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, -- bool needs_lock) --{ -- if (req->flags & REQ_F_BUFFER_SELECTED) { -- struct io_buffer *kbuf; -- -- kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; -- iov[0].iov_base = u64_to_user_ptr(kbuf->addr); -- iov[0].iov_len = kbuf->len; -- return 0; -- } -- if (req->rw.len != 1) -- return -EINVAL; -- --#ifdef CONFIG_COMPAT -- if (req->ctx->compat) -- return io_compat_import(req, iov, needs_lock); --#endif -- -- return __io_iov_buffer_select(req, iov, needs_lock); --} -- --static ssize_t __io_import_iovec(int rw, struct io_kiocb *req, -- struct iovec **iovec, struct iov_iter *iter, -- bool needs_lock) --{ -- void __user *buf = u64_to_user_ptr(req->rw.addr); -- size_t sqe_len = req->rw.len; -- ssize_t ret; -- u8 opcode; -- -- opcode = req->opcode; -- if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) { -- *iovec = NULL; -- return io_import_fixed(req, rw, iter); -- } -- -- /* buffer index only valid with fixed read/write, or buffer select */ -- if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT)) -- return -EINVAL; -- -- if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) { -- if (req->flags & REQ_F_BUFFER_SELECT) { -- buf = io_rw_buffer_select(req, &sqe_len, needs_lock); -- if (IS_ERR(buf)) -- return PTR_ERR(buf); -- req->rw.len = sqe_len; -- } -- -- ret = import_single_range(rw, buf, sqe_len, *iovec, iter); -- *iovec = NULL; -- return ret; -- } -- -- if (req->flags & REQ_F_BUFFER_SELECT) { -- ret = io_iov_buffer_select(req, *iovec, needs_lock); -- if (!ret) { -- ret = (*iovec)->iov_len; -- iov_iter_init(iter, rw, *iovec, 1, ret); -- } -- *iovec = NULL; -- return ret; -- } -- -- return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter, -- req->ctx->compat); --} -- --static ssize_t io_import_iovec(int rw, struct io_kiocb *req, -- struct iovec **iovec, struct iov_iter *iter, -- bool needs_lock) --{ -- struct io_async_rw *iorw = req->async_data; -- -- if (!iorw) -- return __io_import_iovec(rw, req, iovec, iter, needs_lock); -- *iovec = NULL; -- return 0; --} -- --static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb) --{ -- return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos; --} -- --/* -- * For files that don't have ->read_iter() and ->write_iter(), handle them -- * by looping over ->read() or ->write() manually. -- */ --static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) --{ -- struct kiocb *kiocb = &req->rw.kiocb; -- struct file *file = req->file; -- ssize_t ret = 0; -- -- /* -- * Don't support polled IO through this interface, and we can't -- * support non-blocking either. For the latter, this just causes -- * the kiocb to be handled from an async context. -- */ -- if (kiocb->ki_flags & IOCB_HIPRI) -- return -EOPNOTSUPP; -- if (kiocb->ki_flags & IOCB_NOWAIT) -- return -EAGAIN; -- -- while (iov_iter_count(iter)) { -- struct iovec iovec; -- ssize_t nr; -- -- if (!iov_iter_is_bvec(iter)) { -- iovec = iov_iter_iovec(iter); -- } else { -- iovec.iov_base = u64_to_user_ptr(req->rw.addr); -- iovec.iov_len = req->rw.len; -- } -- -- if (rw == READ) { -- nr = file->f_op->read(file, iovec.iov_base, -- iovec.iov_len, io_kiocb_ppos(kiocb)); -- } else { -- nr = file->f_op->write(file, iovec.iov_base, -- iovec.iov_len, io_kiocb_ppos(kiocb)); -- } -- -- if (nr < 0) { -- if (!ret) -- ret = nr; -- break; -- } -- ret += nr; -- if (!iov_iter_is_bvec(iter)) { -- iov_iter_advance(iter, nr); -- } else { -- req->rw.addr += nr; -- req->rw.len -= nr; -- if (!req->rw.len) -- break; -- } -- if (nr != iovec.iov_len) -- break; -- } -- -- return ret; --} -- --static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, -- const struct iovec *fast_iov, struct iov_iter *iter) --{ -- struct io_async_rw *rw = req->async_data; -- -- memcpy(&rw->iter, iter, sizeof(*iter)); -- rw->free_iovec = iovec; -- rw->bytes_done = 0; -- /* can only be fixed buffers, no need to do anything */ -- if (iov_iter_is_bvec(iter)) -- return; -- if (!iovec) { -- unsigned iov_off = 0; -- -- rw->iter.iov = rw->fast_iov; -- if (iter->iov != fast_iov) { -- iov_off = iter->iov - fast_iov; -- rw->iter.iov += iov_off; -- } -- if (rw->fast_iov != fast_iov) -- memcpy(rw->fast_iov + iov_off, fast_iov + iov_off, -- sizeof(struct iovec) * iter->nr_segs); -- } else { -- req->flags |= REQ_F_NEED_CLEANUP; -- } --} -- --static inline int __io_alloc_async_data(struct io_kiocb *req) --{ -- WARN_ON_ONCE(!io_op_defs[req->opcode].async_size); -- req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL); -- return req->async_data == NULL; --} -- --static int io_alloc_async_data(struct io_kiocb *req) --{ -- if (!io_op_defs[req->opcode].needs_async_data) -- return 0; -- -- return __io_alloc_async_data(req); --} -- --static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, -- const struct iovec *fast_iov, -- struct iov_iter *iter, bool force) --{ -- if (!force && !io_op_defs[req->opcode].needs_async_data) -- return 0; -- if (!req->async_data) { -- if (__io_alloc_async_data(req)) -- return -ENOMEM; -- -- io_req_map_rw(req, iovec, fast_iov, iter); -- } -- return 0; --} -- --static inline int io_rw_prep_async(struct io_kiocb *req, int rw) --{ -- struct io_async_rw *iorw = req->async_data; -- struct iovec *iov = iorw->fast_iov; -- ssize_t ret; -- -- ret = __io_import_iovec(rw, req, &iov, &iorw->iter, false); -- if (unlikely(ret < 0)) -- return ret; -- -- iorw->bytes_done = 0; -- iorw->free_iovec = iov; -- if (iov) -- req->flags |= REQ_F_NEED_CLEANUP; -- return 0; --} -- --static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- ssize_t ret; -- -- ret = io_prep_rw(req, sqe); -- if (ret) -- return ret; -- -- if (unlikely(!(req->file->f_mode & FMODE_READ))) -- return -EBADF; -- -- /* either don't need iovec imported or already have it */ -- if (!req->async_data) -- return 0; -- return io_rw_prep_async(req, READ); --} -- --/* -- * This is our waitqueue callback handler, registered through lock_page_async() -- * when we initially tried to do the IO with the iocb armed our waitqueue. -- * This gets called when the page is unlocked, and we generally expect that to -- * happen when the page IO is completed and the page is now uptodate. This will -- * queue a task_work based retry of the operation, attempting to copy the data -- * again. If the latter fails because the page was NOT uptodate, then we will -- * do a thread based blocking retry of the operation. That's the unexpected -- * slow path. -- */ --static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, -- int sync, void *arg) --{ -- struct wait_page_queue *wpq; -- struct io_kiocb *req = wait->private; -- struct wait_page_key *key = arg; -- int ret; -- -- wpq = container_of(wait, struct wait_page_queue, wait); -- -- if (!wake_page_match(wpq, key)) -- return 0; -- -- req->rw.kiocb.ki_flags &= ~IOCB_WAITQ; -- list_del_init(&wait->entry); -- -- init_task_work(&req->task_work, io_req_task_submit); -- percpu_ref_get(&req->ctx->refs); -- -- /* submit ref gets dropped, acquire a new one */ -- refcount_inc(&req->refs); -- ret = io_req_task_work_add(req, true); -- if (unlikely(ret)) { -- struct task_struct *tsk; -- -- /* queue just for cancelation */ -- init_task_work(&req->task_work, io_req_task_cancel); -- tsk = io_wq_get_task(req->ctx->io_wq); -- task_work_add(tsk, &req->task_work, TWA_NONE); -- wake_up_process(tsk); -- } -- return 1; --} -- --/* -- * This controls whether a given IO request should be armed for async page -- * based retry. If we return false here, the request is handed to the async -- * worker threads for retry. If we're doing buffered reads on a regular file, -- * we prepare a private wait_page_queue entry and retry the operation. This -- * will either succeed because the page is now uptodate and unlocked, or it -- * will register a callback when the page is unlocked at IO completion. Through -- * that callback, io_uring uses task_work to setup a retry of the operation. -- * That retry will attempt the buffered read again. The retry will generally -- * succeed, or in rare cases where it fails, we then fall back to using the -- * async worker threads for a blocking retry. -- */ --static bool io_rw_should_retry(struct io_kiocb *req) --{ -- struct io_async_rw *rw = req->async_data; -- struct wait_page_queue *wait = &rw->wpq; -- struct kiocb *kiocb = &req->rw.kiocb; -- -- /* never retry for NOWAIT, we just complete with -EAGAIN */ -- if (req->flags & REQ_F_NOWAIT) -- return false; -- -- /* Only for buffered IO */ -- if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI)) -- return false; -- -- /* -- * just use poll if we can, and don't attempt if the fs doesn't -- * support callback based unlocks -- */ -- if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC)) -- return false; -- -- wait->wait.func = io_async_buf_func; -- wait->wait.private = req; -- wait->wait.flags = 0; -- INIT_LIST_HEAD(&wait->wait.entry); -- kiocb->ki_flags |= IOCB_WAITQ; -- kiocb->ki_flags &= ~IOCB_NOWAIT; -- kiocb->ki_waitq = wait; -- return true; --} -- --static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter) --{ -- if (req->file->f_op->read_iter) -- return call_read_iter(req->file, &req->rw.kiocb, iter); -- else if (req->file->f_op->read) -- return loop_rw_iter(READ, req, iter); -- else -- return -EINVAL; --} -- --static int io_read(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; -- struct kiocb *kiocb = &req->rw.kiocb; -- struct iov_iter __iter, *iter = &__iter; -- struct iov_iter iter_cp; -- struct io_async_rw *rw = req->async_data; -- ssize_t io_size, ret, ret2; -- bool no_async; -- -- if (rw) -- iter = &rw->iter; -- -- ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock); -- if (ret < 0) -- return ret; -- iter_cp = *iter; -- io_size = iov_iter_count(iter); -- req->result = io_size; -- ret = 0; -- -- /* Ensure we clear previously set non-block flag */ -- if (!force_nonblock) -- kiocb->ki_flags &= ~IOCB_NOWAIT; -- else -- kiocb->ki_flags |= IOCB_NOWAIT; -- -- -- /* If the file doesn't support async, just async punt */ -- no_async = force_nonblock && !io_file_supports_async(req->file, READ); -- if (no_async) -- goto copy_iov; -- -- ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size); -- if (unlikely(ret)) -- goto out_free; -- -- ret = io_iter_do_read(req, iter); -- -- if (!ret) { -- goto done; -- } else if (ret == -EIOCBQUEUED) { -- ret = 0; -- goto out_free; -- } else if (ret == -EAGAIN) { -- /* IOPOLL retry should happen for io-wq threads */ -- if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) -- goto done; -- /* no retry on NONBLOCK marked file */ -- if (req->file->f_flags & O_NONBLOCK) -- goto done; -- /* some cases will consume bytes even on error returns */ -- *iter = iter_cp; -- ret = 0; -- goto copy_iov; -- } else if (ret < 0) { -- /* make sure -ERESTARTSYS -> -EINTR is done */ -- goto done; -- } -- -- /* read it all, or we did blocking attempt. no retry. */ -- if (!iov_iter_count(iter) || !force_nonblock || -- (req->file->f_flags & O_NONBLOCK) || !(req->flags & REQ_F_ISREG)) -- goto done; -- -- io_size -= ret; --copy_iov: -- ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true); -- if (ret2) { -- ret = ret2; -- goto out_free; -- } -- if (no_async) -- return -EAGAIN; -- rw = req->async_data; -- /* it's copied and will be cleaned with ->io */ -- iovec = NULL; -- /* now use our persistent iterator, if we aren't already */ -- iter = &rw->iter; --retry: -- rw->bytes_done += ret; -- /* if we can retry, do so with the callbacks armed */ -- if (!io_rw_should_retry(req)) { -- kiocb->ki_flags &= ~IOCB_WAITQ; -- return -EAGAIN; -- } -- -- /* -- * Now retry read with the IOCB_WAITQ parts set in the iocb. If we -- * get -EIOCBQUEUED, then we'll get a notification when the desired -- * page gets unlocked. We can also get a partial read here, and if we -- * do, then just retry at the new offset. -- */ -- ret = io_iter_do_read(req, iter); -- if (ret == -EIOCBQUEUED) { -- ret = 0; -- goto out_free; -- } else if (ret > 0 && ret < io_size) { -- /* we got some bytes, but not all. retry. */ -- kiocb->ki_flags &= ~IOCB_WAITQ; -- goto retry; -- } --done: -- kiocb_done(kiocb, ret, cs); -- ret = 0; --out_free: -- /* it's reportedly faster than delegating the null check to kfree() */ -- if (iovec) -- kfree(iovec); -- return ret; --} -- --static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- ssize_t ret; -- -- ret = io_prep_rw(req, sqe); -- if (ret) -- return ret; -- -- if (unlikely(!(req->file->f_mode & FMODE_WRITE))) -- return -EBADF; -- -- /* either don't need iovec imported or already have it */ -- if (!req->async_data) -- return 0; -- return io_rw_prep_async(req, WRITE); --} -- --static int io_write(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; -- struct kiocb *kiocb = &req->rw.kiocb; -- struct iov_iter __iter, *iter = &__iter; -- struct iov_iter iter_cp; -- struct io_async_rw *rw = req->async_data; -- ssize_t ret, ret2, io_size; -- -- if (rw) -- iter = &rw->iter; -- -- ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock); -- if (ret < 0) -- return ret; -- iter_cp = *iter; -- io_size = iov_iter_count(iter); -- req->result = io_size; -- -- /* Ensure we clear previously set non-block flag */ -- if (!force_nonblock) -- kiocb->ki_flags &= ~IOCB_NOWAIT; -- else -- kiocb->ki_flags |= IOCB_NOWAIT; -- -- /* If the file doesn't support async, just async punt */ -- if (force_nonblock && !io_file_supports_async(req->file, WRITE)) -- goto copy_iov; -- -- /* file path doesn't support NOWAIT for non-direct_IO */ -- if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) && -- (req->flags & REQ_F_ISREG)) -- goto copy_iov; -- -- ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), io_size); -- if (unlikely(ret)) -- goto out_free; -- -- /* -- * Open-code file_start_write here to grab freeze protection, -- * which will be released by another thread in -- * io_complete_rw(). Fool lockdep by telling it the lock got -- * released so that it doesn't complain about the held lock when -- * we return to userspace. -- */ -- if (req->flags & REQ_F_ISREG) { -- sb_start_write(file_inode(req->file)->i_sb); -- __sb_writers_release(file_inode(req->file)->i_sb, -- SB_FREEZE_WRITE); -- } -- kiocb->ki_flags |= IOCB_WRITE; -- -- if (req->file->f_op->write_iter) -- ret2 = call_write_iter(req->file, kiocb, iter); -- else if (req->file->f_op->write) -- ret2 = loop_rw_iter(WRITE, req, iter); -- else -- ret2 = -EINVAL; -- -- /* -- * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just -- * retry them without IOCB_NOWAIT. -- */ -- if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) -- ret2 = -EAGAIN; -- /* no retry on NONBLOCK marked file */ -- if (ret2 == -EAGAIN && (req->file->f_flags & O_NONBLOCK)) -- goto done; -- if (!force_nonblock || ret2 != -EAGAIN) { -- /* IOPOLL retry should happen for io-wq threads */ -- if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN) -- goto copy_iov; --done: -- kiocb_done(kiocb, ret2, cs); -- } else { --copy_iov: -- /* some cases will consume bytes even on error returns */ -- *iter = iter_cp; -- ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false); -- if (!ret) -- return -EAGAIN; -- } --out_free: -- /* it's reportedly faster than delegating the null check to kfree() */ -- if (iovec) -- kfree(iovec); -- return ret; --} -- --static int __io_splice_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ -- struct io_splice* sp = &req->splice; -- unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL; -- -- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- -- sp->file_in = NULL; -- sp->len = READ_ONCE(sqe->len); -- sp->flags = READ_ONCE(sqe->splice_flags); -- -- if (unlikely(sp->flags & ~valid_flags)) -- return -EINVAL; -- -- sp->file_in = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in), -- (sp->flags & SPLICE_F_FD_IN_FIXED)); -- if (!sp->file_in) -- return -EBADF; -- req->flags |= REQ_F_NEED_CLEANUP; -- -- if (!S_ISREG(file_inode(sp->file_in)->i_mode)) { -- /* -- * Splice operation will be punted aync, and here need to -- * modify io_wq_work.flags, so initialize io_wq_work firstly. -- */ -- io_req_init_async(req); -- req->work.flags |= IO_WQ_WORK_UNBOUND; -- } -- -- return 0; --} -- --static int io_tee_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ -- if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off)) -- return -EINVAL; -- return __io_splice_prep(req, sqe); --} -- --static int io_tee(struct io_kiocb *req, bool force_nonblock) --{ -- struct io_splice *sp = &req->splice; -- struct file *in = sp->file_in; -- struct file *out = sp->file_out; -- unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; -- long ret = 0; -- -- if (force_nonblock) -- return -EAGAIN; -- if (sp->len) -- ret = do_tee(in, out, sp->len, flags); -- -- io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED)); -- req->flags &= ~REQ_F_NEED_CLEANUP; -- -- if (ret != sp->len) -- req_set_fail_links(req); -- io_req_complete(req, ret); -- return 0; --} -- --static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- struct io_splice* sp = &req->splice; -- -- sp->off_in = READ_ONCE(sqe->splice_off_in); -- sp->off_out = READ_ONCE(sqe->off); -- return __io_splice_prep(req, sqe); --} -- --static int io_splice(struct io_kiocb *req, bool force_nonblock) --{ -- struct io_splice *sp = &req->splice; -- struct file *in = sp->file_in; -- struct file *out = sp->file_out; -- unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; -- loff_t *poff_in, *poff_out; -- long ret = 0; -- -- if (force_nonblock) -- return -EAGAIN; -- -- poff_in = (sp->off_in == -1) ? NULL : &sp->off_in; -- poff_out = (sp->off_out == -1) ? NULL : &sp->off_out; -- -- if (sp->len) -- ret = do_splice(in, poff_in, out, poff_out, sp->len, flags); -- -- io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED)); -- req->flags &= ~REQ_F_NEED_CLEANUP; -- -- if (ret != sp->len) -- req_set_fail_links(req); -- io_req_complete(req, ret); -- return 0; --} -- --/* -- * IORING_OP_NOP just posts a completion event, nothing else. -- */ --static int io_nop(struct io_kiocb *req, struct io_comp_state *cs) --{ -- struct io_ring_ctx *ctx = req->ctx; -- -- if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- -- __io_req_complete(req, 0, 0, cs); -- return 0; --} -- --static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- struct io_ring_ctx *ctx = req->ctx; -- -- if (!req->file) -- return -EBADF; -- -- if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index || -- sqe->splice_fd_in)) -- return -EINVAL; -- -- req->sync.flags = READ_ONCE(sqe->fsync_flags); -- if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC)) -- return -EINVAL; -- -- req->sync.off = READ_ONCE(sqe->off); -- req->sync.len = READ_ONCE(sqe->len); -- return 0; --} -- --static int io_fsync(struct io_kiocb *req, bool force_nonblock) --{ -- loff_t end = req->sync.off + req->sync.len; -- int ret; -- -- /* fsync always requires a blocking context */ -- if (force_nonblock) -- return -EAGAIN; -- -- ret = vfs_fsync_range(req->file, req->sync.off, -- end > 0 ? end : LLONG_MAX, -- req->sync.flags & IORING_FSYNC_DATASYNC); -- if (ret < 0) -- req_set_fail_links(req); -- io_req_complete(req, ret); -- return 0; --} -- --static int io_fallocate_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ -- if (sqe->ioprio || sqe->buf_index || sqe->rw_flags || -- sqe->splice_fd_in) -- return -EINVAL; -- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- -- req->sync.off = READ_ONCE(sqe->off); -- req->sync.len = READ_ONCE(sqe->addr); -- req->sync.mode = READ_ONCE(sqe->len); -- return 0; --} -- --static int io_fallocate(struct io_kiocb *req, bool force_nonblock) --{ -- int ret; -- -- /* fallocate always requiring blocking context */ -- if (force_nonblock) -- return -EAGAIN; -- ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off, -- req->sync.len); -- if (ret < 0) -- req_set_fail_links(req); -- io_req_complete(req, ret); -- return 0; --} -- --static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- const char __user *fname; -- int ret; -- -- if (unlikely(sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)) -- return -EINVAL; -- if (unlikely(req->flags & REQ_F_FIXED_FILE)) -- return -EBADF; -- -- /* open.how should be already initialised */ -- if (!(req->open.how.flags & O_PATH) && force_o_largefile()) -- req->open.how.flags |= O_LARGEFILE; -- -- req->open.dfd = READ_ONCE(sqe->fd); -- fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); -- req->open.filename = getname(fname); -- if (IS_ERR(req->open.filename)) { -- ret = PTR_ERR(req->open.filename); -- req->open.filename = NULL; -- return ret; -- } -- req->open.nofile = rlimit(RLIMIT_NOFILE); -- req->open.ignore_nonblock = false; -- req->flags |= REQ_F_NEED_CLEANUP; -- return 0; --} -- --static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- u64 flags, mode; -- -- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL))) -- return -EINVAL; -- mode = READ_ONCE(sqe->len); -- flags = READ_ONCE(sqe->open_flags); -- req->open.how = build_open_how(flags, mode); -- return __io_openat_prep(req, sqe); --} -- --static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- struct open_how __user *how; -- size_t len; -- int ret; -- -- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL))) -- return -EINVAL; -- how = u64_to_user_ptr(READ_ONCE(sqe->addr2)); -- len = READ_ONCE(sqe->len); -- if (len < OPEN_HOW_SIZE_VER0) -- return -EINVAL; -- -- ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how, -- len); -- if (ret) -- return ret; -- -- return __io_openat_prep(req, sqe); --} -- --static int io_openat2(struct io_kiocb *req, bool force_nonblock) --{ -- struct open_flags op; -- struct file *file; -- int ret; -- -- if (force_nonblock && !req->open.ignore_nonblock) -- return -EAGAIN; -- -- ret = build_open_flags(&req->open.how, &op); -- if (ret) -- goto err; -- -- ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile); -- if (ret < 0) -- goto err; -- -- file = do_filp_open(req->open.dfd, req->open.filename, &op); -- if (IS_ERR(file)) { -- put_unused_fd(ret); -- ret = PTR_ERR(file); -- /* -- * A work-around to ensure that /proc/self works that way -- * that it should - if we get -EOPNOTSUPP back, then assume -- * that proc_self_get_link() failed us because we're in async -- * context. We should be safe to retry this from the task -- * itself with force_nonblock == false set, as it should not -- * block on lookup. Would be nice to know this upfront and -- * avoid the async dance, but doesn't seem feasible. -- */ -- if (ret == -EOPNOTSUPP && io_wq_current_is_worker()) { -- req->open.ignore_nonblock = true; -- refcount_inc(&req->refs); -- io_req_task_queue(req); -- return 0; -- } -- } else { -- fsnotify_open(file); -- fd_install(ret, file); -- } --err: -- putname(req->open.filename); -- req->flags &= ~REQ_F_NEED_CLEANUP; -- if (ret < 0) -- req_set_fail_links(req); -- io_req_complete(req, ret); -- return 0; --} -- --static int io_openat(struct io_kiocb *req, bool force_nonblock) --{ -- return io_openat2(req, force_nonblock); --} -- --static int io_remove_buffers_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ -- struct io_provide_buf *p = &req->pbuf; -- u64 tmp; -- -- if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off || -- sqe->splice_fd_in) -- return -EINVAL; -- -- tmp = READ_ONCE(sqe->fd); -- if (!tmp || tmp > USHRT_MAX) -- return -EINVAL; -- -- memset(p, 0, sizeof(*p)); -- p->nbufs = tmp; -- p->bgid = READ_ONCE(sqe->buf_group); -- return 0; --} -- --static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf, -- int bgid, unsigned nbufs) --{ -- unsigned i = 0; -- -- /* shouldn't happen */ -- if (!nbufs) -- return 0; -- -- /* the head kbuf is the list itself */ -- while (!list_empty(&buf->list)) { -- struct io_buffer *nxt; -- -- nxt = list_first_entry(&buf->list, struct io_buffer, list); -- list_del(&nxt->list); -- kfree(nxt); -- if (++i == nbufs) -- return i; -- } -- i++; -- kfree(buf); -- xa_erase(&ctx->io_buffers, bgid); -- -- return i; --} -- --static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct io_provide_buf *p = &req->pbuf; -- struct io_ring_ctx *ctx = req->ctx; -- struct io_buffer *head; -- int ret = 0; -- -- io_ring_submit_lock(ctx, !force_nonblock); -- -- lockdep_assert_held(&ctx->uring_lock); -- -- ret = -ENOENT; -- head = xa_load(&ctx->io_buffers, p->bgid); -- if (head) -- ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs); -- if (ret < 0) -- req_set_fail_links(req); -- -- /* need to hold the lock to complete IOPOLL requests */ -- if (ctx->flags & IORING_SETUP_IOPOLL) { -- __io_req_complete(req, ret, 0, cs); -- io_ring_submit_unlock(ctx, !force_nonblock); -- } else { -- io_ring_submit_unlock(ctx, !force_nonblock); -- __io_req_complete(req, ret, 0, cs); -- } -- return 0; --} -- --static int io_provide_buffers_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ -- unsigned long size, tmp_check; -- struct io_provide_buf *p = &req->pbuf; -- u64 tmp; -- -- if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in) -- return -EINVAL; -- -- tmp = READ_ONCE(sqe->fd); -- if (!tmp || tmp > USHRT_MAX) -- return -E2BIG; -- p->nbufs = tmp; -- p->addr = READ_ONCE(sqe->addr); -- p->len = READ_ONCE(sqe->len); -- -- if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs, -- &size)) -- return -EOVERFLOW; -- if (check_add_overflow((unsigned long)p->addr, size, &tmp_check)) -- return -EOVERFLOW; -- -- size = (unsigned long)p->len * p->nbufs; -- if (!access_ok(u64_to_user_ptr(p->addr), size)) -- return -EFAULT; -- -- p->bgid = READ_ONCE(sqe->buf_group); -- tmp = READ_ONCE(sqe->off); -- if (tmp > USHRT_MAX) -- return -E2BIG; -- p->bid = tmp; -- return 0; --} -- --static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head) --{ -- struct io_buffer *buf; -- u64 addr = pbuf->addr; -- int i, bid = pbuf->bid; -- -- for (i = 0; i < pbuf->nbufs; i++) { -- buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT); -- if (!buf) -- break; -- -- buf->addr = addr; -- buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT); -- buf->bid = bid; -- addr += pbuf->len; -- bid++; -- if (!*head) { -- INIT_LIST_HEAD(&buf->list); -- *head = buf; -- } else { -- list_add_tail(&buf->list, &(*head)->list); -- } -- cond_resched(); -- } -- -- return i ? i : -ENOMEM; --} -- --static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct io_provide_buf *p = &req->pbuf; -- struct io_ring_ctx *ctx = req->ctx; -- struct io_buffer *head, *list; -- int ret = 0; -- -- io_ring_submit_lock(ctx, !force_nonblock); -- -- lockdep_assert_held(&ctx->uring_lock); -- -- list = head = xa_load(&ctx->io_buffers, p->bgid); -- -- ret = io_add_buffers(p, &head); -- if (ret >= 0 && !list) { -- ret = xa_insert(&ctx->io_buffers, p->bgid, head, GFP_KERNEL); -- if (ret < 0) -- __io_remove_buffers(ctx, head, p->bgid, -1U); -- } -- if (ret < 0) -- req_set_fail_links(req); -- -- /* need to hold the lock to complete IOPOLL requests */ -- if (ctx->flags & IORING_SETUP_IOPOLL) { -- __io_req_complete(req, ret, 0, cs); -- io_ring_submit_unlock(ctx, !force_nonblock); -- } else { -- io_ring_submit_unlock(ctx, !force_nonblock); -- __io_req_complete(req, ret, 0, cs); -- } -- return 0; --} -- --static int io_epoll_ctl_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ --#if defined(CONFIG_EPOLL) -- if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) -- return -EINVAL; -- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL))) -- return -EINVAL; -- -- req->epoll.epfd = READ_ONCE(sqe->fd); -- req->epoll.op = READ_ONCE(sqe->len); -- req->epoll.fd = READ_ONCE(sqe->off); -- -- if (ep_op_has_event(req->epoll.op)) { -- struct epoll_event __user *ev; -- -- ev = u64_to_user_ptr(READ_ONCE(sqe->addr)); -- if (copy_from_user(&req->epoll.event, ev, sizeof(*ev))) -- return -EFAULT; -- } -- -- return 0; --#else -- return -EOPNOTSUPP; --#endif --} -- --static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ --#if defined(CONFIG_EPOLL) -- struct io_epoll *ie = &req->epoll; -- int ret; -- -- ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock); -- if (force_nonblock && ret == -EAGAIN) -- return -EAGAIN; -- -- if (ret < 0) -- req_set_fail_links(req); -- __io_req_complete(req, ret, 0, cs); -- return 0; --#else -- return -EOPNOTSUPP; --#endif --} -- --static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ --#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) -- if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->splice_fd_in) -- return -EINVAL; -- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- -- req->madvise.addr = READ_ONCE(sqe->addr); -- req->madvise.len = READ_ONCE(sqe->len); -- req->madvise.advice = READ_ONCE(sqe->fadvise_advice); -- return 0; --#else -- return -EOPNOTSUPP; --#endif --} -- --static int io_madvise(struct io_kiocb *req, bool force_nonblock) --{ --#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) -- struct io_madvise *ma = &req->madvise; -- int ret; -- -- if (force_nonblock) -- return -EAGAIN; -- -- ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice); -- if (ret < 0) -- req_set_fail_links(req); -- io_req_complete(req, ret); -- return 0; --#else -- return -EOPNOTSUPP; --#endif --} -- --static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- if (sqe->ioprio || sqe->buf_index || sqe->addr || sqe->splice_fd_in) -- return -EINVAL; -- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- -- req->fadvise.offset = READ_ONCE(sqe->off); -- req->fadvise.len = READ_ONCE(sqe->len); -- req->fadvise.advice = READ_ONCE(sqe->fadvise_advice); -- return 0; --} -- --static int io_fadvise(struct io_kiocb *req, bool force_nonblock) --{ -- struct io_fadvise *fa = &req->fadvise; -- int ret; -- -- if (force_nonblock) { -- switch (fa->advice) { -- case POSIX_FADV_NORMAL: -- case POSIX_FADV_RANDOM: -- case POSIX_FADV_SEQUENTIAL: -- break; -- default: -- return -EAGAIN; -- } -- } -- -- ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice); -- if (ret < 0) -- req_set_fail_links(req); -- io_req_complete(req, ret); -- return 0; --} -- --static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL))) -- return -EINVAL; -- if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) -- return -EINVAL; -- if (req->flags & REQ_F_FIXED_FILE) -- return -EBADF; -- -- req->statx.dfd = READ_ONCE(sqe->fd); -- req->statx.mask = READ_ONCE(sqe->len); -- req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr)); -- req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2)); -- req->statx.flags = READ_ONCE(sqe->statx_flags); -- -- return 0; --} -- --static int io_statx(struct io_kiocb *req, bool force_nonblock) --{ -- struct io_statx *ctx = &req->statx; -- int ret; -- -- if (force_nonblock) -- return -EAGAIN; -- -- ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask, -- ctx->buffer); -- -- if (ret < 0) -- req_set_fail_links(req); -- io_req_complete(req, ret); -- return 0; --} -- --static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- /* -- * If we queue this for async, it must not be cancellable. That would -- * leave the 'file' in an undeterminate state, and here need to modify -- * io_wq_work.flags, so initialize io_wq_work firstly. -- */ -- io_req_init_async(req); -- -- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL))) -- return -EINVAL; -- if (sqe->ioprio || sqe->off || sqe->addr || sqe->len || -- sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in) -- return -EINVAL; -- if (req->flags & REQ_F_FIXED_FILE) -- return -EBADF; -- -- req->close.fd = READ_ONCE(sqe->fd); -- if ((req->file && req->file->f_op == &io_uring_fops)) -- return -EBADF; -- -- req->close.put_file = NULL; -- return 0; --} -- --static int io_close(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct io_close *close = &req->close; -- int ret; -- -- /* might be already done during nonblock submission */ -- if (!close->put_file) { -- ret = close_fd_get_file(close->fd, &close->put_file); -- if (ret < 0) -- return (ret == -ENOENT) ? -EBADF : ret; -- } -- -- /* if the file has a flush method, be safe and punt to async */ -- if (close->put_file->f_op->flush && force_nonblock) { -- /* not safe to cancel at this point */ -- req->work.flags |= IO_WQ_WORK_NO_CANCEL; -- /* was never set, but play safe */ -- req->flags &= ~REQ_F_NOWAIT; -- /* avoid grabbing files - we don't need the files */ -- req->flags |= REQ_F_NO_FILE_TABLE; -- return -EAGAIN; -- } -- -- /* No ->flush() or already async, safely close from here */ -- ret = filp_close(close->put_file, req->work.identity->files); -- if (ret < 0) -- req_set_fail_links(req); -- fput(close->put_file); -- close->put_file = NULL; -- __io_req_complete(req, ret, 0, cs); -- return 0; --} -- --static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- struct io_ring_ctx *ctx = req->ctx; -- -- if (!req->file) -- return -EBADF; -- -- if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index || -- sqe->splice_fd_in)) -- return -EINVAL; -- -- req->sync.off = READ_ONCE(sqe->off); -- req->sync.len = READ_ONCE(sqe->len); -- req->sync.flags = READ_ONCE(sqe->sync_range_flags); -- return 0; --} -- --static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock) --{ -- int ret; -- -- /* sync_file_range always requires a blocking context */ -- if (force_nonblock) -- return -EAGAIN; -- -- ret = sync_file_range(req->file, req->sync.off, req->sync.len, -- req->sync.flags); -- if (ret < 0) -- req_set_fail_links(req); -- io_req_complete(req, ret); -- return 0; --} -- --#if defined(CONFIG_NET) --static int io_setup_async_msg(struct io_kiocb *req, -- struct io_async_msghdr *kmsg) --{ -- struct io_async_msghdr *async_msg = req->async_data; -- -- if (async_msg) -- return -EAGAIN; -- if (io_alloc_async_data(req)) { -- if (kmsg->iov != kmsg->fast_iov) -- kfree(kmsg->iov); -- return -ENOMEM; -- } -- async_msg = req->async_data; -- req->flags |= REQ_F_NEED_CLEANUP; -- memcpy(async_msg, kmsg, sizeof(*kmsg)); -- return -EAGAIN; --} -- --static int io_sendmsg_copy_hdr(struct io_kiocb *req, -- struct io_async_msghdr *iomsg) --{ -- iomsg->iov = iomsg->fast_iov; -- iomsg->msg.msg_name = &iomsg->addr; -- return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg, -- req->sr_msg.msg_flags, &iomsg->iov); --} -- --static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- struct io_async_msghdr *async_msg = req->async_data; -- struct io_sr_msg *sr = &req->sr_msg; -- int ret; -- -- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- if (unlikely(sqe->addr2 || sqe->splice_fd_in || sqe->ioprio)) -- return -EINVAL; -- -- sr->msg_flags = READ_ONCE(sqe->msg_flags); -- sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); -- sr->len = READ_ONCE(sqe->len); -- --#ifdef CONFIG_COMPAT -- if (req->ctx->compat) -- sr->msg_flags |= MSG_CMSG_COMPAT; --#endif -- -- if (!async_msg || !io_op_defs[req->opcode].needs_async_data) -- return 0; -- ret = io_sendmsg_copy_hdr(req, async_msg); -- if (!ret) -- req->flags |= REQ_F_NEED_CLEANUP; -- return ret; --} -- --static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct io_async_msghdr iomsg, *kmsg; -- struct socket *sock; -- unsigned flags; -- int min_ret = 0; -- int ret; -- -- sock = sock_from_file(req->file, &ret); -- if (unlikely(!sock)) -- return ret; -- -- if (req->async_data) { -- kmsg = req->async_data; -- kmsg->msg.msg_name = &kmsg->addr; -- /* if iov is set, it's allocated already */ -- if (!kmsg->iov) -- kmsg->iov = kmsg->fast_iov; -- kmsg->msg.msg_iter.iov = kmsg->iov; -- } else { -- ret = io_sendmsg_copy_hdr(req, &iomsg); -- if (ret) -- return ret; -- kmsg = &iomsg; -- } -- -- flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; -- if (flags & MSG_DONTWAIT) -- req->flags |= REQ_F_NOWAIT; -- else if (force_nonblock) -- flags |= MSG_DONTWAIT; -- -- if (flags & MSG_WAITALL) -- min_ret = iov_iter_count(&kmsg->msg.msg_iter); -- -- ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); -- if (force_nonblock && ret == -EAGAIN) -- return io_setup_async_msg(req, kmsg); -- if (ret == -ERESTARTSYS) -- ret = -EINTR; -- -- if (kmsg->iov != kmsg->fast_iov) -- kfree(kmsg->iov); -- req->flags &= ~REQ_F_NEED_CLEANUP; -- if (ret < min_ret) -- req_set_fail_links(req); -- __io_req_complete(req, ret, 0, cs); -- return 0; --} -- --static int io_send(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct io_sr_msg *sr = &req->sr_msg; -- struct msghdr msg; -- struct iovec iov; -- struct socket *sock; -- unsigned flags; -- int min_ret = 0; -- int ret; -- -- sock = sock_from_file(req->file, &ret); -- if (unlikely(!sock)) -- return ret; -- -- ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter); -- if (unlikely(ret)) -- return ret; -- -- msg.msg_name = NULL; -- msg.msg_control = NULL; -- msg.msg_controllen = 0; -- msg.msg_namelen = 0; -- -- flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; -- if (flags & MSG_DONTWAIT) -- req->flags |= REQ_F_NOWAIT; -- else if (force_nonblock) -- flags |= MSG_DONTWAIT; -- -- if (flags & MSG_WAITALL) -- min_ret = iov_iter_count(&msg.msg_iter); -- -- msg.msg_flags = flags; -- ret = sock_sendmsg(sock, &msg); -- if (force_nonblock && ret == -EAGAIN) -- return -EAGAIN; -- if (ret == -ERESTARTSYS) -- ret = -EINTR; -- -- if (ret < min_ret) -- req_set_fail_links(req); -- __io_req_complete(req, ret, 0, cs); -- return 0; --} -- --static int __io_recvmsg_copy_hdr(struct io_kiocb *req, -- struct io_async_msghdr *iomsg) --{ -- struct io_sr_msg *sr = &req->sr_msg; -- struct iovec __user *uiov; -- size_t iov_len; -- int ret; -- -- ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg, -- &iomsg->uaddr, &uiov, &iov_len); -- if (ret) -- return ret; -- -- if (req->flags & REQ_F_BUFFER_SELECT) { -- if (iov_len > 1) -- return -EINVAL; -- if (copy_from_user(iomsg->iov, uiov, sizeof(*uiov))) -- return -EFAULT; -- sr->len = iomsg->iov[0].iov_len; -- iov_iter_init(&iomsg->msg.msg_iter, READ, iomsg->iov, 1, -- sr->len); -- iomsg->iov = NULL; -- } else { -- ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV, -- &iomsg->iov, &iomsg->msg.msg_iter, -- false); -- if (ret > 0) -- ret = 0; -- } -- -- return ret; --} -- --#ifdef CONFIG_COMPAT --static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, -- struct io_async_msghdr *iomsg) --{ -- struct compat_msghdr __user *msg_compat; -- struct io_sr_msg *sr = &req->sr_msg; -- struct compat_iovec __user *uiov; -- compat_uptr_t ptr; -- compat_size_t len; -- int ret; -- -- msg_compat = (struct compat_msghdr __user *) sr->umsg; -- ret = __get_compat_msghdr(&iomsg->msg, msg_compat, &iomsg->uaddr, -- &ptr, &len); -- if (ret) -- return ret; -- -- uiov = compat_ptr(ptr); -- if (req->flags & REQ_F_BUFFER_SELECT) { -- compat_ssize_t clen; -- -- if (len > 1) -- return -EINVAL; -- if (!access_ok(uiov, sizeof(*uiov))) -- return -EFAULT; -- if (__get_user(clen, &uiov->iov_len)) -- return -EFAULT; -- if (clen < 0) -- return -EINVAL; -- sr->len = clen; -- iomsg->iov[0].iov_len = clen; -- iomsg->iov = NULL; -- } else { -- ret = __import_iovec(READ, (struct iovec __user *)uiov, len, -- UIO_FASTIOV, &iomsg->iov, -- &iomsg->msg.msg_iter, true); -- if (ret < 0) -- return ret; -- } -- -- return 0; --} --#endif -- --static int io_recvmsg_copy_hdr(struct io_kiocb *req, -- struct io_async_msghdr *iomsg) --{ -- iomsg->msg.msg_name = &iomsg->addr; -- iomsg->iov = iomsg->fast_iov; -- --#ifdef CONFIG_COMPAT -- if (req->ctx->compat) -- return __io_compat_recvmsg_copy_hdr(req, iomsg); --#endif -- -- return __io_recvmsg_copy_hdr(req, iomsg); --} -- --static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req, -- bool needs_lock) --{ -- struct io_sr_msg *sr = &req->sr_msg; -- struct io_buffer *kbuf; -- -- kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock); -- if (IS_ERR(kbuf)) -- return kbuf; -- -- sr->kbuf = kbuf; -- req->flags |= REQ_F_BUFFER_SELECTED; -- return kbuf; --} -- --static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req) --{ -- return io_put_kbuf(req, req->sr_msg.kbuf); --} -- --static int io_recvmsg_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ -- struct io_async_msghdr *async_msg = req->async_data; -- struct io_sr_msg *sr = &req->sr_msg; -- int ret; -- -- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- if (unlikely(sqe->addr2 || sqe->splice_fd_in || sqe->ioprio)) -- return -EINVAL; -- -- sr->msg_flags = READ_ONCE(sqe->msg_flags); -- sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); -- sr->len = READ_ONCE(sqe->len); -- sr->bgid = READ_ONCE(sqe->buf_group); -- --#ifdef CONFIG_COMPAT -- if (req->ctx->compat) -- sr->msg_flags |= MSG_CMSG_COMPAT; --#endif -- -- if (!async_msg || !io_op_defs[req->opcode].needs_async_data) -- return 0; -- ret = io_recvmsg_copy_hdr(req, async_msg); -- if (!ret) -- req->flags |= REQ_F_NEED_CLEANUP; -- return ret; --} -- --static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct io_async_msghdr iomsg, *kmsg; -- struct socket *sock; -- struct io_buffer *kbuf; -- unsigned flags; -- int min_ret = 0; -- int ret, cflags = 0; -- -- sock = sock_from_file(req->file, &ret); -- if (unlikely(!sock)) -- return ret; -- -- if (req->async_data) { -- kmsg = req->async_data; -- kmsg->msg.msg_name = &kmsg->addr; -- /* if iov is set, it's allocated already */ -- if (!kmsg->iov) -- kmsg->iov = kmsg->fast_iov; -- kmsg->msg.msg_iter.iov = kmsg->iov; -- } else { -- ret = io_recvmsg_copy_hdr(req, &iomsg); -- if (ret) -- return ret; -- kmsg = &iomsg; -- } -- -- if (req->flags & REQ_F_BUFFER_SELECT) { -- kbuf = io_recv_buffer_select(req, !force_nonblock); -- if (IS_ERR(kbuf)) -- return PTR_ERR(kbuf); -- kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr); -- iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov, -- 1, req->sr_msg.len); -- } -- -- flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; -- if (flags & MSG_DONTWAIT) -- req->flags |= REQ_F_NOWAIT; -- else if (force_nonblock) -- flags |= MSG_DONTWAIT; -- -- if (flags & MSG_WAITALL) -- min_ret = iov_iter_count(&kmsg->msg.msg_iter); -- -- ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg, -- kmsg->uaddr, flags); -- if (force_nonblock && ret == -EAGAIN) -- return io_setup_async_msg(req, kmsg); -- if (ret == -ERESTARTSYS) -- ret = -EINTR; -- -- if (req->flags & REQ_F_BUFFER_SELECTED) -- cflags = io_put_recv_kbuf(req); -- if (kmsg->iov != kmsg->fast_iov) -- kfree(kmsg->iov); -- req->flags &= ~REQ_F_NEED_CLEANUP; -- if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)))) -- req_set_fail_links(req); -- __io_req_complete(req, ret, cflags, cs); -- return 0; --} -- --static int io_recv(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct io_buffer *kbuf; -- struct io_sr_msg *sr = &req->sr_msg; -- struct msghdr msg; -- void __user *buf = sr->buf; -- struct socket *sock; -- struct iovec iov; -- unsigned flags; -- int min_ret = 0; -- int ret, cflags = 0; -- -- sock = sock_from_file(req->file, &ret); -- if (unlikely(!sock)) -- return ret; -- -- if (req->flags & REQ_F_BUFFER_SELECT) { -- kbuf = io_recv_buffer_select(req, !force_nonblock); -- if (IS_ERR(kbuf)) -- return PTR_ERR(kbuf); -- buf = u64_to_user_ptr(kbuf->addr); -- } -- -- ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter); -- if (unlikely(ret)) -- goto out_free; -- -- msg.msg_name = NULL; -- msg.msg_control = NULL; -- msg.msg_controllen = 0; -- msg.msg_namelen = 0; -- msg.msg_iocb = NULL; -- msg.msg_flags = 0; -- -- flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; -- if (flags & MSG_DONTWAIT) -- req->flags |= REQ_F_NOWAIT; -- else if (force_nonblock) -- flags |= MSG_DONTWAIT; -- -- if (flags & MSG_WAITALL) -- min_ret = iov_iter_count(&msg.msg_iter); -- -- ret = sock_recvmsg(sock, &msg, flags); -- if (force_nonblock && ret == -EAGAIN) -- return -EAGAIN; -- if (ret == -ERESTARTSYS) -- ret = -EINTR; --out_free: -- if (req->flags & REQ_F_BUFFER_SELECTED) -- cflags = io_put_recv_kbuf(req); -- if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)))) -- req_set_fail_links(req); -- __io_req_complete(req, ret, cflags, cs); -- return 0; --} -- --static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- struct io_accept *accept = &req->accept; -- -- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL))) -- return -EINVAL; -- if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->splice_fd_in) -- return -EINVAL; -- -- accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); -- accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); -- accept->flags = READ_ONCE(sqe->accept_flags); -- accept->nofile = rlimit(RLIMIT_NOFILE); -- return 0; --} -- --static int io_accept(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct io_accept *accept = &req->accept; -- unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; -- int ret; -- -- if (req->file->f_flags & O_NONBLOCK) -- req->flags |= REQ_F_NOWAIT; -- -- ret = __sys_accept4_file(req->file, file_flags, accept->addr, -- accept->addr_len, accept->flags, -- accept->nofile); -- if (ret == -EAGAIN && force_nonblock) -- return -EAGAIN; -- if (ret < 0) { -- if (ret == -ERESTARTSYS) -- ret = -EINTR; -- req_set_fail_links(req); -- } -- __io_req_complete(req, ret, 0, cs); -- return 0; --} -- --static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- struct io_connect *conn = &req->connect; -- struct io_async_connect *io = req->async_data; -- -- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL))) -- return -EINVAL; -- if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags || -- sqe->splice_fd_in) -- return -EINVAL; -- -- conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); -- conn->addr_len = READ_ONCE(sqe->addr2); -- -- if (!io) -- return 0; -- -- return move_addr_to_kernel(conn->addr, conn->addr_len, -- &io->address); --} -- --static int io_connect(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct io_async_connect __io, *io; -- unsigned file_flags; -- int ret; -- -- if (req->async_data) { -- io = req->async_data; -- } else { -- ret = move_addr_to_kernel(req->connect.addr, -- req->connect.addr_len, -- &__io.address); -- if (ret) -- goto out; -- io = &__io; -- } -- -- file_flags = force_nonblock ? O_NONBLOCK : 0; -- -- ret = __sys_connect_file(req->file, &io->address, -- req->connect.addr_len, file_flags); -- if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) { -- if (req->async_data) -- return -EAGAIN; -- if (io_alloc_async_data(req)) { -- ret = -ENOMEM; -- goto out; -- } -- io = req->async_data; -- memcpy(req->async_data, &__io, sizeof(__io)); -- return -EAGAIN; -- } -- if (ret == -ERESTARTSYS) -- ret = -EINTR; --out: -- if (ret < 0) -- req_set_fail_links(req); -- __io_req_complete(req, ret, 0, cs); -- return 0; --} --#else /* !CONFIG_NET */ --static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- return -EOPNOTSUPP; --} -- --static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- return -EOPNOTSUPP; --} -- --static int io_send(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- return -EOPNOTSUPP; --} -- --static int io_recvmsg_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ -- return -EOPNOTSUPP; --} -- --static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- return -EOPNOTSUPP; --} -- --static int io_recv(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- return -EOPNOTSUPP; --} -- --static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- return -EOPNOTSUPP; --} -- --static int io_accept(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- return -EOPNOTSUPP; --} -- --static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- return -EOPNOTSUPP; --} -- --static int io_connect(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- return -EOPNOTSUPP; --} --#endif /* CONFIG_NET */ -- --struct io_poll_table { -- struct poll_table_struct pt; -- struct io_kiocb *req; -- int nr_entries; -- int error; --}; -- --static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll, -- __poll_t mask, task_work_func_t func) --{ -- bool twa_signal_ok; -- int ret; -- -- /* for instances that support it check for an event match first: */ -- if (mask && !(mask & poll->events)) -- return 0; -- -- trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask); -- -- list_del_init(&poll->wait.entry); -- -- req->result = mask; -- init_task_work(&req->task_work, func); -- percpu_ref_get(&req->ctx->refs); -- -- /* -- * If we using the signalfd wait_queue_head for this wakeup, then -- * it's not safe to use TWA_SIGNAL as we could be recursing on the -- * tsk->sighand->siglock on doing the wakeup. Should not be needed -- * either, as the normal wakeup will suffice. -- */ -- twa_signal_ok = (poll->head != &req->task->sighand->signalfd_wqh); -- -- /* -- * If this fails, then the task is exiting. When a task exits, the -- * work gets canceled, so just cancel this request as well instead -- * of executing it. We can't safely execute it anyway, as we may not -- * have the needed state needed for it anyway. -- */ -- ret = io_req_task_work_add(req, twa_signal_ok); -- if (unlikely(ret)) { -- struct task_struct *tsk; -- -- WRITE_ONCE(poll->canceled, true); -- tsk = io_wq_get_task(req->ctx->io_wq); -- task_work_add(tsk, &req->task_work, TWA_NONE); -- wake_up_process(tsk); -- } -- return 1; --} -- --static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll) -- __acquires(&req->ctx->completion_lock) --{ -- struct io_ring_ctx *ctx = req->ctx; -- -- if (!req->result && !READ_ONCE(poll->canceled)) { -- struct poll_table_struct pt = { ._key = poll->events }; -- -- req->result = vfs_poll(req->file, &pt) & poll->events; -- } -- -- spin_lock_irq(&ctx->completion_lock); -- if (!req->result && !READ_ONCE(poll->canceled)) { -- add_wait_queue(poll->head, &poll->wait); -- return true; -- } -- -- return false; --} -- --static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req) --{ -- /* pure poll stashes this in ->async_data, poll driven retry elsewhere */ -- if (req->opcode == IORING_OP_POLL_ADD) -- return req->async_data; -- return req->apoll->double_poll; --} -- --static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req) --{ -- if (req->opcode == IORING_OP_POLL_ADD) -- return &req->poll; -- return &req->apoll->poll; --} -- --static void io_poll_remove_double(struct io_kiocb *req) --{ -- struct io_poll_iocb *poll = io_poll_get_double(req); -- -- lockdep_assert_held(&req->ctx->completion_lock); -- -- if (poll && poll->head) { -- struct wait_queue_head *head = poll->head; -- -- spin_lock(&head->lock); -- list_del_init(&poll->wait.entry); -- if (poll->wait.private) -- refcount_dec(&req->refs); -- poll->head = NULL; -- spin_unlock(&head->lock); -- } --} -- --static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error) --{ -- struct io_ring_ctx *ctx = req->ctx; -- -- io_poll_remove_double(req); -- req->poll.done = true; -- io_cqring_fill_event(req, error ? error : mangle_poll(mask)); -- io_commit_cqring(ctx); --} -- --static void io_poll_task_func(struct callback_head *cb) --{ -- struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); -- struct io_ring_ctx *ctx = req->ctx; -- struct io_kiocb *nxt; -- -- if (io_poll_rewait(req, &req->poll)) { -- spin_unlock_irq(&ctx->completion_lock); -- } else { -- hash_del(&req->hash_node); -- io_poll_complete(req, req->result, 0); -- spin_unlock_irq(&ctx->completion_lock); -- -- nxt = io_put_req_find_next(req); -- io_cqring_ev_posted(ctx); -- if (nxt) -- __io_req_task_submit(nxt); -- } -- -- percpu_ref_put(&ctx->refs); --} -- --static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode, -- int sync, void *key) --{ -- struct io_kiocb *req = wait->private; -- struct io_poll_iocb *poll = io_poll_get_single(req); -- __poll_t mask = key_to_poll(key); -- -- /* for instances that support it check for an event match first: */ -- if (mask && !(mask & poll->events)) -- return 0; -- -- list_del_init(&wait->entry); -- -- if (poll && poll->head) { -- bool done; -- -- spin_lock(&poll->head->lock); -- done = list_empty(&poll->wait.entry); -- if (!done) -- list_del_init(&poll->wait.entry); -- /* make sure double remove sees this as being gone */ -- wait->private = NULL; -- spin_unlock(&poll->head->lock); -- if (!done) { -- /* use wait func handler, so it matches the rq type */ -- poll->wait.func(&poll->wait, mode, sync, key); -- } -- } -- refcount_dec(&req->refs); -- return 1; --} -- --static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events, -- wait_queue_func_t wake_func) --{ -- poll->head = NULL; -- poll->done = false; -- poll->canceled = false; -- poll->events = events; -- INIT_LIST_HEAD(&poll->wait.entry); -- init_waitqueue_func_entry(&poll->wait, wake_func); --} -- --static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, -- struct wait_queue_head *head, -- struct io_poll_iocb **poll_ptr) --{ -- struct io_kiocb *req = pt->req; -- -- /* -- * The file being polled uses multiple waitqueues for poll handling -- * (e.g. one for read, one for write). Setup a separate io_poll_iocb -- * if this happens. -- */ -- if (unlikely(pt->nr_entries)) { -- struct io_poll_iocb *poll_one = poll; -- -- /* already have a 2nd entry, fail a third attempt */ -- if (*poll_ptr) { -- pt->error = -EINVAL; -- return; -- } -- /* double add on the same waitqueue head, ignore */ -- if (poll->head == head) -- return; -- poll = kmalloc(sizeof(*poll), GFP_ATOMIC); -- if (!poll) { -- pt->error = -ENOMEM; -- return; -- } -- io_init_poll_iocb(poll, poll_one->events, io_poll_double_wake); -- refcount_inc(&req->refs); -- poll->wait.private = req; -- *poll_ptr = poll; -- } -- -- pt->nr_entries++; -- poll->head = head; -- -- if (poll->events & EPOLLEXCLUSIVE) -- add_wait_queue_exclusive(head, &poll->wait); -- else -- add_wait_queue(head, &poll->wait); --} -- --static void io_async_queue_proc(struct file *file, struct wait_queue_head *head, -- struct poll_table_struct *p) --{ -- struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); -- struct async_poll *apoll = pt->req->apoll; -- -- __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll); --} -- --static void io_async_task_func(struct callback_head *cb) --{ -- struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); -- struct async_poll *apoll = req->apoll; -- struct io_ring_ctx *ctx = req->ctx; -- -- trace_io_uring_task_run(req->ctx, req->opcode, req->user_data); -- -- if (io_poll_rewait(req, &apoll->poll)) { -- spin_unlock_irq(&ctx->completion_lock); -- percpu_ref_put(&ctx->refs); -- return; -- } -- -- /* If req is still hashed, it cannot have been canceled. Don't check. */ -- if (hash_hashed(&req->hash_node)) -- hash_del(&req->hash_node); -- -- io_poll_remove_double(req); -- spin_unlock_irq(&ctx->completion_lock); -- -- if (!READ_ONCE(apoll->poll.canceled)) -- __io_req_task_submit(req); -- else -- __io_req_task_cancel(req, -ECANCELED); -- -- percpu_ref_put(&ctx->refs); -- kfree(apoll->double_poll); -- kfree(apoll); --} -- --static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync, -- void *key) --{ -- struct io_kiocb *req = wait->private; -- struct io_poll_iocb *poll = &req->apoll->poll; -- -- trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data, -- key_to_poll(key)); -- -- return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func); --} -- --static void io_poll_req_insert(struct io_kiocb *req) --{ -- struct io_ring_ctx *ctx = req->ctx; -- struct hlist_head *list; -- -- list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)]; -- hlist_add_head(&req->hash_node, list); --} -- --static __poll_t __io_arm_poll_handler(struct io_kiocb *req, -- struct io_poll_iocb *poll, -- struct io_poll_table *ipt, __poll_t mask, -- wait_queue_func_t wake_func) -- __acquires(&ctx->completion_lock) --{ -- struct io_ring_ctx *ctx = req->ctx; -- bool cancel = false; -- -- if (req->file->f_op->may_pollfree) { -- spin_lock_irq(&ctx->completion_lock); -- return -EOPNOTSUPP; -- } -- -- INIT_HLIST_NODE(&req->hash_node); -- io_init_poll_iocb(poll, mask, wake_func); -- poll->file = req->file; -- poll->wait.private = req; -- -- ipt->pt._key = mask; -- ipt->req = req; -- ipt->error = 0; -- ipt->nr_entries = 0; -- -- mask = vfs_poll(req->file, &ipt->pt) & poll->events; -- if (unlikely(!ipt->nr_entries) && !ipt->error) -- ipt->error = -EINVAL; -- -- spin_lock_irq(&ctx->completion_lock); -- if (ipt->error) -- io_poll_remove_double(req); -- if (likely(poll->head)) { -- spin_lock(&poll->head->lock); -- if (unlikely(list_empty(&poll->wait.entry))) { -- if (ipt->error) -- cancel = true; -- ipt->error = 0; -- mask = 0; -- } -- if (mask || ipt->error) -- list_del_init(&poll->wait.entry); -- else if (cancel) -- WRITE_ONCE(poll->canceled, true); -- else if (!poll->done) /* actually waiting for an event */ -- io_poll_req_insert(req); -- spin_unlock(&poll->head->lock); -- } -- -- return mask; --} -- --static bool io_arm_poll_handler(struct io_kiocb *req) --{ -- const struct io_op_def *def = &io_op_defs[req->opcode]; -- struct io_ring_ctx *ctx = req->ctx; -- struct async_poll *apoll; -- struct io_poll_table ipt; -- __poll_t mask, ret; -- int rw; -- -- if (!req->file || !file_can_poll(req->file)) -- return false; -- if (req->flags & REQ_F_POLLED) -- return false; -- if (def->pollin) -- rw = READ; -- else if (def->pollout) -- rw = WRITE; -- else -- return false; -- /* if we can't nonblock try, then no point in arming a poll handler */ -- if (!io_file_supports_async(req->file, rw)) -- return false; -- -- apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC); -- if (unlikely(!apoll)) -- return false; -- apoll->double_poll = NULL; -- -- req->flags |= REQ_F_POLLED; -- req->apoll = apoll; -- -- mask = 0; -- if (def->pollin) -- mask |= POLLIN | POLLRDNORM; -- if (def->pollout) -- mask |= POLLOUT | POLLWRNORM; -- -- /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */ -- if ((req->opcode == IORING_OP_RECVMSG) && -- (req->sr_msg.msg_flags & MSG_ERRQUEUE)) -- mask &= ~POLLIN; -- -- mask |= POLLERR | POLLPRI; -- -- ipt.pt._qproc = io_async_queue_proc; -- -- ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, -- io_async_wake); -- if (ret || ipt.error) { -- io_poll_remove_double(req); -- spin_unlock_irq(&ctx->completion_lock); -- kfree(apoll->double_poll); -- kfree(apoll); -- return false; -- } -- spin_unlock_irq(&ctx->completion_lock); -- trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask, -- apoll->poll.events); -- return true; --} -- --static bool __io_poll_remove_one(struct io_kiocb *req, -- struct io_poll_iocb *poll) --{ -- bool do_complete = false; -- -- spin_lock(&poll->head->lock); -- WRITE_ONCE(poll->canceled, true); -- if (!list_empty(&poll->wait.entry)) { -- list_del_init(&poll->wait.entry); -- do_complete = true; -- } -- spin_unlock(&poll->head->lock); -- hash_del(&req->hash_node); -- return do_complete; --} -- --static bool io_poll_remove_one(struct io_kiocb *req) --{ -- bool do_complete; -- -- io_poll_remove_double(req); -- -- if (req->opcode == IORING_OP_POLL_ADD) { -- do_complete = __io_poll_remove_one(req, &req->poll); -- } else { -- struct async_poll *apoll = req->apoll; -- -- /* non-poll requests have submit ref still */ -- do_complete = __io_poll_remove_one(req, &apoll->poll); -- if (do_complete) { -- io_put_req(req); -- kfree(apoll->double_poll); -- kfree(apoll); -- } -- } -- -- if (do_complete) { -- io_cqring_fill_event(req, -ECANCELED); -- io_commit_cqring(req->ctx); -- req_set_fail_links(req); -- io_put_req_deferred(req, 1); -- } -- -- return do_complete; --} -- --/* -- * Returns true if we found and killed one or more poll requests -- */ --static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, -- struct files_struct *files) --{ -- struct hlist_node *tmp; -- struct io_kiocb *req; -- int posted = 0, i; -- -- spin_lock_irq(&ctx->completion_lock); -- for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { -- struct hlist_head *list; -- -- list = &ctx->cancel_hash[i]; -- hlist_for_each_entry_safe(req, tmp, list, hash_node) { -- if (io_match_task(req, tsk, files)) -- posted += io_poll_remove_one(req); -- } -- } -- spin_unlock_irq(&ctx->completion_lock); -- -- if (posted) -- io_cqring_ev_posted(ctx); -- -- return posted != 0; --} -- --static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr) --{ -- struct hlist_head *list; -- struct io_kiocb *req; -- -- list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)]; -- hlist_for_each_entry(req, list, hash_node) { -- if (sqe_addr != req->user_data) -- continue; -- if (io_poll_remove_one(req)) -- return 0; -- return -EALREADY; -- } -- -- return -ENOENT; --} -- --static int io_poll_remove_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ -- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index || -- sqe->poll_events) -- return -EINVAL; -- -- req->poll.addr = READ_ONCE(sqe->addr); -- return 0; --} -- --/* -- * Find a running poll command that matches one specified in sqe->addr, -- * and remove it if found. -- */ --static int io_poll_remove(struct io_kiocb *req) --{ -- struct io_ring_ctx *ctx = req->ctx; -- u64 addr; -- int ret; -- -- addr = req->poll.addr; -- spin_lock_irq(&ctx->completion_lock); -- ret = io_poll_cancel(ctx, addr); -- spin_unlock_irq(&ctx->completion_lock); -- -- if (ret < 0) -- req_set_fail_links(req); -- io_req_complete(req, ret); -- return 0; --} -- --static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, -- void *key) --{ -- struct io_kiocb *req = wait->private; -- struct io_poll_iocb *poll = &req->poll; -- -- return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func); --} -- --static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head, -- struct poll_table_struct *p) --{ -- struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); -- -- __io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->async_data); --} -- --static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- struct io_poll_iocb *poll = &req->poll; -- u32 events; -- -- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index) -- return -EINVAL; -- -- events = READ_ONCE(sqe->poll32_events); --#ifdef __BIG_ENDIAN -- events = swahw32(events); --#endif -- poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP | -- (events & EPOLLEXCLUSIVE); -- return 0; --} -- --static int io_poll_add(struct io_kiocb *req) --{ -- struct io_poll_iocb *poll = &req->poll; -- struct io_ring_ctx *ctx = req->ctx; -- struct io_poll_table ipt; -- __poll_t mask; -- -- ipt.pt._qproc = io_poll_queue_proc; -- -- mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events, -- io_poll_wake); -- -- if (mask) { /* no async, we'd stolen it */ -- ipt.error = 0; -- io_poll_complete(req, mask, 0); -- } -- spin_unlock_irq(&ctx->completion_lock); -- -- if (mask) { -- io_cqring_ev_posted(ctx); -- io_put_req(req); -- } -- return ipt.error; --} -- --static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) --{ -- struct io_timeout_data *data = container_of(timer, -- struct io_timeout_data, timer); -- struct io_kiocb *req = data->req; -- struct io_ring_ctx *ctx = req->ctx; -- unsigned long flags; -- -- spin_lock_irqsave(&ctx->completion_lock, flags); -- list_del_init(&req->timeout.list); -- atomic_set(&req->ctx->cq_timeouts, -- atomic_read(&req->ctx->cq_timeouts) + 1); -- -- io_cqring_fill_event(req, -ETIME); -- io_commit_cqring(ctx); -- spin_unlock_irqrestore(&ctx->completion_lock, flags); -- -- io_cqring_ev_posted(ctx); -- req_set_fail_links(req); -- io_put_req(req); -- return HRTIMER_NORESTART; --} -- --static int __io_timeout_cancel(struct io_kiocb *req) --{ -- struct io_timeout_data *io = req->async_data; -- int ret; -- -- ret = hrtimer_try_to_cancel(&io->timer); -- if (ret == -1) -- return -EALREADY; -- list_del_init(&req->timeout.list); -- -- req_set_fail_links(req); -- io_cqring_fill_event(req, -ECANCELED); -- io_put_req_deferred(req, 1); -- return 0; --} -- --static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data) --{ -- struct io_kiocb *req; -- int ret = -ENOENT; -- -- list_for_each_entry(req, &ctx->timeout_list, timeout.list) { -- if (user_data == req->user_data) { -- ret = 0; -- break; -- } -- } -- -- if (ret == -ENOENT) -- return ret; -- -- return __io_timeout_cancel(req); --} -- --static int io_timeout_remove_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ -- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) -- return -EINVAL; -- if (sqe->ioprio || sqe->buf_index || sqe->len || sqe->timeout_flags || -- sqe->splice_fd_in) -- return -EINVAL; -- -- req->timeout_rem.addr = READ_ONCE(sqe->addr); -- return 0; --} -- --/* -- * Remove or update an existing timeout command -- */ --static int io_timeout_remove(struct io_kiocb *req) --{ -- struct io_ring_ctx *ctx = req->ctx; -- int ret; -- -- spin_lock_irq(&ctx->completion_lock); -- ret = io_timeout_cancel(ctx, req->timeout_rem.addr); -- -- io_cqring_fill_event(req, ret); -- io_commit_cqring(ctx); -- spin_unlock_irq(&ctx->completion_lock); -- io_cqring_ev_posted(ctx); -- if (ret < 0) -- req_set_fail_links(req); -- io_put_req(req); -- return 0; --} -- --static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, -- bool is_timeout_link) --{ -- struct io_timeout_data *data; -- unsigned flags; -- u32 off = READ_ONCE(sqe->off); -- -- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- if (sqe->ioprio || sqe->buf_index || sqe->len != 1 || -- sqe->splice_fd_in) -- return -EINVAL; -- if (off && is_timeout_link) -- return -EINVAL; -- flags = READ_ONCE(sqe->timeout_flags); -- if (flags & ~IORING_TIMEOUT_ABS) -- return -EINVAL; -- -- req->timeout.off = off; -- -- if (!req->async_data && io_alloc_async_data(req)) -- return -ENOMEM; -- -- data = req->async_data; -- data->req = req; -- -- if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr))) -- return -EFAULT; -- -- if (flags & IORING_TIMEOUT_ABS) -- data->mode = HRTIMER_MODE_ABS; -- else -- data->mode = HRTIMER_MODE_REL; -- -- INIT_LIST_HEAD(&req->timeout.list); -- hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode); -- return 0; --} -- --static int io_timeout(struct io_kiocb *req) --{ -- struct io_ring_ctx *ctx = req->ctx; -- struct io_timeout_data *data = req->async_data; -- struct list_head *entry; -- u32 tail, off = req->timeout.off; -- -- spin_lock_irq(&ctx->completion_lock); -- -- /* -- * sqe->off holds how many events that need to occur for this -- * timeout event to be satisfied. If it isn't set, then this is -- * a pure timeout request, sequence isn't used. -- */ -- if (io_is_timeout_noseq(req)) { -- entry = ctx->timeout_list.prev; -- goto add; -- } -- -- tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); -- req->timeout.target_seq = tail + off; -- -- /* Update the last seq here in case io_flush_timeouts() hasn't. -- * This is safe because ->completion_lock is held, and submissions -- * and completions are never mixed in the same ->completion_lock section. -- */ -- ctx->cq_last_tm_flush = tail; -- -- /* -- * Insertion sort, ensuring the first entry in the list is always -- * the one we need first. -- */ -- list_for_each_prev(entry, &ctx->timeout_list) { -- struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, -- timeout.list); -- -- if (io_is_timeout_noseq(nxt)) -- continue; -- /* nxt.seq is behind @tail, otherwise would've been completed */ -- if (off >= nxt->timeout.target_seq - tail) -- break; -- } --add: -- list_add(&req->timeout.list, entry); -- data->timer.function = io_timeout_fn; -- hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); -- spin_unlock_irq(&ctx->completion_lock); -- return 0; --} -- --static bool io_cancel_cb(struct io_wq_work *work, void *data) --{ -- struct io_kiocb *req = container_of(work, struct io_kiocb, work); -- -- return req->user_data == (unsigned long) data; --} -- --static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr) --{ -- enum io_wq_cancel cancel_ret; -- int ret = 0; -- -- cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr, false); -- switch (cancel_ret) { -- case IO_WQ_CANCEL_OK: -- ret = 0; -- break; -- case IO_WQ_CANCEL_RUNNING: -- ret = -EALREADY; -- break; -- case IO_WQ_CANCEL_NOTFOUND: -- ret = -ENOENT; -- break; -- } -- -- return ret; --} -- --static void io_async_find_and_cancel(struct io_ring_ctx *ctx, -- struct io_kiocb *req, __u64 sqe_addr, -- int success_ret) --{ -- unsigned long flags; -- int ret; -- -- ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr); -- if (ret != -ENOENT) { -- spin_lock_irqsave(&ctx->completion_lock, flags); -- goto done; -- } -- -- spin_lock_irqsave(&ctx->completion_lock, flags); -- ret = io_timeout_cancel(ctx, sqe_addr); -- if (ret != -ENOENT) -- goto done; -- ret = io_poll_cancel(ctx, sqe_addr); --done: -- if (!ret) -- ret = success_ret; -- io_cqring_fill_event(req, ret); -- io_commit_cqring(ctx); -- spin_unlock_irqrestore(&ctx->completion_lock, flags); -- io_cqring_ev_posted(ctx); -- -- if (ret < 0) -- req_set_fail_links(req); -- io_put_req(req); --} -- --static int io_async_cancel_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ -- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -- return -EINVAL; -- if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) -- return -EINVAL; -- if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags || -- sqe->splice_fd_in) -- return -EINVAL; -- -- req->cancel.addr = READ_ONCE(sqe->addr); -- return 0; --} -- --static int io_async_cancel(struct io_kiocb *req) --{ -- struct io_ring_ctx *ctx = req->ctx; -- -- io_async_find_and_cancel(ctx, req, req->cancel.addr, 0); -- return 0; --} -- --static int io_files_update_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ -- if (unlikely(req->ctx->flags & IORING_SETUP_SQPOLL)) -- return -EINVAL; -- if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) -- return -EINVAL; -- if (sqe->ioprio || sqe->rw_flags) -- return -EINVAL; -- -- req->files_update.offset = READ_ONCE(sqe->off); -- req->files_update.nr_args = READ_ONCE(sqe->len); -- if (!req->files_update.nr_args) -- return -EINVAL; -- req->files_update.arg = READ_ONCE(sqe->addr); -- return 0; --} -- --static int io_files_update(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct io_ring_ctx *ctx = req->ctx; -- struct io_uring_files_update up; -- int ret; -- -- if (force_nonblock) -- return -EAGAIN; -- -- up.offset = req->files_update.offset; -- up.fds = req->files_update.arg; -- -- mutex_lock(&ctx->uring_lock); -- ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args); -- mutex_unlock(&ctx->uring_lock); -- -- if (ret < 0) -- req_set_fail_links(req); -- __io_req_complete(req, ret, 0, cs); -- return 0; --} -- --static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- switch (req->opcode) { -- case IORING_OP_NOP: -- return 0; -- case IORING_OP_READV: -- case IORING_OP_READ_FIXED: -- case IORING_OP_READ: -- return io_read_prep(req, sqe); -- case IORING_OP_WRITEV: -- case IORING_OP_WRITE_FIXED: -- case IORING_OP_WRITE: -- return io_write_prep(req, sqe); -- case IORING_OP_POLL_ADD: -- return io_poll_add_prep(req, sqe); -- case IORING_OP_POLL_REMOVE: -- return io_poll_remove_prep(req, sqe); -- case IORING_OP_FSYNC: -- return io_prep_fsync(req, sqe); -- case IORING_OP_SYNC_FILE_RANGE: -- return io_prep_sfr(req, sqe); -- case IORING_OP_SENDMSG: -- case IORING_OP_SEND: -- return io_sendmsg_prep(req, sqe); -- case IORING_OP_RECVMSG: -- case IORING_OP_RECV: -- return io_recvmsg_prep(req, sqe); -- case IORING_OP_CONNECT: -- return io_connect_prep(req, sqe); -- case IORING_OP_TIMEOUT: -- return io_timeout_prep(req, sqe, false); -- case IORING_OP_TIMEOUT_REMOVE: -- return io_timeout_remove_prep(req, sqe); -- case IORING_OP_ASYNC_CANCEL: -- return io_async_cancel_prep(req, sqe); -- case IORING_OP_LINK_TIMEOUT: -- return io_timeout_prep(req, sqe, true); -- case IORING_OP_ACCEPT: -- return io_accept_prep(req, sqe); -- case IORING_OP_FALLOCATE: -- return io_fallocate_prep(req, sqe); -- case IORING_OP_OPENAT: -- return io_openat_prep(req, sqe); -- case IORING_OP_CLOSE: -- return io_close_prep(req, sqe); -- case IORING_OP_FILES_UPDATE: -- return io_files_update_prep(req, sqe); -- case IORING_OP_STATX: -- return io_statx_prep(req, sqe); -- case IORING_OP_FADVISE: -- return io_fadvise_prep(req, sqe); -- case IORING_OP_MADVISE: -- return io_madvise_prep(req, sqe); -- case IORING_OP_OPENAT2: -- return io_openat2_prep(req, sqe); -- case IORING_OP_EPOLL_CTL: -- return io_epoll_ctl_prep(req, sqe); -- case IORING_OP_SPLICE: -- return io_splice_prep(req, sqe); -- case IORING_OP_PROVIDE_BUFFERS: -- return io_provide_buffers_prep(req, sqe); -- case IORING_OP_REMOVE_BUFFERS: -- return io_remove_buffers_prep(req, sqe); -- case IORING_OP_TEE: -- return io_tee_prep(req, sqe); -- } -- -- printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n", -- req->opcode); -- return-EINVAL; --} -- --static int io_req_defer_prep(struct io_kiocb *req, -- const struct io_uring_sqe *sqe) --{ -- if (!sqe) -- return 0; -- if (io_alloc_async_data(req)) -- return -EAGAIN; -- return io_req_prep(req, sqe); --} -- --static u32 io_get_sequence(struct io_kiocb *req) --{ -- struct io_kiocb *pos; -- struct io_ring_ctx *ctx = req->ctx; -- u32 total_submitted, nr_reqs = 1; -- -- if (req->flags & REQ_F_LINK_HEAD) -- list_for_each_entry(pos, &req->link_list, link_list) -- nr_reqs++; -- -- total_submitted = ctx->cached_sq_head - ctx->cached_sq_dropped; -- return total_submitted - nr_reqs; --} -- --static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe) --{ -- struct io_ring_ctx *ctx = req->ctx; -- struct io_defer_entry *de; -- int ret; -- u32 seq; -- -- /* Still need defer if there is pending req in defer list. */ -- if (likely(list_empty_careful(&ctx->defer_list) && -- !(req->flags & REQ_F_IO_DRAIN))) -- return 0; -- -- seq = io_get_sequence(req); -- /* Still a chance to pass the sequence check */ -- if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) -- return 0; -- -- if (!req->async_data) { -- ret = io_req_defer_prep(req, sqe); -- if (ret) -- return ret; -- } -- io_prep_async_link(req); -- de = kmalloc(sizeof(*de), GFP_KERNEL); -- if (!de) -- return -ENOMEM; -- -- spin_lock_irq(&ctx->completion_lock); -- if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) { -- spin_unlock_irq(&ctx->completion_lock); -- kfree(de); -- io_queue_async_work(req); -- return -EIOCBQUEUED; -- } -- -- trace_io_uring_defer(ctx, req, req->user_data); -- de->req = req; -- de->seq = seq; -- list_add_tail(&de->list, &ctx->defer_list); -- spin_unlock_irq(&ctx->completion_lock); -- return -EIOCBQUEUED; --} -- --static void io_req_drop_files(struct io_kiocb *req) --{ -- struct io_ring_ctx *ctx = req->ctx; -- struct io_uring_task *tctx = req->task->io_uring; -- unsigned long flags; -- -- if (req->work.flags & IO_WQ_WORK_FILES) { -- put_files_struct(req->work.identity->files); -- put_nsproxy(req->work.identity->nsproxy); -- } -- spin_lock_irqsave(&ctx->inflight_lock, flags); -- list_del(&req->inflight_entry); -- spin_unlock_irqrestore(&ctx->inflight_lock, flags); -- req->flags &= ~REQ_F_INFLIGHT; -- req->work.flags &= ~IO_WQ_WORK_FILES; -- if (atomic_read(&tctx->in_idle)) -- wake_up(&tctx->wait); --} -- --static void __io_clean_op(struct io_kiocb *req) --{ -- if (req->flags & REQ_F_BUFFER_SELECTED) { -- switch (req->opcode) { -- case IORING_OP_READV: -- case IORING_OP_READ_FIXED: -- case IORING_OP_READ: -- kfree((void *)(unsigned long)req->rw.addr); -- break; -- case IORING_OP_RECVMSG: -- case IORING_OP_RECV: -- kfree(req->sr_msg.kbuf); -- break; -- } -- req->flags &= ~REQ_F_BUFFER_SELECTED; -- } -- -- if (req->flags & REQ_F_NEED_CLEANUP) { -- switch (req->opcode) { -- case IORING_OP_READV: -- case IORING_OP_READ_FIXED: -- case IORING_OP_READ: -- case IORING_OP_WRITEV: -- case IORING_OP_WRITE_FIXED: -- case IORING_OP_WRITE: { -- struct io_async_rw *io = req->async_data; -- if (io->free_iovec) -- kfree(io->free_iovec); -- break; -- } -- case IORING_OP_RECVMSG: -- case IORING_OP_SENDMSG: { -- struct io_async_msghdr *io = req->async_data; -- if (io->iov != io->fast_iov) -- kfree(io->iov); -- break; -- } -- case IORING_OP_SPLICE: -- case IORING_OP_TEE: -- io_put_file(req, req->splice.file_in, -- (req->splice.flags & SPLICE_F_FD_IN_FIXED)); -- break; -- case IORING_OP_OPENAT: -- case IORING_OP_OPENAT2: -- if (req->open.filename) -- putname(req->open.filename); -- break; -- } -- req->flags &= ~REQ_F_NEED_CLEANUP; -- } --} -- --static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, -- struct io_comp_state *cs) --{ -- struct io_ring_ctx *ctx = req->ctx; -- int ret; -- -- switch (req->opcode) { -- case IORING_OP_NOP: -- ret = io_nop(req, cs); -- break; -- case IORING_OP_READV: -- case IORING_OP_READ_FIXED: -- case IORING_OP_READ: -- ret = io_read(req, force_nonblock, cs); -- break; -- case IORING_OP_WRITEV: -- case IORING_OP_WRITE_FIXED: -- case IORING_OP_WRITE: -- ret = io_write(req, force_nonblock, cs); -- break; -- case IORING_OP_FSYNC: -- ret = io_fsync(req, force_nonblock); -- break; -- case IORING_OP_POLL_ADD: -- ret = io_poll_add(req); -- break; -- case IORING_OP_POLL_REMOVE: -- ret = io_poll_remove(req); -- break; -- case IORING_OP_SYNC_FILE_RANGE: -- ret = io_sync_file_range(req, force_nonblock); -- break; -- case IORING_OP_SENDMSG: -- ret = io_sendmsg(req, force_nonblock, cs); -- break; -- case IORING_OP_SEND: -- ret = io_send(req, force_nonblock, cs); -- break; -- case IORING_OP_RECVMSG: -- ret = io_recvmsg(req, force_nonblock, cs); -- break; -- case IORING_OP_RECV: -- ret = io_recv(req, force_nonblock, cs); -- break; -- case IORING_OP_TIMEOUT: -- ret = io_timeout(req); -- break; -- case IORING_OP_TIMEOUT_REMOVE: -- ret = io_timeout_remove(req); -- break; -- case IORING_OP_ACCEPT: -- ret = io_accept(req, force_nonblock, cs); -- break; -- case IORING_OP_CONNECT: -- ret = io_connect(req, force_nonblock, cs); -- break; -- case IORING_OP_ASYNC_CANCEL: -- ret = io_async_cancel(req); -- break; -- case IORING_OP_FALLOCATE: -- ret = io_fallocate(req, force_nonblock); -- break; -- case IORING_OP_OPENAT: -- ret = io_openat(req, force_nonblock); -- break; -- case IORING_OP_CLOSE: -- ret = io_close(req, force_nonblock, cs); -- break; -- case IORING_OP_FILES_UPDATE: -- ret = io_files_update(req, force_nonblock, cs); -- break; -- case IORING_OP_STATX: -- ret = io_statx(req, force_nonblock); -- break; -- case IORING_OP_FADVISE: -- ret = io_fadvise(req, force_nonblock); -- break; -- case IORING_OP_MADVISE: -- ret = io_madvise(req, force_nonblock); -- break; -- case IORING_OP_OPENAT2: -- ret = io_openat2(req, force_nonblock); -- break; -- case IORING_OP_EPOLL_CTL: -- ret = io_epoll_ctl(req, force_nonblock, cs); -- break; -- case IORING_OP_SPLICE: -- ret = io_splice(req, force_nonblock); -- break; -- case IORING_OP_PROVIDE_BUFFERS: -- ret = io_provide_buffers(req, force_nonblock, cs); -- break; -- case IORING_OP_REMOVE_BUFFERS: -- ret = io_remove_buffers(req, force_nonblock, cs); -- break; -- case IORING_OP_TEE: -- ret = io_tee(req, force_nonblock); -- break; -- default: -- ret = -EINVAL; -- break; -- } -- -- if (ret) -- return ret; -- -- /* If the op doesn't have a file, we're not polling for it */ -- if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) { -- const bool in_async = io_wq_current_is_worker(); -- -- /* workqueue context doesn't hold uring_lock, grab it now */ -- if (in_async) -- mutex_lock(&ctx->uring_lock); -- -- io_iopoll_req_issued(req); -- -- if (in_async) -- mutex_unlock(&ctx->uring_lock); -- } -- -- return 0; --} -- --static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work) --{ -- struct io_kiocb *req = container_of(work, struct io_kiocb, work); -- struct io_kiocb *timeout; -- int ret = 0; -- -- timeout = io_prep_linked_timeout(req); -- if (timeout) -- io_queue_linked_timeout(timeout); -- -- /* if NO_CANCEL is set, we must still run the work */ -- if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) == -- IO_WQ_WORK_CANCEL) { -- ret = -ECANCELED; -- } -- -- if (!ret) { -- do { -- ret = io_issue_sqe(req, false, NULL); -- /* -- * We can get EAGAIN for polled IO even though we're -- * forcing a sync submission from here, since we can't -- * wait for request slots on the block side. -- */ -- if (ret != -EAGAIN) -- break; -- cond_resched(); -- } while (1); -- } -- -- if (ret) { -- struct io_ring_ctx *lock_ctx = NULL; -- -- if (req->ctx->flags & IORING_SETUP_IOPOLL) -- lock_ctx = req->ctx; -- -- /* -- * io_iopoll_complete() does not hold completion_lock to -- * complete polled io, so here for polled io, we can not call -- * io_req_complete() directly, otherwise there maybe concurrent -- * access to cqring, defer_list, etc, which is not safe. Given -- * that io_iopoll_complete() is always called under uring_lock, -- * so here for polled io, we also get uring_lock to complete -- * it. -- */ -- if (lock_ctx) -- mutex_lock(&lock_ctx->uring_lock); -- -- req_set_fail_links(req); -- io_req_complete(req, ret); -- -- if (lock_ctx) -- mutex_unlock(&lock_ctx->uring_lock); -- } -- -- return io_steal_work(req); --} -- --static inline struct file *io_file_from_index(struct io_ring_ctx *ctx, -- int index) --{ -- struct fixed_file_table *table; -- -- table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT]; -- return table->files[index & IORING_FILE_TABLE_MASK]; --} -- --static struct file *io_file_get(struct io_submit_state *state, -- struct io_kiocb *req, int fd, bool fixed) --{ -- struct io_ring_ctx *ctx = req->ctx; -- struct file *file; -- -- if (fixed) { -- if (unlikely((unsigned int)fd >= ctx->nr_user_files)) -- return NULL; -- fd = array_index_nospec(fd, ctx->nr_user_files); -- file = io_file_from_index(ctx, fd); -- if (file) { -- req->fixed_file_refs = &ctx->file_data->node->refs; -- percpu_ref_get(req->fixed_file_refs); -- } -- } else { -- trace_io_uring_file_get(ctx, fd); -- file = __io_file_get(state, fd); -- } -- -- if (file && file->f_op == &io_uring_fops && -- !(req->flags & REQ_F_INFLIGHT)) { -- io_req_init_async(req); -- req->flags |= REQ_F_INFLIGHT; -- -- spin_lock_irq(&ctx->inflight_lock); -- list_add(&req->inflight_entry, &ctx->inflight_list); -- spin_unlock_irq(&ctx->inflight_lock); -- } -- -- return file; --} -- --static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req, -- int fd) --{ -- bool fixed; -- -- fixed = (req->flags & REQ_F_FIXED_FILE) != 0; -- if (unlikely(!fixed && io_async_submit(req->ctx))) -- return -EBADF; -- -- req->file = io_file_get(state, req, fd, fixed); -- if (req->file || io_op_defs[req->opcode].needs_file_no_error) -- return 0; -- return -EBADF; --} -- --static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) --{ -- struct io_timeout_data *data = container_of(timer, -- struct io_timeout_data, timer); -- struct io_kiocb *req = data->req; -- struct io_ring_ctx *ctx = req->ctx; -- struct io_kiocb *prev = NULL; -- unsigned long flags; -- -- spin_lock_irqsave(&ctx->completion_lock, flags); -- -- /* -- * We don't expect the list to be empty, that will only happen if we -- * race with the completion of the linked work. -- */ -- if (!list_empty(&req->link_list)) { -- prev = list_entry(req->link_list.prev, struct io_kiocb, -- link_list); -- list_del_init(&req->link_list); -- if (!refcount_inc_not_zero(&prev->refs)) -- prev = NULL; -- } -- -- list_del(&req->timeout.list); -- spin_unlock_irqrestore(&ctx->completion_lock, flags); -- -- if (prev) { -- io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME); -- io_put_req_deferred(prev, 1); -- } else { -- io_cqring_add_event(req, -ETIME, 0); -- io_put_req_deferred(req, 1); -- } -- return HRTIMER_NORESTART; --} -- --static void __io_queue_linked_timeout(struct io_kiocb *req) --{ -- /* -- * If the list is now empty, then our linked request finished before -- * we got a chance to setup the timer -- */ -- if (!list_empty(&req->link_list)) { -- struct io_timeout_data *data = req->async_data; -- -- data->timer.function = io_link_timeout_fn; -- hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), -- data->mode); -- } --} -- --static void io_queue_linked_timeout(struct io_kiocb *req) --{ -- struct io_ring_ctx *ctx = req->ctx; -- -- spin_lock_irq(&ctx->completion_lock); -- __io_queue_linked_timeout(req); -- spin_unlock_irq(&ctx->completion_lock); -- -- /* drop submission reference */ -- io_put_req(req); --} -- --static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) --{ -- struct io_kiocb *nxt; -- -- if (!(req->flags & REQ_F_LINK_HEAD)) -- return NULL; -- if (req->flags & REQ_F_LINK_TIMEOUT) -- return NULL; -- -- nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, -- link_list); -- if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT) -- return NULL; -- -- nxt->flags |= REQ_F_LTIMEOUT_ACTIVE; -- req->flags |= REQ_F_LINK_TIMEOUT; -- return nxt; --} -- --static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) --{ -- struct io_kiocb *linked_timeout; -- const struct cred *old_creds = NULL; -- int ret; -- --again: -- linked_timeout = io_prep_linked_timeout(req); -- -- if ((req->flags & REQ_F_WORK_INITIALIZED) && -- (req->work.flags & IO_WQ_WORK_CREDS) && -- req->work.identity->creds != current_cred()) { -- if (old_creds) -- revert_creds(old_creds); -- if (old_creds == req->work.identity->creds) -- old_creds = NULL; /* restored original creds */ -- else -- old_creds = override_creds(req->work.identity->creds); -- } -- -- ret = io_issue_sqe(req, true, cs); -- -- /* -- * We async punt it if the file wasn't marked NOWAIT, or if the file -- * doesn't support non-blocking read/write attempts -- */ -- if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) { -- if (!io_arm_poll_handler(req)) { -- /* -- * Queued up for async execution, worker will release -- * submit reference when the iocb is actually submitted. -- */ -- io_queue_async_work(req); -- } -- -- if (linked_timeout) -- io_queue_linked_timeout(linked_timeout); -- } else if (likely(!ret)) { -- /* drop submission reference */ -- req = io_put_req_find_next(req); -- if (linked_timeout) -- io_queue_linked_timeout(linked_timeout); -- -- if (req) { -- if (!(req->flags & REQ_F_FORCE_ASYNC)) -- goto again; -- io_queue_async_work(req); -- } -- } else { -- /* un-prep timeout, so it'll be killed as any other linked */ -- req->flags &= ~REQ_F_LINK_TIMEOUT; -- req_set_fail_links(req); -- io_put_req(req); -- io_req_complete(req, ret); -- } -- -- if (old_creds) -- revert_creds(old_creds); --} -- --static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, -- struct io_comp_state *cs) --{ -- int ret; -- -- ret = io_req_defer(req, sqe); -- if (ret) { -- if (ret != -EIOCBQUEUED) { --fail_req: -- req_set_fail_links(req); -- io_put_req(req); -- io_req_complete(req, ret); -- } -- } else if (req->flags & REQ_F_FORCE_ASYNC) { -- if (!req->async_data) { -- ret = io_req_defer_prep(req, sqe); -- if (unlikely(ret)) -- goto fail_req; -- } -- io_queue_async_work(req); -- } else { -- if (sqe) { -- ret = io_req_prep(req, sqe); -- if (unlikely(ret)) -- goto fail_req; -- } -- __io_queue_sqe(req, cs); -- } --} -- --static inline void io_queue_link_head(struct io_kiocb *req, -- struct io_comp_state *cs) --{ -- if (unlikely(req->flags & REQ_F_FAIL_LINK)) { -- io_put_req(req); -- io_req_complete(req, -ECANCELED); -- } else -- io_queue_sqe(req, NULL, cs); --} -- --static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, -- struct io_kiocb **link, struct io_comp_state *cs) --{ -- struct io_ring_ctx *ctx = req->ctx; -- int ret; -- -- /* -- * If we already have a head request, queue this one for async -- * submittal once the head completes. If we don't have a head but -- * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be -- * submitted sync once the chain is complete. If none of those -- * conditions are true (normal request), then just queue it. -- */ -- if (*link) { -- struct io_kiocb *head = *link; -- -- /* -- * Taking sequential execution of a link, draining both sides -- * of the link also fullfils IOSQE_IO_DRAIN semantics for all -- * requests in the link. So, it drains the head and the -- * next after the link request. The last one is done via -- * drain_next flag to persist the effect across calls. -- */ -- if (req->flags & REQ_F_IO_DRAIN) { -- head->flags |= REQ_F_IO_DRAIN; -- ctx->drain_next = 1; -- } -- ret = io_req_defer_prep(req, sqe); -- if (unlikely(ret)) { -- /* fail even hard links since we don't submit */ -- head->flags |= REQ_F_FAIL_LINK; -- return ret; -- } -- trace_io_uring_link(ctx, req, head); -- list_add_tail(&req->link_list, &head->link_list); -- -- /* last request of a link, enqueue the link */ -- if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) { -- io_queue_link_head(head, cs); -- *link = NULL; -- } -- } else { -- if (unlikely(ctx->drain_next)) { -- req->flags |= REQ_F_IO_DRAIN; -- ctx->drain_next = 0; -- } -- if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { -- req->flags |= REQ_F_LINK_HEAD; -- INIT_LIST_HEAD(&req->link_list); -- -- ret = io_req_defer_prep(req, sqe); -- if (unlikely(ret)) -- req->flags |= REQ_F_FAIL_LINK; -- *link = req; -- } else { -- io_queue_sqe(req, sqe, cs); -- } -- } -- -- return 0; --} -- --/* -- * Batched submission is done, ensure local IO is flushed out. -- */ --static void io_submit_state_end(struct io_submit_state *state) --{ -- if (!list_empty(&state->comp.list)) -- io_submit_flush_completions(&state->comp); -- blk_finish_plug(&state->plug); -- io_state_file_put(state); -- if (state->free_reqs) -- kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs); --} -- --/* -- * Start submission side cache. -- */ --static void io_submit_state_start(struct io_submit_state *state, -- struct io_ring_ctx *ctx, unsigned int max_ios) --{ -- blk_start_plug(&state->plug); -- state->comp.nr = 0; -- INIT_LIST_HEAD(&state->comp.list); -- state->comp.ctx = ctx; -- state->free_reqs = 0; -- state->file = NULL; -- state->ios_left = max_ios; --} -- --static void io_commit_sqring(struct io_ring_ctx *ctx) --{ -- struct io_rings *rings = ctx->rings; -- -- /* -- * Ensure any loads from the SQEs are done at this point, -- * since once we write the new head, the application could -- * write new data to them. -- */ -- smp_store_release(&rings->sq.head, ctx->cached_sq_head); --} -- --/* -- * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory -- * that is mapped by userspace. This means that care needs to be taken to -- * ensure that reads are stable, as we cannot rely on userspace always -- * being a good citizen. If members of the sqe are validated and then later -- * used, it's important that those reads are done through READ_ONCE() to -- * prevent a re-load down the line. -- */ --static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) --{ -- u32 *sq_array = ctx->sq_array; -- unsigned head; -- -- /* -- * The cached sq head (or cq tail) serves two purposes: -- * -- * 1) allows us to batch the cost of updating the user visible -- * head updates. -- * 2) allows the kernel side to track the head on its own, even -- * though the application is the one updating it. -- */ -- head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]); -- if (likely(head < ctx->sq_entries)) -- return &ctx->sq_sqes[head]; -- -- /* drop invalid entries */ -- ctx->cached_sq_dropped++; -- WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped); -- return NULL; --} -- --static inline void io_consume_sqe(struct io_ring_ctx *ctx) --{ -- ctx->cached_sq_head++; --} -- --/* -- * Check SQE restrictions (opcode and flags). -- * -- * Returns 'true' if SQE is allowed, 'false' otherwise. -- */ --static inline bool io_check_restriction(struct io_ring_ctx *ctx, -- struct io_kiocb *req, -- unsigned int sqe_flags) --{ -- if (!ctx->restricted) -- return true; -- -- if (!test_bit(req->opcode, ctx->restrictions.sqe_op)) -- return false; -- -- if ((sqe_flags & ctx->restrictions.sqe_flags_required) != -- ctx->restrictions.sqe_flags_required) -- return false; -- -- if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed | -- ctx->restrictions.sqe_flags_required)) -- return false; -- -- return true; --} -- --#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ -- IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ -- IOSQE_BUFFER_SELECT) -- --static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, -- const struct io_uring_sqe *sqe, -- struct io_submit_state *state) --{ -- unsigned int sqe_flags; -- int id, ret; -- -- req->opcode = READ_ONCE(sqe->opcode); -- req->user_data = READ_ONCE(sqe->user_data); -- req->async_data = NULL; -- req->file = NULL; -- req->ctx = ctx; -- req->flags = 0; -- /* one is dropped after submission, the other at completion */ -- refcount_set(&req->refs, 2); -- req->task = current; -- req->result = 0; -- -- if (unlikely(req->opcode >= IORING_OP_LAST)) -- return -EINVAL; -- -- if (unlikely(io_sq_thread_acquire_mm(ctx, req))) -- return -EFAULT; -- -- sqe_flags = READ_ONCE(sqe->flags); -- /* enforce forwards compatibility on users */ -- if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) -- return -EINVAL; -- -- if (unlikely(!io_check_restriction(ctx, req, sqe_flags))) -- return -EACCES; -- -- if ((sqe_flags & IOSQE_BUFFER_SELECT) && -- !io_op_defs[req->opcode].buffer_select) -- return -EOPNOTSUPP; -- -- id = READ_ONCE(sqe->personality); -- if (id) { -- struct io_identity *iod; -- -- iod = xa_load(&ctx->personalities, id); -- if (unlikely(!iod)) -- return -EINVAL; -- refcount_inc(&iod->count); -- -- __io_req_init_async(req); -- get_cred(iod->creds); -- req->work.identity = iod; -- req->work.flags |= IO_WQ_WORK_CREDS; -- } -- -- /* same numerical values with corresponding REQ_F_*, safe to copy */ -- req->flags |= sqe_flags; -- -- if (!io_op_defs[req->opcode].needs_file) -- return 0; -- -- ret = io_req_set_file(state, req, READ_ONCE(sqe->fd)); -- state->ios_left--; -- return ret; --} -- --static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) --{ -- struct io_submit_state state; -- struct io_kiocb *link = NULL; -- int i, submitted = 0; -- -- /* if we have a backlog and couldn't flush it all, return BUSY */ -- if (test_bit(0, &ctx->sq_check_overflow)) { -- if (!__io_cqring_overflow_flush(ctx, false, NULL, NULL)) -- return -EBUSY; -- } -- -- /* make sure SQ entry isn't read before tail */ -- nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx)); -- -- if (!percpu_ref_tryget_many(&ctx->refs, nr)) -- return -EAGAIN; -- -- percpu_counter_add(¤t->io_uring->inflight, nr); -- refcount_add(nr, ¤t->usage); -- -- io_submit_state_start(&state, ctx, nr); -- -- for (i = 0; i < nr; i++) { -- const struct io_uring_sqe *sqe; -- struct io_kiocb *req; -- int err; -- -- sqe = io_get_sqe(ctx); -- if (unlikely(!sqe)) { -- io_consume_sqe(ctx); -- break; -- } -- req = io_alloc_req(ctx, &state); -- if (unlikely(!req)) { -- if (!submitted) -- submitted = -EAGAIN; -- break; -- } -- io_consume_sqe(ctx); -- /* will complete beyond this point, count as submitted */ -- submitted++; -- -- err = io_init_req(ctx, req, sqe, &state); -- if (unlikely(err)) { --fail_req: -- io_put_req(req); -- io_req_complete(req, err); -- break; -- } -- -- trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, -- true, io_async_submit(ctx)); -- err = io_submit_sqe(req, sqe, &link, &state.comp); -- if (err) -- goto fail_req; -- } -- -- if (unlikely(submitted != nr)) { -- int ref_used = (submitted == -EAGAIN) ? 0 : submitted; -- struct io_uring_task *tctx = current->io_uring; -- int unused = nr - ref_used; -- -- percpu_ref_put_many(&ctx->refs, unused); -- percpu_counter_sub(&tctx->inflight, unused); -- put_task_struct_many(current, unused); -- } -- if (link) -- io_queue_link_head(link, &state.comp); -- io_submit_state_end(&state); -- -- /* Commit SQ ring head once we've consumed and submitted all SQEs */ -- io_commit_sqring(ctx); -- -- return submitted; --} -- --static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx) --{ -- /* Tell userspace we may need a wakeup call */ -- spin_lock_irq(&ctx->completion_lock); -- ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP; -- spin_unlock_irq(&ctx->completion_lock); --} -- --static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx) --{ -- spin_lock_irq(&ctx->completion_lock); -- ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP; -- spin_unlock_irq(&ctx->completion_lock); --} -- --static int io_sq_wake_function(struct wait_queue_entry *wqe, unsigned mode, -- int sync, void *key) --{ -- struct io_ring_ctx *ctx = container_of(wqe, struct io_ring_ctx, sqo_wait_entry); -- int ret; -- -- ret = autoremove_wake_function(wqe, mode, sync, key); -- if (ret) { -- unsigned long flags; -- -- spin_lock_irqsave(&ctx->completion_lock, flags); -- ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP; -- spin_unlock_irqrestore(&ctx->completion_lock, flags); -- } -- return ret; --} -- --enum sq_ret { -- SQT_IDLE = 1, -- SQT_SPIN = 2, -- SQT_DID_WORK = 4, --}; -- --static enum sq_ret __io_sq_thread(struct io_ring_ctx *ctx, -- unsigned long start_jiffies, bool cap_entries) --{ -- unsigned long timeout = start_jiffies + ctx->sq_thread_idle; -- struct io_sq_data *sqd = ctx->sq_data; -- unsigned int to_submit; -- int ret = 0; -- --again: -- if (!list_empty(&ctx->iopoll_list)) { -- unsigned nr_events = 0; -- -- mutex_lock(&ctx->uring_lock); -- if (!list_empty(&ctx->iopoll_list) && !need_resched()) -- io_do_iopoll(ctx, &nr_events, 0); -- mutex_unlock(&ctx->uring_lock); -- } -- -- to_submit = io_sqring_entries(ctx); -- -- /* -- * If submit got -EBUSY, flag us as needing the application -- * to enter the kernel to reap and flush events. -- */ -- if (!to_submit || ret == -EBUSY || need_resched()) { -- /* -- * Drop cur_mm before scheduling, we can't hold it for -- * long periods (or over schedule()). Do this before -- * adding ourselves to the waitqueue, as the unuse/drop -- * may sleep. -- */ -- io_sq_thread_drop_mm(); -- -- /* -- * We're polling. If we're within the defined idle -- * period, then let us spin without work before going -- * to sleep. The exception is if we got EBUSY doing -- * more IO, we should wait for the application to -- * reap events and wake us up. -- */ -- if (!list_empty(&ctx->iopoll_list) || need_resched() || -- (!time_after(jiffies, timeout) && ret != -EBUSY && -- !percpu_ref_is_dying(&ctx->refs))) -- return SQT_SPIN; -- -- prepare_to_wait(&sqd->wait, &ctx->sqo_wait_entry, -- TASK_INTERRUPTIBLE); -- -- /* -- * While doing polled IO, before going to sleep, we need -- * to check if there are new reqs added to iopoll_list, -- * it is because reqs may have been punted to io worker -- * and will be added to iopoll_list later, hence check -- * the iopoll_list again. -- */ -- if ((ctx->flags & IORING_SETUP_IOPOLL) && -- !list_empty_careful(&ctx->iopoll_list)) { -- finish_wait(&sqd->wait, &ctx->sqo_wait_entry); -- goto again; -- } -- -- to_submit = io_sqring_entries(ctx); -- if (!to_submit || ret == -EBUSY) -- return SQT_IDLE; -- } -- -- finish_wait(&sqd->wait, &ctx->sqo_wait_entry); -- io_ring_clear_wakeup_flag(ctx); -- -- /* if we're handling multiple rings, cap submit size for fairness */ -- if (cap_entries && to_submit > 8) -- to_submit = 8; -- -- mutex_lock(&ctx->uring_lock); -- if (likely(!percpu_ref_is_dying(&ctx->refs) && !ctx->sqo_dead)) -- ret = io_submit_sqes(ctx, to_submit); -- mutex_unlock(&ctx->uring_lock); -- -- if (!io_sqring_full(ctx) && wq_has_sleeper(&ctx->sqo_sq_wait)) -- wake_up(&ctx->sqo_sq_wait); -- -- return SQT_DID_WORK; --} -- --static void io_sqd_init_new(struct io_sq_data *sqd) --{ -- struct io_ring_ctx *ctx; -- -- while (!list_empty(&sqd->ctx_new_list)) { -- ctx = list_first_entry(&sqd->ctx_new_list, struct io_ring_ctx, sqd_list); -- init_wait(&ctx->sqo_wait_entry); -- ctx->sqo_wait_entry.func = io_sq_wake_function; -- list_move_tail(&ctx->sqd_list, &sqd->ctx_list); -- complete(&ctx->sq_thread_comp); -- } --} -- --static int io_sq_thread(void *data) --{ -- struct cgroup_subsys_state *cur_css = NULL; -- const struct cred *old_cred = NULL; -- struct io_sq_data *sqd = data; -- struct io_ring_ctx *ctx; -- unsigned long start_jiffies; -- -- start_jiffies = jiffies; -- while (!kthread_should_stop()) { -- enum sq_ret ret = 0; -- bool cap_entries; -- -- /* -- * Any changes to the sqd lists are synchronized through the -- * kthread parking. This synchronizes the thread vs users, -- * the users are synchronized on the sqd->ctx_lock. -- */ -- if (kthread_should_park()) { -- kthread_parkme(); -- /* -- * When sq thread is unparked, in case the previous park operation -- * comes from io_put_sq_data(), which means that sq thread is going -- * to be stopped, so here needs to have a check. -- */ -- if (kthread_should_stop()) -- break; -- } -- -- if (unlikely(!list_empty(&sqd->ctx_new_list))) -- io_sqd_init_new(sqd); -- -- cap_entries = !list_is_singular(&sqd->ctx_list); -- -- list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { -- if (current->cred != ctx->creds) { -- if (old_cred) -- revert_creds(old_cred); -- old_cred = override_creds(ctx->creds); -- } -- io_sq_thread_associate_blkcg(ctx, &cur_css); --#ifdef CONFIG_AUDIT -- current->loginuid = ctx->loginuid; -- current->sessionid = ctx->sessionid; --#endif -- -- ret |= __io_sq_thread(ctx, start_jiffies, cap_entries); -- -- io_sq_thread_drop_mm(); -- } -- -- if (ret & SQT_SPIN) { -- io_run_task_work(); -- io_sq_thread_drop_mm(); -- cond_resched(); -- } else if (ret == SQT_IDLE) { -- if (kthread_should_park()) -- continue; -- list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) -- io_ring_set_wakeup_flag(ctx); -- schedule(); -- start_jiffies = jiffies; -- list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) -- io_ring_clear_wakeup_flag(ctx); -- } -- } -- -- io_run_task_work(); -- io_sq_thread_drop_mm(); -- -- if (cur_css) -- io_sq_thread_unassociate_blkcg(); -- if (old_cred) -- revert_creds(old_cred); -- -- kthread_parkme(); -- -- return 0; --} -- --struct io_wait_queue { -- struct wait_queue_entry wq; -- struct io_ring_ctx *ctx; -- unsigned to_wait; -- unsigned nr_timeouts; --}; -- --static inline bool io_should_wake(struct io_wait_queue *iowq) --{ -- struct io_ring_ctx *ctx = iowq->ctx; -- -- /* -- * Wake up if we have enough events, or if a timeout occurred since we -- * started waiting. For timeouts, we always want to return to userspace, -- * regardless of event count. -- */ -- return io_cqring_events(ctx) >= iowq->to_wait || -- atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; --} -- --static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, -- int wake_flags, void *key) --{ -- struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue, -- wq); -- -- /* -- * Cannot safely flush overflowed CQEs from here, ensure we wake up -- * the task, and the next invocation will do it. -- */ -- if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->cq_check_overflow)) -- return autoremove_wake_function(curr, mode, wake_flags, key); -- return -1; --} -- --static int io_run_task_work_sig(void) --{ -- if (io_run_task_work()) -- return 1; -- if (!signal_pending(current)) -- return 0; -- if (current->jobctl & JOBCTL_TASK_WORK) { -- spin_lock_irq(¤t->sighand->siglock); -- current->jobctl &= ~JOBCTL_TASK_WORK; -- recalc_sigpending(); -- spin_unlock_irq(¤t->sighand->siglock); -- return 1; -- } -- return -EINTR; --} -- --/* -- * Wait until events become available, if we don't already have some. The -- * application must reap them itself, as they reside on the shared cq ring. -- */ --static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, -- const sigset_t __user *sig, size_t sigsz) --{ -- struct io_wait_queue iowq = { -- .wq = { -- .private = current, -- .func = io_wake_function, -- .entry = LIST_HEAD_INIT(iowq.wq.entry), -- }, -- .ctx = ctx, -- .to_wait = min_events, -- }; -- struct io_rings *rings = ctx->rings; -- int ret = 0; -- -- do { -- io_cqring_overflow_flush(ctx, false, NULL, NULL); -- if (io_cqring_events(ctx) >= min_events) -- return 0; -- if (!io_run_task_work()) -- break; -- } while (1); -- -- if (sig) { --#ifdef CONFIG_COMPAT -- if (in_compat_syscall()) -- ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig, -- sigsz); -- else --#endif -- ret = set_user_sigmask(sig, sigsz); -- -- if (ret) -- return ret; -- } -- -- iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts); -- trace_io_uring_cqring_wait(ctx, min_events); -- do { -- io_cqring_overflow_flush(ctx, false, NULL, NULL); -- prepare_to_wait_exclusive(&ctx->wait, &iowq.wq, -- TASK_INTERRUPTIBLE); -- /* make sure we run task_work before checking for signals */ -- ret = io_run_task_work_sig(); -- if (ret > 0) { -- finish_wait(&ctx->wait, &iowq.wq); -- continue; -- } -- else if (ret < 0) -- break; -- if (io_should_wake(&iowq)) -- break; -- if (test_bit(0, &ctx->cq_check_overflow)) { -- finish_wait(&ctx->wait, &iowq.wq); -- continue; -- } -- schedule(); -- } while (1); -- finish_wait(&ctx->wait, &iowq.wq); -- -- restore_saved_sigmask_unless(ret == -EINTR); -- -- return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0; --} -- --static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) --{ --#if defined(CONFIG_UNIX) -- if (ctx->ring_sock) { -- struct sock *sock = ctx->ring_sock->sk; -- struct sk_buff *skb; -- -- while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL) -- kfree_skb(skb); -- } --#else -- int i; -- -- for (i = 0; i < ctx->nr_user_files; i++) { -- struct file *file; -- -- file = io_file_from_index(ctx, i); -- if (file) -- fput(file); -- } --#endif --} -- --static void io_file_ref_kill(struct percpu_ref *ref) --{ -- struct fixed_file_data *data; -- -- data = container_of(ref, struct fixed_file_data, refs); -- complete(&data->done); --} -- --static void io_sqe_files_set_node(struct fixed_file_data *file_data, -- struct fixed_file_ref_node *ref_node) --{ -- spin_lock_bh(&file_data->lock); -- file_data->node = ref_node; -- list_add_tail(&ref_node->node, &file_data->ref_list); -- spin_unlock_bh(&file_data->lock); -- percpu_ref_get(&file_data->refs); --} -- -- --static void io_sqe_files_kill_node(struct fixed_file_data *data) --{ -- struct fixed_file_ref_node *ref_node = NULL; -- -- spin_lock_bh(&data->lock); -- ref_node = data->node; -- spin_unlock_bh(&data->lock); -- if (ref_node) -- percpu_ref_kill(&ref_node->refs); --} -- --static int io_file_ref_quiesce(struct fixed_file_data *data, -- struct io_ring_ctx *ctx) --{ -- int ret; -- struct fixed_file_ref_node *backup_node; -- -- if (data->quiesce) -- return -ENXIO; -- -- data->quiesce = true; -- do { -- backup_node = alloc_fixed_file_ref_node(ctx); -- if (!backup_node) -- break; -- -- io_sqe_files_kill_node(data); -- percpu_ref_kill(&data->refs); -- flush_delayed_work(&ctx->file_put_work); -- -- ret = wait_for_completion_interruptible(&data->done); -- if (!ret) -- break; -- -- percpu_ref_resurrect(&data->refs); -- io_sqe_files_set_node(data, backup_node); -- backup_node = NULL; -- reinit_completion(&data->done); -- mutex_unlock(&ctx->uring_lock); -- ret = io_run_task_work_sig(); -- mutex_lock(&ctx->uring_lock); -- -- if (ret < 0) -- break; -- backup_node = alloc_fixed_file_ref_node(ctx); -- ret = -ENOMEM; -- if (!backup_node) -- break; -- } while (1); -- data->quiesce = false; -- -- if (backup_node) -- destroy_fixed_file_ref_node(backup_node); -- return ret; --} -- --static int io_sqe_files_unregister(struct io_ring_ctx *ctx) --{ -- struct fixed_file_data *data = ctx->file_data; -- unsigned nr_tables, i; -- int ret; -- -- /* -- * percpu_ref_is_dying() is to stop parallel files unregister -- * Since we possibly drop uring lock later in this function to -- * run task work. -- */ -- if (!data || percpu_ref_is_dying(&data->refs)) -- return -ENXIO; -- ret = io_file_ref_quiesce(data, ctx); -- if (ret) -- return ret; -- -- __io_sqe_files_unregister(ctx); -- nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE); -- for (i = 0; i < nr_tables; i++) -- kfree(data->table[i].files); -- kfree(data->table); -- percpu_ref_exit(&data->refs); -- kfree(data); -- ctx->file_data = NULL; -- ctx->nr_user_files = 0; -- return 0; --} -- --static void io_put_sq_data(struct io_sq_data *sqd) --{ -- if (refcount_dec_and_test(&sqd->refs)) { -- /* -- * The park is a bit of a work-around, without it we get -- * warning spews on shutdown with SQPOLL set and affinity -- * set to a single CPU. -- */ -- if (sqd->thread) { -- kthread_park(sqd->thread); -- kthread_stop(sqd->thread); -- } -- -- kfree(sqd); -- } --} -- --static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p) --{ -- struct io_ring_ctx *ctx_attach; -- struct io_sq_data *sqd; -- struct fd f; -- -- f = fdget(p->wq_fd); -- if (!f.file) -- return ERR_PTR(-ENXIO); -- if (f.file->f_op != &io_uring_fops) { -- fdput(f); -- return ERR_PTR(-EINVAL); -- } -- -- ctx_attach = f.file->private_data; -- sqd = ctx_attach->sq_data; -- if (!sqd) { -- fdput(f); -- return ERR_PTR(-EINVAL); -- } -- -- refcount_inc(&sqd->refs); -- fdput(f); -- return sqd; --} -- --static struct io_sq_data *io_get_sq_data(struct io_uring_params *p) --{ -- struct io_sq_data *sqd; -- -- if (p->flags & IORING_SETUP_ATTACH_WQ) -- return io_attach_sq_data(p); -- -- sqd = kzalloc(sizeof(*sqd), GFP_KERNEL); -- if (!sqd) -- return ERR_PTR(-ENOMEM); -- -- refcount_set(&sqd->refs, 1); -- INIT_LIST_HEAD(&sqd->ctx_list); -- INIT_LIST_HEAD(&sqd->ctx_new_list); -- mutex_init(&sqd->ctx_lock); -- mutex_init(&sqd->lock); -- init_waitqueue_head(&sqd->wait); -- return sqd; --} -- --static void io_sq_thread_unpark(struct io_sq_data *sqd) -- __releases(&sqd->lock) --{ -- if (!sqd->thread) -- return; -- kthread_unpark(sqd->thread); -- mutex_unlock(&sqd->lock); --} -- --static void io_sq_thread_park(struct io_sq_data *sqd) -- __acquires(&sqd->lock) --{ -- if (!sqd->thread) -- return; -- mutex_lock(&sqd->lock); -- kthread_park(sqd->thread); --} -- --static void io_sq_thread_stop(struct io_ring_ctx *ctx) --{ -- struct io_sq_data *sqd = ctx->sq_data; -- -- if (sqd) { -- if (sqd->thread) { -- /* -- * We may arrive here from the error branch in -- * io_sq_offload_create() where the kthread is created -- * without being waked up, thus wake it up now to make -- * sure the wait will complete. -- */ -- wake_up_process(sqd->thread); -- wait_for_completion(&ctx->sq_thread_comp); -- -- io_sq_thread_park(sqd); -- } -- -- mutex_lock(&sqd->ctx_lock); -- list_del(&ctx->sqd_list); -- mutex_unlock(&sqd->ctx_lock); -- -- if (sqd->thread) { -- finish_wait(&sqd->wait, &ctx->sqo_wait_entry); -- io_sq_thread_unpark(sqd); -- } -- -- io_put_sq_data(sqd); -- ctx->sq_data = NULL; -- } --} -- --static void io_finish_async(struct io_ring_ctx *ctx) --{ -- io_sq_thread_stop(ctx); -- -- if (ctx->io_wq) { -- io_wq_destroy(ctx->io_wq); -- ctx->io_wq = NULL; -- } --} -- --#if defined(CONFIG_UNIX) --/* -- * Ensure the UNIX gc is aware of our file set, so we are certain that -- * the io_uring can be safely unregistered on process exit, even if we have -- * loops in the file referencing. -- */ --static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset) --{ -- struct sock *sk = ctx->ring_sock->sk; -- struct scm_fp_list *fpl; -- struct sk_buff *skb; -- int i, nr_files; -- -- fpl = kzalloc(sizeof(*fpl), GFP_KERNEL); -- if (!fpl) -- return -ENOMEM; -- -- skb = alloc_skb(0, GFP_KERNEL); -- if (!skb) { -- kfree(fpl); -- return -ENOMEM; -- } -- -- skb->sk = sk; -- skb->scm_io_uring = 1; -- -- nr_files = 0; -- fpl->user = get_uid(ctx->user); -- for (i = 0; i < nr; i++) { -- struct file *file = io_file_from_index(ctx, i + offset); -- -- if (!file) -- continue; -- fpl->fp[nr_files] = get_file(file); -- unix_inflight(fpl->user, fpl->fp[nr_files]); -- nr_files++; -- } -- -- if (nr_files) { -- fpl->max = SCM_MAX_FD; -- fpl->count = nr_files; -- UNIXCB(skb).fp = fpl; -- skb->destructor = unix_destruct_scm; -- refcount_add(skb->truesize, &sk->sk_wmem_alloc); -- skb_queue_head(&sk->sk_receive_queue, skb); -- -- for (i = 0; i < nr; i++) { -- struct file *file = io_file_from_index(ctx, i + offset); -- -- if (file) -- fput(file); -- } -- } else { -- kfree_skb(skb); -- free_uid(fpl->user); -- kfree(fpl); -- } -- -- return 0; --} -- --/* -- * If UNIX sockets are enabled, fd passing can cause a reference cycle which -- * causes regular reference counting to break down. We rely on the UNIX -- * garbage collection to take care of this problem for us. -- */ --static int io_sqe_files_scm(struct io_ring_ctx *ctx) --{ -- unsigned left, total; -- int ret = 0; -- -- total = 0; -- left = ctx->nr_user_files; -- while (left) { -- unsigned this_files = min_t(unsigned, left, SCM_MAX_FD); -- -- ret = __io_sqe_files_scm(ctx, this_files, total); -- if (ret) -- break; -- left -= this_files; -- total += this_files; -- } -- -- if (!ret) -- return 0; -- -- while (total < ctx->nr_user_files) { -- struct file *file = io_file_from_index(ctx, total); -- -- if (file) -- fput(file); -- total++; -- } -- -- return ret; --} --#else --static int io_sqe_files_scm(struct io_ring_ctx *ctx) --{ -- return 0; --} --#endif -- --static int io_sqe_alloc_file_tables(struct fixed_file_data *file_data, -- unsigned nr_tables, unsigned nr_files) --{ -- int i; -- -- for (i = 0; i < nr_tables; i++) { -- struct fixed_file_table *table = &file_data->table[i]; -- unsigned this_files; -- -- this_files = min(nr_files, IORING_MAX_FILES_TABLE); -- table->files = kcalloc(this_files, sizeof(struct file *), -- GFP_KERNEL_ACCOUNT); -- if (!table->files) -- break; -- nr_files -= this_files; -- } -- -- if (i == nr_tables) -- return 0; -- -- for (i = 0; i < nr_tables; i++) { -- struct fixed_file_table *table = &file_data->table[i]; -- kfree(table->files); -- } -- return 1; --} -- --static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file) --{ --#if defined(CONFIG_UNIX) -- struct sock *sock = ctx->ring_sock->sk; -- struct sk_buff_head list, *head = &sock->sk_receive_queue; -- struct sk_buff *skb; -- int i; -- -- __skb_queue_head_init(&list); -- -- /* -- * Find the skb that holds this file in its SCM_RIGHTS. When found, -- * remove this entry and rearrange the file array. -- */ -- skb = skb_dequeue(head); -- while (skb) { -- struct scm_fp_list *fp; -- -- fp = UNIXCB(skb).fp; -- for (i = 0; i < fp->count; i++) { -- int left; -- -- if (fp->fp[i] != file) -- continue; -- -- unix_notinflight(fp->user, fp->fp[i]); -- left = fp->count - 1 - i; -- if (left) { -- memmove(&fp->fp[i], &fp->fp[i + 1], -- left * sizeof(struct file *)); -- } -- fp->count--; -- if (!fp->count) { -- kfree_skb(skb); -- skb = NULL; -- } else { -- __skb_queue_tail(&list, skb); -- } -- fput(file); -- file = NULL; -- break; -- } -- -- if (!file) -- break; -- -- __skb_queue_tail(&list, skb); -- -- skb = skb_dequeue(head); -- } -- -- if (skb_peek(&list)) { -- spin_lock_irq(&head->lock); -- while ((skb = __skb_dequeue(&list)) != NULL) -- __skb_queue_tail(head, skb); -- spin_unlock_irq(&head->lock); -- } --#else -- fput(file); --#endif --} -- --struct io_file_put { -- struct list_head list; -- struct file *file; --}; -- --static void __io_file_put_work(struct fixed_file_ref_node *ref_node) --{ -- struct fixed_file_data *file_data = ref_node->file_data; -- struct io_ring_ctx *ctx = file_data->ctx; -- struct io_file_put *pfile, *tmp; -- -- list_for_each_entry_safe(pfile, tmp, &ref_node->file_list, list) { -- list_del(&pfile->list); -- io_ring_file_put(ctx, pfile->file); -- kfree(pfile); -- } -- -- percpu_ref_exit(&ref_node->refs); -- kfree(ref_node); -- percpu_ref_put(&file_data->refs); --} -- --static void io_file_put_work(struct work_struct *work) --{ -- struct io_ring_ctx *ctx; -- struct llist_node *node; -- -- ctx = container_of(work, struct io_ring_ctx, file_put_work.work); -- node = llist_del_all(&ctx->file_put_llist); -- -- while (node) { -- struct fixed_file_ref_node *ref_node; -- struct llist_node *next = node->next; -- -- ref_node = llist_entry(node, struct fixed_file_ref_node, llist); -- __io_file_put_work(ref_node); -- node = next; -- } --} -- --static void io_file_data_ref_zero(struct percpu_ref *ref) --{ -- struct fixed_file_ref_node *ref_node; -- struct fixed_file_data *data; -- struct io_ring_ctx *ctx; -- bool first_add = false; -- int delay = HZ; -- -- ref_node = container_of(ref, struct fixed_file_ref_node, refs); -- data = ref_node->file_data; -- ctx = data->ctx; -- -- spin_lock_bh(&data->lock); -- ref_node->done = true; -- -- while (!list_empty(&data->ref_list)) { -- ref_node = list_first_entry(&data->ref_list, -- struct fixed_file_ref_node, node); -- /* recycle ref nodes in order */ -- if (!ref_node->done) -- break; -- list_del(&ref_node->node); -- first_add |= llist_add(&ref_node->llist, &ctx->file_put_llist); -- } -- spin_unlock_bh(&data->lock); -- -- if (percpu_ref_is_dying(&data->refs)) -- delay = 0; -- -- if (!delay) -- mod_delayed_work(system_wq, &ctx->file_put_work, 0); -- else if (first_add) -- queue_delayed_work(system_wq, &ctx->file_put_work, delay); --} -- --static struct fixed_file_ref_node *alloc_fixed_file_ref_node( -- struct io_ring_ctx *ctx) --{ -- struct fixed_file_ref_node *ref_node; -- -- ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL); -- if (!ref_node) -- return NULL; -- -- if (percpu_ref_init(&ref_node->refs, io_file_data_ref_zero, -- 0, GFP_KERNEL)) { -- kfree(ref_node); -- return NULL; -- } -- INIT_LIST_HEAD(&ref_node->node); -- INIT_LIST_HEAD(&ref_node->file_list); -- ref_node->file_data = ctx->file_data; -- ref_node->done = false; -- return ref_node; --} -- --static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node) --{ -- percpu_ref_exit(&ref_node->refs); -- kfree(ref_node); --} -- --static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, -- unsigned nr_args) --{ -- __s32 __user *fds = (__s32 __user *) arg; -- unsigned nr_tables, i; -- struct file *file; -- int fd, ret = -ENOMEM; -- struct fixed_file_ref_node *ref_node; -- struct fixed_file_data *file_data; -- -- if (ctx->file_data) -- return -EBUSY; -- if (!nr_args) -- return -EINVAL; -- if (nr_args > IORING_MAX_FIXED_FILES) -- return -EMFILE; -- if (nr_args > rlimit(RLIMIT_NOFILE)) -- return -EMFILE; -- -- file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL_ACCOUNT); -- if (!file_data) -- return -ENOMEM; -- file_data->ctx = ctx; -- init_completion(&file_data->done); -- INIT_LIST_HEAD(&file_data->ref_list); -- spin_lock_init(&file_data->lock); -- -- nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE); -- file_data->table = kcalloc(nr_tables, sizeof(*file_data->table), -- GFP_KERNEL_ACCOUNT); -- if (!file_data->table) -- goto out_free; -- -- if (percpu_ref_init(&file_data->refs, io_file_ref_kill, -- PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) -- goto out_free; -- -- if (io_sqe_alloc_file_tables(file_data, nr_tables, nr_args)) -- goto out_ref; -- ctx->file_data = file_data; -- -- for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { -- struct fixed_file_table *table; -- unsigned index; -- -- if (copy_from_user(&fd, &fds[i], sizeof(fd))) { -- ret = -EFAULT; -- goto out_fput; -- } -- /* allow sparse sets */ -- if (fd == -1) -- continue; -- -- file = fget(fd); -- ret = -EBADF; -- if (!file) -- goto out_fput; -- -- /* -- * Don't allow io_uring instances to be registered. If UNIX -- * isn't enabled, then this causes a reference cycle and this -- * instance can never get freed. If UNIX is enabled we'll -- * handle it just fine, but there's still no point in allowing -- * a ring fd as it doesn't support regular read/write anyway. -- */ -- if (file->f_op == &io_uring_fops) { -- fput(file); -- goto out_fput; -- } -- table = &file_data->table[i >> IORING_FILE_TABLE_SHIFT]; -- index = i & IORING_FILE_TABLE_MASK; -- table->files[index] = file; -- } -- -- ret = io_sqe_files_scm(ctx); -- if (ret) { -- io_sqe_files_unregister(ctx); -- return ret; -- } -- -- ref_node = alloc_fixed_file_ref_node(ctx); -- if (!ref_node) { -- io_sqe_files_unregister(ctx); -- return -ENOMEM; -- } -- -- io_sqe_files_set_node(file_data, ref_node); -- return ret; --out_fput: -- for (i = 0; i < ctx->nr_user_files; i++) { -- file = io_file_from_index(ctx, i); -- if (file) -- fput(file); -- } -- for (i = 0; i < nr_tables; i++) -- kfree(file_data->table[i].files); -- ctx->nr_user_files = 0; --out_ref: -- percpu_ref_exit(&file_data->refs); --out_free: -- kfree(file_data->table); -- kfree(file_data); -- ctx->file_data = NULL; -- return ret; --} -- --static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file, -- int index) --{ --#if defined(CONFIG_UNIX) -- struct sock *sock = ctx->ring_sock->sk; -- struct sk_buff_head *head = &sock->sk_receive_queue; -- struct sk_buff *skb; -- -- /* -- * See if we can merge this file into an existing skb SCM_RIGHTS -- * file set. If there's no room, fall back to allocating a new skb -- * and filling it in. -- */ -- spin_lock_irq(&head->lock); -- skb = skb_peek(head); -- if (skb) { -- struct scm_fp_list *fpl = UNIXCB(skb).fp; -- -- if (fpl->count < SCM_MAX_FD) { -- __skb_unlink(skb, head); -- spin_unlock_irq(&head->lock); -- fpl->fp[fpl->count] = get_file(file); -- unix_inflight(fpl->user, fpl->fp[fpl->count]); -- fpl->count++; -- spin_lock_irq(&head->lock); -- __skb_queue_head(head, skb); -- } else { -- skb = NULL; -- } -- } -- spin_unlock_irq(&head->lock); -- -- if (skb) { -- fput(file); -- return 0; -- } -- -- return __io_sqe_files_scm(ctx, 1, index); --#else -- return 0; --#endif --} -- --static int io_queue_file_removal(struct fixed_file_data *data, -- struct file *file) --{ -- struct io_file_put *pfile; -- struct fixed_file_ref_node *ref_node = data->node; -- -- pfile = kzalloc(sizeof(*pfile), GFP_KERNEL); -- if (!pfile) -- return -ENOMEM; -- -- pfile->file = file; -- list_add(&pfile->list, &ref_node->file_list); -- -- return 0; --} -- --static int __io_sqe_files_update(struct io_ring_ctx *ctx, -- struct io_uring_files_update *up, -- unsigned nr_args) --{ -- struct fixed_file_data *data = ctx->file_data; -- struct fixed_file_ref_node *ref_node; -- struct file *file; -- __s32 __user *fds; -- int fd, i, err; -- __u32 done; -- bool needs_switch = false; -- -- if (check_add_overflow(up->offset, nr_args, &done)) -- return -EOVERFLOW; -- if (done > ctx->nr_user_files) -- return -EINVAL; -- -- ref_node = alloc_fixed_file_ref_node(ctx); -- if (!ref_node) -- return -ENOMEM; -- -- done = 0; -- fds = u64_to_user_ptr(up->fds); -- while (nr_args) { -- struct fixed_file_table *table; -- unsigned index; -- -- err = 0; -- if (copy_from_user(&fd, &fds[done], sizeof(fd))) { -- err = -EFAULT; -- break; -- } -- i = array_index_nospec(up->offset, ctx->nr_user_files); -- table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; -- index = i & IORING_FILE_TABLE_MASK; -- if (table->files[index]) { -- file = table->files[index]; -- err = io_queue_file_removal(data, file); -- if (err) -- break; -- table->files[index] = NULL; -- needs_switch = true; -- } -- if (fd != -1) { -- file = fget(fd); -- if (!file) { -- err = -EBADF; -- break; -- } -- /* -- * Don't allow io_uring instances to be registered. If -- * UNIX isn't enabled, then this causes a reference -- * cycle and this instance can never get freed. If UNIX -- * is enabled we'll handle it just fine, but there's -- * still no point in allowing a ring fd as it doesn't -- * support regular read/write anyway. -- */ -- if (file->f_op == &io_uring_fops) { -- fput(file); -- err = -EBADF; -- break; -- } -- table->files[index] = file; -- err = io_sqe_file_register(ctx, file, i); -- if (err) { -- table->files[index] = NULL; -- fput(file); -- break; -- } -- } -- nr_args--; -- done++; -- up->offset++; -- } -- -- if (needs_switch) { -- percpu_ref_kill(&data->node->refs); -- io_sqe_files_set_node(data, ref_node); -- } else -- destroy_fixed_file_ref_node(ref_node); -- -- return done ? done : err; --} -- --static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg, -- unsigned nr_args) --{ -- struct io_uring_files_update up; -- -- if (!ctx->file_data) -- return -ENXIO; -- if (!nr_args) -- return -EINVAL; -- if (copy_from_user(&up, arg, sizeof(up))) -- return -EFAULT; -- if (up.resv) -- return -EINVAL; -- -- return __io_sqe_files_update(ctx, &up, nr_args); --} -- --static void io_free_work(struct io_wq_work *work) --{ -- struct io_kiocb *req = container_of(work, struct io_kiocb, work); -- -- /* Consider that io_steal_work() relies on this ref */ -- io_put_req(req); --} -- --static int io_init_wq_offload(struct io_ring_ctx *ctx, -- struct io_uring_params *p) --{ -- struct io_wq_data data; -- struct fd f; -- struct io_ring_ctx *ctx_attach; -- unsigned int concurrency; -- int ret = 0; -- -- data.user = ctx->user; -- data.free_work = io_free_work; -- data.do_work = io_wq_submit_work; -- -- if (!(p->flags & IORING_SETUP_ATTACH_WQ)) { -- /* Do QD, or 4 * CPUS, whatever is smallest */ -- concurrency = min(ctx->sq_entries, 4 * num_online_cpus()); -- -- ctx->io_wq = io_wq_create(concurrency, &data); -- if (IS_ERR(ctx->io_wq)) { -- ret = PTR_ERR(ctx->io_wq); -- ctx->io_wq = NULL; -- } -- return ret; -- } -- -- f = fdget(p->wq_fd); -- if (!f.file) -- return -EBADF; -- -- if (f.file->f_op != &io_uring_fops) { -- ret = -EINVAL; -- goto out_fput; -- } -- -- ctx_attach = f.file->private_data; -- /* @io_wq is protected by holding the fd */ -- if (!io_wq_get(ctx_attach->io_wq, &data)) { -- ret = -EINVAL; -- goto out_fput; -- } -- -- ctx->io_wq = ctx_attach->io_wq; --out_fput: -- fdput(f); -- return ret; --} -- --static int io_uring_alloc_task_context(struct task_struct *task) --{ -- struct io_uring_task *tctx; -- int ret; -- -- tctx = kmalloc(sizeof(*tctx), GFP_KERNEL); -- if (unlikely(!tctx)) -- return -ENOMEM; -- -- ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL); -- if (unlikely(ret)) { -- kfree(tctx); -- return ret; -- } -- -- xa_init(&tctx->xa); -- init_waitqueue_head(&tctx->wait); -- tctx->last = NULL; -- atomic_set(&tctx->in_idle, 0); -- tctx->sqpoll = false; -- io_init_identity(&tctx->__identity); -- tctx->identity = &tctx->__identity; -- task->io_uring = tctx; -- return 0; --} -- --void __io_uring_free(struct task_struct *tsk) --{ -- struct io_uring_task *tctx = tsk->io_uring; -- -- WARN_ON_ONCE(!xa_empty(&tctx->xa)); -- WARN_ON_ONCE(refcount_read(&tctx->identity->count) != 1); -- if (tctx->identity != &tctx->__identity) -- kfree(tctx->identity); -- percpu_counter_destroy(&tctx->inflight); -- kfree(tctx); -- tsk->io_uring = NULL; --} -- --static int io_sq_offload_create(struct io_ring_ctx *ctx, -- struct io_uring_params *p) --{ -- int ret; -- -- if (ctx->flags & IORING_SETUP_SQPOLL) { -- struct io_sq_data *sqd; -- -- ret = -EPERM; -- if (!capable(CAP_SYS_ADMIN)) -- goto err; -- -- sqd = io_get_sq_data(p); -- if (IS_ERR(sqd)) { -- ret = PTR_ERR(sqd); -- goto err; -- } -- -- ctx->sq_data = sqd; -- io_sq_thread_park(sqd); -- mutex_lock(&sqd->ctx_lock); -- list_add(&ctx->sqd_list, &sqd->ctx_new_list); -- mutex_unlock(&sqd->ctx_lock); -- io_sq_thread_unpark(sqd); -- -- ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle); -- if (!ctx->sq_thread_idle) -- ctx->sq_thread_idle = HZ; -- -- if (sqd->thread) -- goto done; -- -- if (p->flags & IORING_SETUP_SQ_AFF) { -- int cpu = p->sq_thread_cpu; -- -- ret = -EINVAL; -- if (cpu >= nr_cpu_ids) -- goto err; -- if (!cpu_online(cpu)) -- goto err; -- -- sqd->thread = kthread_create_on_cpu(io_sq_thread, sqd, -- cpu, "io_uring-sq"); -- } else { -- sqd->thread = kthread_create(io_sq_thread, sqd, -- "io_uring-sq"); -- } -- if (IS_ERR(sqd->thread)) { -- ret = PTR_ERR(sqd->thread); -- sqd->thread = NULL; -- goto err; -- } -- ret = io_uring_alloc_task_context(sqd->thread); -- if (ret) -- goto err; -- } else if (p->flags & IORING_SETUP_SQ_AFF) { -- /* Can't have SQ_AFF without SQPOLL */ -- ret = -EINVAL; -- goto err; -- } -- --done: -- ret = io_init_wq_offload(ctx, p); -- if (ret) -- goto err; -- -- return 0; --err: -- io_finish_async(ctx); -- return ret; --} -- --static void io_sq_offload_start(struct io_ring_ctx *ctx) --{ -- struct io_sq_data *sqd = ctx->sq_data; -- -- ctx->flags &= ~IORING_SETUP_R_DISABLED; -- if ((ctx->flags & IORING_SETUP_SQPOLL) && sqd && sqd->thread) -- wake_up_process(sqd->thread); --} -- --static inline void __io_unaccount_mem(struct user_struct *user, -- unsigned long nr_pages) --{ -- atomic_long_sub(nr_pages, &user->locked_vm); --} -- --static inline int __io_account_mem(struct user_struct *user, -- unsigned long nr_pages) --{ -- unsigned long page_limit, cur_pages, new_pages; -- -- /* Don't allow more pages than we can safely lock */ -- page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; -- -- do { -- cur_pages = atomic_long_read(&user->locked_vm); -- new_pages = cur_pages + nr_pages; -- if (new_pages > page_limit) -- return -ENOMEM; -- } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages, -- new_pages) != cur_pages); -- -- return 0; --} -- --static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages, -- enum io_mem_account acct) --{ -- if (ctx->limit_mem) -- __io_unaccount_mem(ctx->user, nr_pages); -- -- if (ctx->mm_account) { -- if (acct == ACCT_LOCKED) -- ctx->mm_account->locked_vm -= nr_pages; -- else if (acct == ACCT_PINNED) -- atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm); -- } --} -- --static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages, -- enum io_mem_account acct) --{ -- int ret; -- -- if (ctx->limit_mem) { -- ret = __io_account_mem(ctx->user, nr_pages); -- if (ret) -- return ret; -- } -- -- if (ctx->mm_account) { -- if (acct == ACCT_LOCKED) -- ctx->mm_account->locked_vm += nr_pages; -- else if (acct == ACCT_PINNED) -- atomic64_add(nr_pages, &ctx->mm_account->pinned_vm); -- } -- -- return 0; --} -- --static void io_mem_free(void *ptr) --{ -- struct page *page; -- -- if (!ptr) -- return; -- -- page = virt_to_head_page(ptr); -- if (put_page_testzero(page)) -- free_compound_page(page); --} -- --static void *io_mem_alloc(size_t size) --{ -- gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP | -- __GFP_NORETRY; -- -- return (void *) __get_free_pages(gfp_flags, get_order(size)); --} -- --static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries, -- size_t *sq_offset) --{ -- struct io_rings *rings; -- size_t off, sq_array_size; -- -- off = struct_size(rings, cqes, cq_entries); -- if (off == SIZE_MAX) -- return SIZE_MAX; -- --#ifdef CONFIG_SMP -- off = ALIGN(off, SMP_CACHE_BYTES); -- if (off == 0) -- return SIZE_MAX; --#endif -- -- if (sq_offset) -- *sq_offset = off; -- -- sq_array_size = array_size(sizeof(u32), sq_entries); -- if (sq_array_size == SIZE_MAX) -- return SIZE_MAX; -- -- if (check_add_overflow(off, sq_array_size, &off)) -- return SIZE_MAX; -- -- return off; --} -- --static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries) --{ -- size_t pages; -- -- pages = (size_t)1 << get_order( -- rings_size(sq_entries, cq_entries, NULL)); -- pages += (size_t)1 << get_order( -- array_size(sizeof(struct io_uring_sqe), sq_entries)); -- -- return pages; --} -- --static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx) --{ -- int i, j; -- -- if (!ctx->user_bufs) -- return -ENXIO; -- -- for (i = 0; i < ctx->nr_user_bufs; i++) { -- struct io_mapped_ubuf *imu = &ctx->user_bufs[i]; -- -- for (j = 0; j < imu->nr_bvecs; j++) -- unpin_user_page(imu->bvec[j].bv_page); -- -- if (imu->acct_pages) -- io_unaccount_mem(ctx, imu->acct_pages, ACCT_PINNED); -- kvfree(imu->bvec); -- imu->nr_bvecs = 0; -- } -- -- kfree(ctx->user_bufs); -- ctx->user_bufs = NULL; -- ctx->nr_user_bufs = 0; -- return 0; --} -- --static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst, -- void __user *arg, unsigned index) --{ -- struct iovec __user *src; -- --#ifdef CONFIG_COMPAT -- if (ctx->compat) { -- struct compat_iovec __user *ciovs; -- struct compat_iovec ciov; -- -- ciovs = (struct compat_iovec __user *) arg; -- if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov))) -- return -EFAULT; -- -- dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base); -- dst->iov_len = ciov.iov_len; -- return 0; -- } --#endif -- src = (struct iovec __user *) arg; -- if (copy_from_user(dst, &src[index], sizeof(*dst))) -- return -EFAULT; -- return 0; --} -- --/* -- * Not super efficient, but this is just a registration time. And we do cache -- * the last compound head, so generally we'll only do a full search if we don't -- * match that one. -- * -- * We check if the given compound head page has already been accounted, to -- * avoid double accounting it. This allows us to account the full size of the -- * page, not just the constituent pages of a huge page. -- */ --static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages, -- int nr_pages, struct page *hpage) --{ -- int i, j; -- -- /* check current page array */ -- for (i = 0; i < nr_pages; i++) { -- if (!PageCompound(pages[i])) -- continue; -- if (compound_head(pages[i]) == hpage) -- return true; -- } -- -- /* check previously registered pages */ -- for (i = 0; i < ctx->nr_user_bufs; i++) { -- struct io_mapped_ubuf *imu = &ctx->user_bufs[i]; -- -- for (j = 0; j < imu->nr_bvecs; j++) { -- if (!PageCompound(imu->bvec[j].bv_page)) -- continue; -- if (compound_head(imu->bvec[j].bv_page) == hpage) -- return true; -- } -- } -- -- return false; --} -- --static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages, -- int nr_pages, struct io_mapped_ubuf *imu, -- struct page **last_hpage) --{ -- int i, ret; -- -- for (i = 0; i < nr_pages; i++) { -- if (!PageCompound(pages[i])) { -- imu->acct_pages++; -- } else { -- struct page *hpage; -- -- hpage = compound_head(pages[i]); -- if (hpage == *last_hpage) -- continue; -- *last_hpage = hpage; -- if (headpage_already_acct(ctx, pages, i, hpage)) -- continue; -- imu->acct_pages += page_size(hpage) >> PAGE_SHIFT; -- } -- } -- -- if (!imu->acct_pages) -- return 0; -- -- ret = io_account_mem(ctx, imu->acct_pages, ACCT_PINNED); -- if (ret) -- imu->acct_pages = 0; -- return ret; --} -- --static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg, -- unsigned nr_args) --{ -- struct vm_area_struct **vmas = NULL; -- struct page **pages = NULL; -- struct page *last_hpage = NULL; -- int i, j, got_pages = 0; -- int ret = -EINVAL; -- -- if (ctx->user_bufs) -- return -EBUSY; -- if (!nr_args || nr_args > UIO_MAXIOV) -- return -EINVAL; -- -- ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf), -- GFP_KERNEL); -- if (!ctx->user_bufs) -- return -ENOMEM; -- -- for (i = 0; i < nr_args; i++) { -- struct io_mapped_ubuf *imu = &ctx->user_bufs[i]; -- unsigned long off, start, end, ubuf; -- int pret, nr_pages; -- struct iovec iov; -- size_t size; -- -- ret = io_copy_iov(ctx, &iov, arg, i); -- if (ret) -- goto err; -- -- /* -- * Don't impose further limits on the size and buffer -- * constraints here, we'll -EINVAL later when IO is -- * submitted if they are wrong. -- */ -- ret = -EFAULT; -- if (!iov.iov_base || !iov.iov_len) -- goto err; -- -- /* arbitrary limit, but we need something */ -- if (iov.iov_len > SZ_1G) -- goto err; -- -- ubuf = (unsigned long) iov.iov_base; -- end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT; -- start = ubuf >> PAGE_SHIFT; -- nr_pages = end - start; -- -- ret = 0; -- if (!pages || nr_pages > got_pages) { -- kvfree(vmas); -- kvfree(pages); -- pages = kvmalloc_array(nr_pages, sizeof(struct page *), -- GFP_KERNEL); -- vmas = kvmalloc_array(nr_pages, -- sizeof(struct vm_area_struct *), -- GFP_KERNEL); -- if (!pages || !vmas) { -- ret = -ENOMEM; -- goto err; -- } -- got_pages = nr_pages; -- } -- -- imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec), -- GFP_KERNEL); -- ret = -ENOMEM; -- if (!imu->bvec) -- goto err; -- -- ret = 0; -- mmap_read_lock(current->mm); -- pret = pin_user_pages(ubuf, nr_pages, -- FOLL_WRITE | FOLL_LONGTERM, -- pages, vmas); -- if (pret == nr_pages) { -- /* don't support file backed memory */ -- for (j = 0; j < nr_pages; j++) { -- struct vm_area_struct *vma = vmas[j]; -- -- if (vma->vm_file && -- !is_file_hugepages(vma->vm_file)) { -- ret = -EOPNOTSUPP; -- break; -- } -- } -- } else { -- ret = pret < 0 ? pret : -EFAULT; -- } -- mmap_read_unlock(current->mm); -- if (ret) { -- /* -- * if we did partial map, or found file backed vmas, -- * release any pages we did get -- */ -- if (pret > 0) -- unpin_user_pages(pages, pret); -- kvfree(imu->bvec); -- goto err; -- } -- -- ret = io_buffer_account_pin(ctx, pages, pret, imu, &last_hpage); -- if (ret) { -- unpin_user_pages(pages, pret); -- kvfree(imu->bvec); -- goto err; -- } -- -- off = ubuf & ~PAGE_MASK; -- size = iov.iov_len; -- for (j = 0; j < nr_pages; j++) { -- size_t vec_len; -- -- vec_len = min_t(size_t, size, PAGE_SIZE - off); -- imu->bvec[j].bv_page = pages[j]; -- imu->bvec[j].bv_len = vec_len; -- imu->bvec[j].bv_offset = off; -- off = 0; -- size -= vec_len; -- } -- /* store original address for later verification */ -- imu->ubuf = ubuf; -- imu->len = iov.iov_len; -- imu->nr_bvecs = nr_pages; -- -- ctx->nr_user_bufs++; -- } -- kvfree(pages); -- kvfree(vmas); -- return 0; --err: -- kvfree(pages); -- kvfree(vmas); -- io_sqe_buffer_unregister(ctx); -- return ret; --} -- --static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg) --{ -- __s32 __user *fds = arg; -- int fd; -- -- if (ctx->cq_ev_fd) -- return -EBUSY; -- -- if (copy_from_user(&fd, fds, sizeof(*fds))) -- return -EFAULT; -- -- ctx->cq_ev_fd = eventfd_ctx_fdget(fd); -- if (IS_ERR(ctx->cq_ev_fd)) { -- int ret = PTR_ERR(ctx->cq_ev_fd); -- ctx->cq_ev_fd = NULL; -- return ret; -- } -- -- return 0; --} -- --static int io_eventfd_unregister(struct io_ring_ctx *ctx) --{ -- if (ctx->cq_ev_fd) { -- eventfd_ctx_put(ctx->cq_ev_fd); -- ctx->cq_ev_fd = NULL; -- return 0; -- } -- -- return -ENXIO; --} -- --static void io_destroy_buffers(struct io_ring_ctx *ctx) --{ -- struct io_buffer *buf; -- unsigned long index; -- -- xa_for_each(&ctx->io_buffers, index, buf) -- __io_remove_buffers(ctx, buf, index, -1U); --} -- --static void io_ring_ctx_free(struct io_ring_ctx *ctx) --{ -- io_finish_async(ctx); -- io_sqe_buffer_unregister(ctx); -- -- if (ctx->sqo_task) { -- put_task_struct(ctx->sqo_task); -- ctx->sqo_task = NULL; -- } -- --#ifdef CONFIG_BLK_CGROUP -- if (ctx->sqo_blkcg_css) -- css_put(ctx->sqo_blkcg_css); --#endif -- -- mutex_lock(&ctx->uring_lock); -- io_sqe_files_unregister(ctx); -- mutex_unlock(&ctx->uring_lock); -- io_eventfd_unregister(ctx); -- io_destroy_buffers(ctx); -- --#if defined(CONFIG_UNIX) -- if (ctx->ring_sock) { -- ctx->ring_sock->file = NULL; /* so that iput() is called */ -- sock_release(ctx->ring_sock); -- } --#endif -- -- if (ctx->mm_account) { -- mmdrop(ctx->mm_account); -- ctx->mm_account = NULL; -- } -- -- io_mem_free(ctx->rings); -- io_mem_free(ctx->sq_sqes); -- -- percpu_ref_exit(&ctx->refs); -- free_uid(ctx->user); -- put_cred(ctx->creds); -- kfree(ctx->cancel_hash); -- kmem_cache_free(req_cachep, ctx->fallback_req); -- kfree(ctx); --} -- --static __poll_t io_uring_poll(struct file *file, poll_table *wait) --{ -- struct io_ring_ctx *ctx = file->private_data; -- __poll_t mask = 0; -- -- poll_wait(file, &ctx->cq_wait, wait); -- /* -- * synchronizes with barrier from wq_has_sleeper call in -- * io_commit_cqring -- */ -- smp_rmb(); -- if (!io_sqring_full(ctx)) -- mask |= EPOLLOUT | EPOLLWRNORM; -- -- /* -- * Don't flush cqring overflow list here, just do a simple check. -- * Otherwise there could possible be ABBA deadlock: -- * CPU0 CPU1 -- * ---- ---- -- * lock(&ctx->uring_lock); -- * lock(&ep->mtx); -- * lock(&ctx->uring_lock); -- * lock(&ep->mtx); -- * -- * Users may get EPOLLIN meanwhile seeing nothing in cqring, this -- * pushs them to do the flush. -- */ -- if (io_cqring_events(ctx) || test_bit(0, &ctx->cq_check_overflow)) -- mask |= EPOLLIN | EPOLLRDNORM; -- -- return mask; --} -- --static int io_uring_fasync(int fd, struct file *file, int on) --{ -- struct io_ring_ctx *ctx = file->private_data; -- -- return fasync_helper(fd, file, on, &ctx->cq_fasync); --} -- --static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) --{ -- struct io_identity *iod; -- -- iod = xa_erase(&ctx->personalities, id); -- if (iod) { -- put_cred(iod->creds); -- if (refcount_dec_and_test(&iod->count)) -- kfree(iod); -- return 0; -- } -- -- return -EINVAL; --} -- --static void io_ring_exit_work(struct work_struct *work) --{ -- struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, -- exit_work); -- -- /* -- * If we're doing polled IO and end up having requests being -- * submitted async (out-of-line), then completions can come in while -- * we're waiting for refs to drop. We need to reap these manually, -- * as nobody else will be looking for them. -- */ -- do { -- io_iopoll_try_reap_events(ctx); -- } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)); -- io_ring_ctx_free(ctx); --} -- --static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data) --{ -- struct io_kiocb *req = container_of(work, struct io_kiocb, work); -- -- return req->ctx == data; --} -- --static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) --{ -- unsigned long index; -- struct io_identify *iod; -- -- mutex_lock(&ctx->uring_lock); -- percpu_ref_kill(&ctx->refs); -- /* if force is set, the ring is going away. always drop after that */ -- -- if (WARN_ON_ONCE((ctx->flags & IORING_SETUP_SQPOLL) && !ctx->sqo_dead)) -- ctx->sqo_dead = 1; -- -- ctx->cq_overflow_flushed = 1; -- if (ctx->rings) -- __io_cqring_overflow_flush(ctx, true, NULL, NULL); -- mutex_unlock(&ctx->uring_lock); -- -- io_kill_timeouts(ctx, NULL, NULL); -- io_poll_remove_all(ctx, NULL, NULL); -- -- if (ctx->io_wq) -- io_wq_cancel_cb(ctx->io_wq, io_cancel_ctx_cb, ctx, true); -- -- /* if we failed setting up the ctx, we might not have any rings */ -- io_iopoll_try_reap_events(ctx); -- xa_for_each(&ctx->personalities, index, iod) -- io_unregister_personality(ctx, index); -- -- /* -- * Do this upfront, so we won't have a grace period where the ring -- * is closed but resources aren't reaped yet. This can cause -- * spurious failure in setting up a new ring. -- */ -- io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries), -- ACCT_LOCKED); -- -- INIT_WORK(&ctx->exit_work, io_ring_exit_work); -- /* -- * Use system_unbound_wq to avoid spawning tons of event kworkers -- * if we're exiting a ton of rings at the same time. It just adds -- * noise and overhead, there's no discernable change in runtime -- * over using system_wq. -- */ -- queue_work(system_unbound_wq, &ctx->exit_work); --} -- --static int io_uring_release(struct inode *inode, struct file *file) --{ -- struct io_ring_ctx *ctx = file->private_data; -- -- file->private_data = NULL; -- io_ring_ctx_wait_and_kill(ctx); -- return 0; --} -- --struct io_task_cancel { -- struct task_struct *task; -- struct files_struct *files; --}; -- --static bool io_cancel_task_cb(struct io_wq_work *work, void *data) --{ -- struct io_kiocb *req = container_of(work, struct io_kiocb, work); -- struct io_task_cancel *cancel = data; -- bool ret; -- -- if (cancel->files && (req->flags & REQ_F_LINK_TIMEOUT)) { -- unsigned long flags; -- struct io_ring_ctx *ctx = req->ctx; -- -- /* protect against races with linked timeouts */ -- spin_lock_irqsave(&ctx->completion_lock, flags); -- ret = io_match_task(req, cancel->task, cancel->files); -- spin_unlock_irqrestore(&ctx->completion_lock, flags); -- } else { -- ret = io_match_task(req, cancel->task, cancel->files); -- } -- return ret; --} -- --static void io_cancel_defer_files(struct io_ring_ctx *ctx, -- struct task_struct *task, -- struct files_struct *files) --{ -- struct io_defer_entry *de = NULL; -- LIST_HEAD(list); -- -- spin_lock_irq(&ctx->completion_lock); -- list_for_each_entry_reverse(de, &ctx->defer_list, list) { -- if (io_match_task(de->req, task, files)) { -- list_cut_position(&list, &ctx->defer_list, &de->list); -- break; -- } -- } -- spin_unlock_irq(&ctx->completion_lock); -- -- while (!list_empty(&list)) { -- de = list_first_entry(&list, struct io_defer_entry, list); -- list_del_init(&de->list); -- req_set_fail_links(de->req); -- io_put_req(de->req); -- io_req_complete(de->req, -ECANCELED); -- kfree(de); -- } --} -- --static int io_uring_count_inflight(struct io_ring_ctx *ctx, -- struct task_struct *task, -- struct files_struct *files) --{ -- struct io_kiocb *req; -- int cnt = 0; -- -- spin_lock_irq(&ctx->inflight_lock); -- list_for_each_entry(req, &ctx->inflight_list, inflight_entry) -- cnt += io_match_task(req, task, files); -- spin_unlock_irq(&ctx->inflight_lock); -- return cnt; --} -- --static void io_uring_cancel_files(struct io_ring_ctx *ctx, -- struct task_struct *task, -- struct files_struct *files) --{ -- while (!list_empty_careful(&ctx->inflight_list)) { -- struct io_task_cancel cancel = { .task = task, .files = files }; -- DEFINE_WAIT(wait); -- int inflight; -- -- inflight = io_uring_count_inflight(ctx, task, files); -- if (!inflight) -- break; -- -- io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, &cancel, true); -- io_poll_remove_all(ctx, task, files); -- io_kill_timeouts(ctx, task, files); -- /* cancellations _may_ trigger task work */ -- io_run_task_work(); -- -- prepare_to_wait(&task->io_uring->wait, &wait, -- TASK_UNINTERRUPTIBLE); -- if (inflight == io_uring_count_inflight(ctx, task, files)) -- schedule(); -- finish_wait(&task->io_uring->wait, &wait); -- } --} -- --static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, -- struct task_struct *task) --{ -- while (1) { -- struct io_task_cancel cancel = { .task = task, .files = NULL, }; -- enum io_wq_cancel cret; -- bool ret = false; -- -- cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, &cancel, true); -- if (cret != IO_WQ_CANCEL_NOTFOUND) -- ret = true; -- -- /* SQPOLL thread does its own polling */ -- if (!(ctx->flags & IORING_SETUP_SQPOLL)) { -- while (!list_empty_careful(&ctx->iopoll_list)) { -- io_iopoll_try_reap_events(ctx); -- ret = true; -- } -- } -- -- ret |= io_poll_remove_all(ctx, task, NULL); -- ret |= io_kill_timeouts(ctx, task, NULL); -- if (!ret) -- break; -- io_run_task_work(); -- cond_resched(); -- } --} -- --static void io_disable_sqo_submit(struct io_ring_ctx *ctx) --{ -- mutex_lock(&ctx->uring_lock); -- ctx->sqo_dead = 1; -- if (ctx->flags & IORING_SETUP_R_DISABLED) -- io_sq_offload_start(ctx); -- mutex_unlock(&ctx->uring_lock); -- -- /* make sure callers enter the ring to get error */ -- if (ctx->rings) -- io_ring_set_wakeup_flag(ctx); --} -- --/* -- * We need to iteratively cancel requests, in case a request has dependent -- * hard links. These persist even for failure of cancelations, hence keep -- * looping until none are found. -- */ --static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, -- struct files_struct *files) --{ -- struct task_struct *task = current; -- -- if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { -- io_disable_sqo_submit(ctx); -- task = ctx->sq_data->thread; -- atomic_inc(&task->io_uring->in_idle); -- io_sq_thread_park(ctx->sq_data); -- } -- -- io_cancel_defer_files(ctx, task, files); -- io_cqring_overflow_flush(ctx, true, task, files); -- -- if (!files) -- __io_uring_cancel_task_requests(ctx, task); -- else -- io_uring_cancel_files(ctx, task, files); -- -- if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { -- atomic_dec(&task->io_uring->in_idle); -- io_sq_thread_unpark(ctx->sq_data); -- } --} -- --/* -- * Note that this task has used io_uring. We use it for cancelation purposes. -- */ --static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) --{ -- struct io_uring_task *tctx = current->io_uring; -- int ret; -- -- if (unlikely(!tctx)) { -- ret = io_uring_alloc_task_context(current); -- if (unlikely(ret)) -- return ret; -- tctx = current->io_uring; -- } -- if (tctx->last != file) { -- void *old = xa_load(&tctx->xa, (unsigned long)file); -- -- if (!old) { -- get_file(file); -- ret = xa_err(xa_store(&tctx->xa, (unsigned long)file, -- file, GFP_KERNEL)); -- if (ret) { -- fput(file); -- return ret; -- } -- } -- tctx->last = file; -- } -- -- /* -- * This is race safe in that the task itself is doing this, hence it -- * cannot be going through the exit/cancel paths at the same time. -- * This cannot be modified while exit/cancel is running. -- */ -- if (!tctx->sqpoll && (ctx->flags & IORING_SETUP_SQPOLL)) -- tctx->sqpoll = true; -- -- return 0; --} -- --/* -- * Remove this io_uring_file -> task mapping. -- */ --static void io_uring_del_task_file(struct file *file) --{ -- struct io_uring_task *tctx = current->io_uring; -- -- if (tctx->last == file) -- tctx->last = NULL; -- file = xa_erase(&tctx->xa, (unsigned long)file); -- if (file) -- fput(file); --} -- --static void io_uring_remove_task_files(struct io_uring_task *tctx) --{ -- struct file *file; -- unsigned long index; -- -- xa_for_each(&tctx->xa, index, file) -- io_uring_del_task_file(file); --} -- --void __io_uring_files_cancel(struct files_struct *files) --{ -- struct io_uring_task *tctx = current->io_uring; -- struct file *file; -- unsigned long index; -- -- /* make sure overflow events are dropped */ -- atomic_inc(&tctx->in_idle); -- xa_for_each(&tctx->xa, index, file) -- io_uring_cancel_task_requests(file->private_data, files); -- atomic_dec(&tctx->in_idle); -- -- if (files) -- io_uring_remove_task_files(tctx); --} -- --static s64 tctx_inflight(struct io_uring_task *tctx) --{ -- unsigned long index; -- struct file *file; -- s64 inflight; -- -- inflight = percpu_counter_sum(&tctx->inflight); -- if (!tctx->sqpoll) -- return inflight; -- -- /* -- * If we have SQPOLL rings, then we need to iterate and find them, and -- * add the pending count for those. -- */ -- xa_for_each(&tctx->xa, index, file) { -- struct io_ring_ctx *ctx = file->private_data; -- -- if (ctx->flags & IORING_SETUP_SQPOLL) { -- struct io_uring_task *__tctx = ctx->sqo_task->io_uring; -- -- inflight += percpu_counter_sum(&__tctx->inflight); -- } -- } -- -- return inflight; --} -- --/* -- * Find any io_uring fd that this task has registered or done IO on, and cancel -- * requests. -- */ --void __io_uring_task_cancel(void) --{ -- struct io_uring_task *tctx = current->io_uring; -- DEFINE_WAIT(wait); -- s64 inflight; -- -- /* make sure overflow events are dropped */ -- atomic_inc(&tctx->in_idle); -- -- /* trigger io_disable_sqo_submit() */ -- if (tctx->sqpoll) -- __io_uring_files_cancel(NULL); -- -- do { -- /* read completions before cancelations */ -- inflight = tctx_inflight(tctx); -- if (!inflight) -- break; -- __io_uring_files_cancel(NULL); -- -- prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE); -- -- /* -- * If we've seen completions, retry without waiting. This -- * avoids a race where a completion comes in before we did -- * prepare_to_wait(). -- */ -- if (inflight == tctx_inflight(tctx)) -- schedule(); -- finish_wait(&tctx->wait, &wait); -- } while (1); -- -- atomic_dec(&tctx->in_idle); -- -- io_uring_remove_task_files(tctx); --} -- --static int io_uring_flush(struct file *file, void *data) --{ -- struct io_uring_task *tctx = current->io_uring; -- struct io_ring_ctx *ctx = file->private_data; -- -- if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) -- io_uring_cancel_task_requests(ctx, NULL); -- -- if (!tctx) -- return 0; -- -- /* we should have cancelled and erased it before PF_EXITING */ -- WARN_ON_ONCE((current->flags & PF_EXITING) && -- xa_load(&tctx->xa, (unsigned long)file)); -- -- /* -- * fput() is pending, will be 2 if the only other ref is our potential -- * task file note. If the task is exiting, drop regardless of count. -- */ -- if (atomic_long_read(&file->f_count) != 2) -- return 0; -- -- if (ctx->flags & IORING_SETUP_SQPOLL) { -- /* there is only one file note, which is owned by sqo_task */ -- WARN_ON_ONCE(ctx->sqo_task != current && -- xa_load(&tctx->xa, (unsigned long)file)); -- /* sqo_dead check is for when this happens after cancellation */ -- WARN_ON_ONCE(ctx->sqo_task == current && !ctx->sqo_dead && -- !xa_load(&tctx->xa, (unsigned long)file)); -- -- io_disable_sqo_submit(ctx); -- } -- -- if (!(ctx->flags & IORING_SETUP_SQPOLL) || ctx->sqo_task == current) -- io_uring_del_task_file(file); -- return 0; --} -- --static void *io_uring_validate_mmap_request(struct file *file, -- loff_t pgoff, size_t sz) --{ -- struct io_ring_ctx *ctx = file->private_data; -- loff_t offset = pgoff << PAGE_SHIFT; -- struct page *page; -- void *ptr; -- -- switch (offset) { -- case IORING_OFF_SQ_RING: -- case IORING_OFF_CQ_RING: -- ptr = ctx->rings; -- break; -- case IORING_OFF_SQES: -- ptr = ctx->sq_sqes; -- break; -- default: -- return ERR_PTR(-EINVAL); -- } -- -- page = virt_to_head_page(ptr); -- if (sz > page_size(page)) -- return ERR_PTR(-EINVAL); -- -- return ptr; --} -- --#ifdef CONFIG_MMU -- --static int io_uring_mmap(struct file *file, struct vm_area_struct *vma) --{ -- size_t sz = vma->vm_end - vma->vm_start; -- unsigned long pfn; -- void *ptr; -- -- ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz); -- if (IS_ERR(ptr)) -- return PTR_ERR(ptr); -- -- pfn = virt_to_phys(ptr) >> PAGE_SHIFT; -- return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot); --} -- --#else /* !CONFIG_MMU */ -- --static int io_uring_mmap(struct file *file, struct vm_area_struct *vma) --{ -- return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL; --} -- --static unsigned int io_uring_nommu_mmap_capabilities(struct file *file) --{ -- return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE; --} -- --static unsigned long io_uring_nommu_get_unmapped_area(struct file *file, -- unsigned long addr, unsigned long len, -- unsigned long pgoff, unsigned long flags) --{ -- void *ptr; -- -- ptr = io_uring_validate_mmap_request(file, pgoff, len); -- if (IS_ERR(ptr)) -- return PTR_ERR(ptr); -- -- return (unsigned long) ptr; --} -- --#endif /* !CONFIG_MMU */ -- --static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx) --{ -- int ret = 0; -- DEFINE_WAIT(wait); -- -- do { -- if (!io_sqring_full(ctx)) -- break; -- -- prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE); -- -- if (unlikely(ctx->sqo_dead)) { -- ret = -EOWNERDEAD; -- break; -- } -- -- if (!io_sqring_full(ctx)) -- break; -- -- schedule(); -- } while (!signal_pending(current)); -- -- finish_wait(&ctx->sqo_sq_wait, &wait); -- return ret; --} -- --SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, -- u32, min_complete, u32, flags, const sigset_t __user *, sig, -- size_t, sigsz) --{ -- struct io_ring_ctx *ctx; -- long ret = -EBADF; -- int submitted = 0; -- struct fd f; -- -- io_run_task_work(); -- -- if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP | -- IORING_ENTER_SQ_WAIT)) -- return -EINVAL; -- -- f = fdget(fd); -- if (!f.file) -- return -EBADF; -- -- ret = -EOPNOTSUPP; -- if (f.file->f_op != &io_uring_fops) -- goto out_fput; -- -- ret = -ENXIO; -- ctx = f.file->private_data; -- if (!percpu_ref_tryget(&ctx->refs)) -- goto out_fput; -- -- ret = -EBADFD; -- if (ctx->flags & IORING_SETUP_R_DISABLED) -- goto out; -- -- /* -- * For SQ polling, the thread will do all submissions and completions. -- * Just return the requested submit count, and wake the thread if -- * we were asked to. -- */ -- ret = 0; -- if (ctx->flags & IORING_SETUP_SQPOLL) { -- io_cqring_overflow_flush(ctx, false, NULL, NULL); -- -- if (unlikely(ctx->sqo_dead)) { -- ret = -EOWNERDEAD; -- goto out; -- } -- if (flags & IORING_ENTER_SQ_WAKEUP) -- wake_up(&ctx->sq_data->wait); -- if (flags & IORING_ENTER_SQ_WAIT) { -- ret = io_sqpoll_wait_sq(ctx); -- if (ret) -- goto out; -- } -- submitted = to_submit; -- } else if (to_submit) { -- ret = io_uring_add_task_file(ctx, f.file); -- if (unlikely(ret)) -- goto out; -- mutex_lock(&ctx->uring_lock); -- submitted = io_submit_sqes(ctx, to_submit); -- mutex_unlock(&ctx->uring_lock); -- -- if (submitted != to_submit) -- goto out; -- } -- if (flags & IORING_ENTER_GETEVENTS) { -- min_complete = min(min_complete, ctx->cq_entries); -- -- /* -- * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user -- * space applications don't need to do io completion events -- * polling again, they can rely on io_sq_thread to do polling -- * work, which can reduce cpu usage and uring_lock contention. -- */ -- if (ctx->flags & IORING_SETUP_IOPOLL && -- !(ctx->flags & IORING_SETUP_SQPOLL)) { -- ret = io_iopoll_check(ctx, min_complete); -- } else { -- ret = io_cqring_wait(ctx, min_complete, sig, sigsz); -- } -- } -- --out: -- percpu_ref_put(&ctx->refs); --out_fput: -- fdput(f); -- return submitted ? submitted : ret; --} -- --#ifdef CONFIG_PROC_FS --static int io_uring_show_cred(struct seq_file *m, unsigned int id, -- const struct io_identity *iod) --{ -- const struct cred *cred = iod->creds; -- struct user_namespace *uns = seq_user_ns(m); -- struct group_info *gi; -- kernel_cap_t cap; -- unsigned __capi; -- int g; -- -- seq_printf(m, "%5d\n", id); -- seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid)); -- seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid)); -- seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid)); -- seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid)); -- seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid)); -- seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid)); -- seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid)); -- seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid)); -- seq_puts(m, "\n\tGroups:\t"); -- gi = cred->group_info; -- for (g = 0; g < gi->ngroups; g++) { -- seq_put_decimal_ull(m, g ? " " : "", -- from_kgid_munged(uns, gi->gid[g])); -- } -- seq_puts(m, "\n\tCapEff:\t"); -- cap = cred->cap_effective; -- CAP_FOR_EACH_U32(__capi) -- seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8); -- seq_putc(m, '\n'); -- return 0; --} -- --static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) --{ -- struct io_sq_data *sq = NULL; -- bool has_lock; -- int i; -- -- /* -- * Avoid ABBA deadlock between the seq lock and the io_uring mutex, -- * since fdinfo case grabs it in the opposite direction of normal use -- * cases. If we fail to get the lock, we just don't iterate any -- * structures that could be going away outside the io_uring mutex. -- */ -- has_lock = mutex_trylock(&ctx->uring_lock); -- -- if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) -- sq = ctx->sq_data; -- -- seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1); -- seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1); -- seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files); -- for (i = 0; has_lock && i < ctx->nr_user_files; i++) { -- struct fixed_file_table *table; -- struct file *f; -- -- table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; -- f = table->files[i & IORING_FILE_TABLE_MASK]; -- if (f) -- seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname); -- else -- seq_printf(m, "%5u: \n", i); -- } -- seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs); -- for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) { -- struct io_mapped_ubuf *buf = &ctx->user_bufs[i]; -- -- seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, -- (unsigned int) buf->len); -- } -- if (has_lock && !xa_empty(&ctx->personalities)) { -- unsigned long index; -- const struct io_identity *iod; -- -- seq_printf(m, "Personalities:\n"); -- xa_for_each(&ctx->personalities, index, iod) -- io_uring_show_cred(m, index, iod); -- } -- seq_printf(m, "PollList:\n"); -- spin_lock_irq(&ctx->completion_lock); -- for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { -- struct hlist_head *list = &ctx->cancel_hash[i]; -- struct io_kiocb *req; -- -- hlist_for_each_entry(req, list, hash_node) -- seq_printf(m, " op=%d, task_works=%d\n", req->opcode, -- req->task->task_works != NULL); -- } -- spin_unlock_irq(&ctx->completion_lock); -- if (has_lock) -- mutex_unlock(&ctx->uring_lock); --} -- --static void io_uring_show_fdinfo(struct seq_file *m, struct file *f) --{ -- struct io_ring_ctx *ctx = f->private_data; -- -- if (percpu_ref_tryget(&ctx->refs)) { -- __io_uring_show_fdinfo(ctx, m); -- percpu_ref_put(&ctx->refs); -- } --} --#endif -- --static const struct file_operations io_uring_fops = { -- .release = io_uring_release, -- .flush = io_uring_flush, -- .mmap = io_uring_mmap, --#ifndef CONFIG_MMU -- .get_unmapped_area = io_uring_nommu_get_unmapped_area, -- .mmap_capabilities = io_uring_nommu_mmap_capabilities, --#endif -- .poll = io_uring_poll, -- .fasync = io_uring_fasync, --#ifdef CONFIG_PROC_FS -- .show_fdinfo = io_uring_show_fdinfo, --#endif --}; -- --static int io_allocate_scq_urings(struct io_ring_ctx *ctx, -- struct io_uring_params *p) --{ -- struct io_rings *rings; -- size_t size, sq_array_offset; -- -- /* make sure these are sane, as we already accounted them */ -- ctx->sq_entries = p->sq_entries; -- ctx->cq_entries = p->cq_entries; -- -- size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset); -- if (size == SIZE_MAX) -- return -EOVERFLOW; -- -- rings = io_mem_alloc(size); -- if (!rings) -- return -ENOMEM; -- -- ctx->rings = rings; -- ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); -- rings->sq_ring_mask = p->sq_entries - 1; -- rings->cq_ring_mask = p->cq_entries - 1; -- rings->sq_ring_entries = p->sq_entries; -- rings->cq_ring_entries = p->cq_entries; -- ctx->sq_mask = rings->sq_ring_mask; -- ctx->cq_mask = rings->cq_ring_mask; -- -- size = array_size(sizeof(struct io_uring_sqe), p->sq_entries); -- if (size == SIZE_MAX) { -- io_mem_free(ctx->rings); -- ctx->rings = NULL; -- return -EOVERFLOW; -- } -- -- ctx->sq_sqes = io_mem_alloc(size); -- if (!ctx->sq_sqes) { -- io_mem_free(ctx->rings); -- ctx->rings = NULL; -- return -ENOMEM; -- } -- -- return 0; --} -- --static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file) --{ -- int ret, fd; -- -- fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); -- if (fd < 0) -- return fd; -- -- ret = io_uring_add_task_file(ctx, file); -- if (ret) { -- put_unused_fd(fd); -- return ret; -- } -- fd_install(fd, file); -- return fd; --} -- --/* -- * Allocate an anonymous fd, this is what constitutes the application -- * visible backing of an io_uring instance. The application mmaps this -- * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled, -- * we have to tie this fd to a socket for file garbage collection purposes. -- */ --static struct file *io_uring_get_file(struct io_ring_ctx *ctx) --{ -- struct file *file; --#if defined(CONFIG_UNIX) -- int ret; -- -- ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP, -- &ctx->ring_sock); -- if (ret) -- return ERR_PTR(ret); --#endif -- -- file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx, -- O_RDWR | O_CLOEXEC); --#if defined(CONFIG_UNIX) -- if (IS_ERR(file)) { -- sock_release(ctx->ring_sock); -- ctx->ring_sock = NULL; -- } else { -- ctx->ring_sock->file = file; -- } --#endif -- return file; --} -- --static int io_uring_create(unsigned entries, struct io_uring_params *p, -- struct io_uring_params __user *params) --{ -- struct user_struct *user = NULL; -- struct io_ring_ctx *ctx; -- struct file *file; -- bool limit_mem; -- int ret; -- -- if (!entries) -- return -EINVAL; -- if (entries > IORING_MAX_ENTRIES) { -- if (!(p->flags & IORING_SETUP_CLAMP)) -- return -EINVAL; -- entries = IORING_MAX_ENTRIES; -- } -- -- /* -- * Use twice as many entries for the CQ ring. It's possible for the -- * application to drive a higher depth than the size of the SQ ring, -- * since the sqes are only used at submission time. This allows for -- * some flexibility in overcommitting a bit. If the application has -- * set IORING_SETUP_CQSIZE, it will have passed in the desired number -- * of CQ ring entries manually. -- */ -- p->sq_entries = roundup_pow_of_two(entries); -- if (p->flags & IORING_SETUP_CQSIZE) { -- /* -- * If IORING_SETUP_CQSIZE is set, we do the same roundup -- * to a power-of-two, if it isn't already. We do NOT impose -- * any cq vs sq ring sizing. -- */ -- if (!p->cq_entries) -- return -EINVAL; -- if (p->cq_entries > IORING_MAX_CQ_ENTRIES) { -- if (!(p->flags & IORING_SETUP_CLAMP)) -- return -EINVAL; -- p->cq_entries = IORING_MAX_CQ_ENTRIES; -- } -- p->cq_entries = roundup_pow_of_two(p->cq_entries); -- if (p->cq_entries < p->sq_entries) -- return -EINVAL; -- } else { -- p->cq_entries = 2 * p->sq_entries; -- } -- -- user = get_uid(current_user()); -- limit_mem = !capable(CAP_IPC_LOCK); -- -- if (limit_mem) { -- ret = __io_account_mem(user, -- ring_pages(p->sq_entries, p->cq_entries)); -- if (ret) { -- free_uid(user); -- return ret; -- } -- } -- -- ctx = io_ring_ctx_alloc(p); -- if (!ctx) { -- if (limit_mem) -- __io_unaccount_mem(user, ring_pages(p->sq_entries, -- p->cq_entries)); -- free_uid(user); -- return -ENOMEM; -- } -- ctx->compat = in_compat_syscall(); -- ctx->user = user; -- ctx->creds = get_current_cred(); --#ifdef CONFIG_AUDIT -- ctx->loginuid = current->loginuid; -- ctx->sessionid = current->sessionid; --#endif -- ctx->sqo_task = get_task_struct(current); -- -- /* -- * This is just grabbed for accounting purposes. When a process exits, -- * the mm is exited and dropped before the files, hence we need to hang -- * on to this mm purely for the purposes of being able to unaccount -- * memory (locked/pinned vm). It's not used for anything else. -- */ -- mmgrab(current->mm); -- ctx->mm_account = current->mm; -- --#ifdef CONFIG_BLK_CGROUP -- /* -- * The sq thread will belong to the original cgroup it was inited in. -- * If the cgroup goes offline (e.g. disabling the io controller), then -- * issued bios will be associated with the closest cgroup later in the -- * block layer. -- */ -- rcu_read_lock(); -- ctx->sqo_blkcg_css = blkcg_css(); -- ret = css_tryget_online(ctx->sqo_blkcg_css); -- rcu_read_unlock(); -- if (!ret) { -- /* don't init against a dying cgroup, have the user try again */ -- ctx->sqo_blkcg_css = NULL; -- ret = -ENODEV; -- goto err; -- } --#endif -- -- /* -- * Account memory _before_ installing the file descriptor. Once -- * the descriptor is installed, it can get closed at any time. Also -- * do this before hitting the general error path, as ring freeing -- * will un-account as well. -- */ -- io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries), -- ACCT_LOCKED); -- ctx->limit_mem = limit_mem; -- -- ret = io_allocate_scq_urings(ctx, p); -- if (ret) -- goto err; -- -- ret = io_sq_offload_create(ctx, p); -- if (ret) -- goto err; -- -- if (!(p->flags & IORING_SETUP_R_DISABLED)) -- io_sq_offload_start(ctx); -- -- memset(&p->sq_off, 0, sizeof(p->sq_off)); -- p->sq_off.head = offsetof(struct io_rings, sq.head); -- p->sq_off.tail = offsetof(struct io_rings, sq.tail); -- p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask); -- p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries); -- p->sq_off.flags = offsetof(struct io_rings, sq_flags); -- p->sq_off.dropped = offsetof(struct io_rings, sq_dropped); -- p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; -- -- memset(&p->cq_off, 0, sizeof(p->cq_off)); -- p->cq_off.head = offsetof(struct io_rings, cq.head); -- p->cq_off.tail = offsetof(struct io_rings, cq.tail); -- p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask); -- p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries); -- p->cq_off.overflow = offsetof(struct io_rings, cq_overflow); -- p->cq_off.cqes = offsetof(struct io_rings, cqes); -- p->cq_off.flags = offsetof(struct io_rings, cq_flags); -- -- p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP | -- IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS | -- IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL | -- IORING_FEAT_POLL_32BITS; -- -- if (copy_to_user(params, p, sizeof(*p))) { -- ret = -EFAULT; -- goto err; -- } -- -- file = io_uring_get_file(ctx); -- if (IS_ERR(file)) { -- ret = PTR_ERR(file); -- goto err; -- } -- -- /* -- * Install ring fd as the very last thing, so we don't risk someone -- * having closed it before we finish setup -- */ -- ret = io_uring_install_fd(ctx, file); -- if (ret < 0) { -- io_disable_sqo_submit(ctx); -- /* fput will clean it up */ -- fput(file); -- return ret; -- } -- -- trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags); -- return ret; --err: -- io_disable_sqo_submit(ctx); -- io_ring_ctx_wait_and_kill(ctx); -- return ret; --} -- --/* -- * Sets up an aio uring context, and returns the fd. Applications asks for a -- * ring size, we return the actual sq/cq ring sizes (among other things) in the -- * params structure passed in. -- */ --static long io_uring_setup(u32 entries, struct io_uring_params __user *params) --{ -- struct io_uring_params p; -- int i; -- -- if (copy_from_user(&p, params, sizeof(p))) -- return -EFAULT; -- for (i = 0; i < ARRAY_SIZE(p.resv); i++) { -- if (p.resv[i]) -- return -EINVAL; -- } -- -- if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL | -- IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE | -- IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ | -- IORING_SETUP_R_DISABLED)) -- return -EINVAL; -- -- return io_uring_create(entries, &p, params); --} -- --SYSCALL_DEFINE2(io_uring_setup, u32, entries, -- struct io_uring_params __user *, params) --{ -- return io_uring_setup(entries, params); --} -- --static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) --{ -- struct io_uring_probe *p; -- size_t size; -- int i, ret; -- -- size = struct_size(p, ops, nr_args); -- if (size == SIZE_MAX) -- return -EOVERFLOW; -- p = kzalloc(size, GFP_KERNEL); -- if (!p) -- return -ENOMEM; -- -- ret = -EFAULT; -- if (copy_from_user(p, arg, size)) -- goto out; -- ret = -EINVAL; -- if (memchr_inv(p, 0, size)) -- goto out; -- -- p->last_op = IORING_OP_LAST - 1; -- if (nr_args > IORING_OP_LAST) -- nr_args = IORING_OP_LAST; -- -- for (i = 0; i < nr_args; i++) { -- p->ops[i].op = i; -- if (!io_op_defs[i].not_supported) -- p->ops[i].flags = IO_URING_OP_SUPPORTED; -- } -- p->ops_len = i; -- -- ret = 0; -- if (copy_to_user(arg, p, size)) -- ret = -EFAULT; --out: -- kfree(p); -- return ret; --} -- --static int io_register_personality(struct io_ring_ctx *ctx) --{ -- struct io_identity *iod; -- u32 id; -- int ret; -- -- iod = kmalloc(sizeof(*iod), GFP_KERNEL); -- if (unlikely(!iod)) -- return -ENOMEM; -- -- io_init_identity(iod); -- iod->creds = get_current_cred(); -- -- ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)iod, -- XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL); -- if (ret < 0) { -- put_cred(iod->creds); -- kfree(iod); -- return ret; -- } -- return id; --} -- --static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg, -- unsigned int nr_args) --{ -- struct io_uring_restriction *res; -- size_t size; -- int i, ret; -- -- /* Restrictions allowed only if rings started disabled */ -- if (!(ctx->flags & IORING_SETUP_R_DISABLED)) -- return -EBADFD; -- -- /* We allow only a single restrictions registration */ -- if (ctx->restrictions.registered) -- return -EBUSY; -- -- if (!arg || nr_args > IORING_MAX_RESTRICTIONS) -- return -EINVAL; -- -- size = array_size(nr_args, sizeof(*res)); -- if (size == SIZE_MAX) -- return -EOVERFLOW; -- -- res = memdup_user(arg, size); -- if (IS_ERR(res)) -- return PTR_ERR(res); -- -- ret = 0; -- -- for (i = 0; i < nr_args; i++) { -- switch (res[i].opcode) { -- case IORING_RESTRICTION_REGISTER_OP: -- if (res[i].register_op >= IORING_REGISTER_LAST) { -- ret = -EINVAL; -- goto out; -- } -- -- __set_bit(res[i].register_op, -- ctx->restrictions.register_op); -- break; -- case IORING_RESTRICTION_SQE_OP: -- if (res[i].sqe_op >= IORING_OP_LAST) { -- ret = -EINVAL; -- goto out; -- } -- -- __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op); -- break; -- case IORING_RESTRICTION_SQE_FLAGS_ALLOWED: -- ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags; -- break; -- case IORING_RESTRICTION_SQE_FLAGS_REQUIRED: -- ctx->restrictions.sqe_flags_required = res[i].sqe_flags; -- break; -- default: -- ret = -EINVAL; -- goto out; -- } -- } -- --out: -- /* Reset all restrictions if an error happened */ -- if (ret != 0) -- memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); -- else -- ctx->restrictions.registered = true; -- -- kfree(res); -- return ret; --} -- --static int io_register_enable_rings(struct io_ring_ctx *ctx) --{ -- if (!(ctx->flags & IORING_SETUP_R_DISABLED)) -- return -EBADFD; -- -- if (ctx->restrictions.registered) -- ctx->restricted = 1; -- -- io_sq_offload_start(ctx); -- return 0; --} -- --static bool io_register_op_must_quiesce(int op) --{ -- switch (op) { -- case IORING_UNREGISTER_FILES: -- case IORING_REGISTER_FILES_UPDATE: -- case IORING_REGISTER_PROBE: -- case IORING_REGISTER_PERSONALITY: -- case IORING_UNREGISTER_PERSONALITY: -- return false; -- default: -- return true; -- } --} -- --static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, -- void __user *arg, unsigned nr_args) -- __releases(ctx->uring_lock) -- __acquires(ctx->uring_lock) --{ -- int ret; -- -- /* -- * We're inside the ring mutex, if the ref is already dying, then -- * someone else killed the ctx or is already going through -- * io_uring_register(). -- */ -- if (percpu_ref_is_dying(&ctx->refs)) -- return -ENXIO; -- -- if (io_register_op_must_quiesce(opcode)) { -- percpu_ref_kill(&ctx->refs); -- -- /* -- * Drop uring mutex before waiting for references to exit. If -- * another thread is currently inside io_uring_enter() it might -- * need to grab the uring_lock to make progress. If we hold it -- * here across the drain wait, then we can deadlock. It's safe -- * to drop the mutex here, since no new references will come in -- * after we've killed the percpu ref. -- */ -- mutex_unlock(&ctx->uring_lock); -- do { -- ret = wait_for_completion_interruptible(&ctx->ref_comp); -- if (!ret) -- break; -- ret = io_run_task_work_sig(); -- if (ret < 0) -- break; -- } while (1); -- mutex_lock(&ctx->uring_lock); -- -- if (ret) { -- io_refs_resurrect(&ctx->refs, &ctx->ref_comp); -- return ret; -- } -- } -- -- if (ctx->restricted) { -- if (opcode >= IORING_REGISTER_LAST) { -- ret = -EINVAL; -- goto out; -- } -- -- if (!test_bit(opcode, ctx->restrictions.register_op)) { -- ret = -EACCES; -- goto out; -- } -- } -- -- switch (opcode) { -- case IORING_REGISTER_BUFFERS: -- ret = io_sqe_buffer_register(ctx, arg, nr_args); -- break; -- case IORING_UNREGISTER_BUFFERS: -- ret = -EINVAL; -- if (arg || nr_args) -- break; -- ret = io_sqe_buffer_unregister(ctx); -- break; -- case IORING_REGISTER_FILES: -- ret = io_sqe_files_register(ctx, arg, nr_args); -- break; -- case IORING_UNREGISTER_FILES: -- ret = -EINVAL; -- if (arg || nr_args) -- break; -- ret = io_sqe_files_unregister(ctx); -- break; -- case IORING_REGISTER_FILES_UPDATE: -- ret = io_sqe_files_update(ctx, arg, nr_args); -- break; -- case IORING_REGISTER_EVENTFD: -- case IORING_REGISTER_EVENTFD_ASYNC: -- ret = -EINVAL; -- if (nr_args != 1) -- break; -- ret = io_eventfd_register(ctx, arg); -- if (ret) -- break; -- if (opcode == IORING_REGISTER_EVENTFD_ASYNC) -- ctx->eventfd_async = 1; -- else -- ctx->eventfd_async = 0; -- break; -- case IORING_UNREGISTER_EVENTFD: -- ret = -EINVAL; -- if (arg || nr_args) -- break; -- ret = io_eventfd_unregister(ctx); -- break; -- case IORING_REGISTER_PROBE: -- ret = -EINVAL; -- if (!arg || nr_args > 256) -- break; -- ret = io_probe(ctx, arg, nr_args); -- break; -- case IORING_REGISTER_PERSONALITY: -- ret = -EINVAL; -- if (arg || nr_args) -- break; -- ret = io_register_personality(ctx); -- break; -- case IORING_UNREGISTER_PERSONALITY: -- ret = -EINVAL; -- if (arg) -- break; -- ret = io_unregister_personality(ctx, nr_args); -- break; -- case IORING_REGISTER_ENABLE_RINGS: -- ret = -EINVAL; -- if (arg || nr_args) -- break; -- ret = io_register_enable_rings(ctx); -- break; -- case IORING_REGISTER_RESTRICTIONS: -- ret = io_register_restrictions(ctx, arg, nr_args); -- break; -- default: -- ret = -EINVAL; -- break; -- } -- --out: -- if (io_register_op_must_quiesce(opcode)) { -- /* bring the ctx back to life */ -- percpu_ref_reinit(&ctx->refs); -- reinit_completion(&ctx->ref_comp); -- } -- return ret; --} -- --SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, -- void __user *, arg, unsigned int, nr_args) --{ -- struct io_ring_ctx *ctx; -- long ret = -EBADF; -- struct fd f; -- -- f = fdget(fd); -- if (!f.file) -- return -EBADF; -- -- ret = -EOPNOTSUPP; -- if (f.file->f_op != &io_uring_fops) -- goto out_fput; -- -- ctx = f.file->private_data; -- -- mutex_lock(&ctx->uring_lock); -- ret = __io_uring_register(ctx, opcode, arg, nr_args); -- mutex_unlock(&ctx->uring_lock); -- trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, -- ctx->cq_ev_fd != NULL, ret); --out_fput: -- fdput(f); -- return ret; --} -- --static int __init io_uring_init(void) --{ --#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \ -- BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \ -- BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \ --} while (0) -- --#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \ -- __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename) -- BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64); -- BUILD_BUG_SQE_ELEM(0, __u8, opcode); -- BUILD_BUG_SQE_ELEM(1, __u8, flags); -- BUILD_BUG_SQE_ELEM(2, __u16, ioprio); -- BUILD_BUG_SQE_ELEM(4, __s32, fd); -- BUILD_BUG_SQE_ELEM(8, __u64, off); -- BUILD_BUG_SQE_ELEM(8, __u64, addr2); -- BUILD_BUG_SQE_ELEM(16, __u64, addr); -- BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in); -- BUILD_BUG_SQE_ELEM(24, __u32, len); -- BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags); -- BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags); -- BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags); -- BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags); -- BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events); -- BUILD_BUG_SQE_ELEM(28, __u32, poll32_events); -- BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags); -- BUILD_BUG_SQE_ELEM(28, __u32, msg_flags); -- BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags); -- BUILD_BUG_SQE_ELEM(28, __u32, accept_flags); -- BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags); -- BUILD_BUG_SQE_ELEM(28, __u32, open_flags); -- BUILD_BUG_SQE_ELEM(28, __u32, statx_flags); -- BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice); -- BUILD_BUG_SQE_ELEM(28, __u32, splice_flags); -- BUILD_BUG_SQE_ELEM(32, __u64, user_data); -- BUILD_BUG_SQE_ELEM(40, __u16, buf_index); -- BUILD_BUG_SQE_ELEM(42, __u16, personality); -- BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in); -- -- BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); -- BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int)); -- req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC); -- return 0; --}; --__initcall(io_uring_init); ---- a/include/linux/io_uring.h -+++ b/include/linux/io_uring.h -@@ -5,50 +5,20 @@ - #include - #include - --struct io_identity { -- struct files_struct *files; -- struct mm_struct *mm; --#ifdef CONFIG_BLK_CGROUP -- struct cgroup_subsys_state *blkcg_css; --#endif -- const struct cred *creds; -- struct nsproxy *nsproxy; -- struct fs_struct *fs; -- unsigned long fsize; --#ifdef CONFIG_AUDIT -- kuid_t loginuid; -- unsigned int sessionid; --#endif -- refcount_t count; --}; -- --struct io_uring_task { -- /* submission side */ -- struct xarray xa; -- struct wait_queue_head wait; -- struct file *last; -- struct percpu_counter inflight; -- struct io_identity __identity; -- struct io_identity *identity; -- atomic_t in_idle; -- bool sqpoll; --}; -- - #if defined(CONFIG_IO_URING) - struct sock *io_uring_get_socket(struct file *file); --void __io_uring_task_cancel(void); --void __io_uring_files_cancel(struct files_struct *files); -+void __io_uring_cancel(bool cancel_all); - void __io_uring_free(struct task_struct *tsk); - --static inline void io_uring_task_cancel(void) -+static inline void io_uring_files_cancel(void) - { -- if (current->io_uring && !xa_empty(¤t->io_uring->xa)) -- __io_uring_task_cancel(); -+ if (current->io_uring) -+ __io_uring_cancel(false); - } --static inline void io_uring_files_cancel(struct files_struct *files) -+static inline void io_uring_task_cancel(void) - { -- if (current->io_uring && !xa_empty(¤t->io_uring->xa)) -- __io_uring_files_cancel(files); -+ if (current->io_uring) -+ __io_uring_cancel(true); - } - static inline void io_uring_free(struct task_struct *tsk) - { -@@ -63,7 +33,7 @@ static inline struct sock *io_uring_get_ - static inline void io_uring_task_cancel(void) - { - } --static inline void io_uring_files_cancel(struct files_struct *files) -+static inline void io_uring_files_cancel(void) - { - } - static inline void io_uring_free(struct task_struct *tsk) ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -885,6 +885,9 @@ struct task_struct { - /* CLONE_CHILD_CLEARTID: */ - int __user *clear_child_tid; - -+ /* PF_IO_WORKER */ -+ void *pf_io_worker; -+ - u64 utime; - u64 stime; - #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME ---- a/include/linux/syscalls.h -+++ b/include/linux/syscalls.h -@@ -341,7 +341,7 @@ asmlinkage long sys_io_uring_setup(u32 e - struct io_uring_params __user *p); - asmlinkage long sys_io_uring_enter(unsigned int fd, u32 to_submit, - u32 min_complete, u32 flags, -- const sigset_t __user *sig, size_t sigsz); -+ const void __user *argp, size_t argsz); - asmlinkage long sys_io_uring_register(unsigned int fd, unsigned int op, - void __user *arg, unsigned int nr_args); - ---- a/include/trace/events/io_uring.h -+++ b/include/trace/events/io_uring.h -@@ -12,11 +12,11 @@ struct io_wq_work; - /** - * io_uring_create - called after a new io_uring context was prepared - * -- * @fd: corresponding file descriptor -- * @ctx: pointer to a ring context structure -+ * @fd: corresponding file descriptor -+ * @ctx: pointer to a ring context structure - * @sq_entries: actual SQ size - * @cq_entries: actual CQ size -- * @flags: SQ ring flags, provided to io_uring_setup(2) -+ * @flags: SQ ring flags, provided to io_uring_setup(2) - * - * Allows to trace io_uring creation and provide pointer to a context, that can - * be used later to find correlated events. -@@ -49,15 +49,15 @@ TRACE_EVENT(io_uring_create, - ); - - /** -- * io_uring_register - called after a buffer/file/eventfd was succesfully -+ * io_uring_register - called after a buffer/file/eventfd was successfully - * registered for a ring - * -- * @ctx: pointer to a ring context structure -- * @opcode: describes which operation to perform -+ * @ctx: pointer to a ring context structure -+ * @opcode: describes which operation to perform - * @nr_user_files: number of registered files - * @nr_user_bufs: number of registered buffers - * @cq_ev_fd: whether eventfs registered or not -- * @ret: return code -+ * @ret: return code - * - * Allows to trace fixed files/buffers/eventfds, that could be registered to - * avoid an overhead of getting references to them for every operation. This -@@ -142,16 +142,16 @@ TRACE_EVENT(io_uring_queue_async_work, - TP_ARGS(ctx, rw, req, work, flags), - - TP_STRUCT__entry ( -- __field( void *, ctx ) -- __field( int, rw ) -- __field( void *, req ) -+ __field( void *, ctx ) -+ __field( int, rw ) -+ __field( void *, req ) - __field( struct io_wq_work *, work ) - __field( unsigned int, flags ) - ), - - TP_fast_assign( - __entry->ctx = ctx; -- __entry->rw = rw; -+ __entry->rw = rw; - __entry->req = req; - __entry->work = work; - __entry->flags = flags; -@@ -196,10 +196,10 @@ TRACE_EVENT(io_uring_defer, - - /** - * io_uring_link - called before the io_uring request added into link_list of -- * another request -+ * another request - * -- * @ctx: pointer to a ring context structure -- * @req: pointer to a linked request -+ * @ctx: pointer to a ring context structure -+ * @req: pointer to a linked request - * @target_req: pointer to a previous request, that would contain @req - * - * Allows to track linked requests, to understand dependencies between requests -@@ -212,8 +212,8 @@ TRACE_EVENT(io_uring_link, - TP_ARGS(ctx, req, target_req), - - TP_STRUCT__entry ( -- __field( void *, ctx ) -- __field( void *, req ) -+ __field( void *, ctx ) -+ __field( void *, req ) - __field( void *, target_req ) - ), - -@@ -244,7 +244,7 @@ TRACE_EVENT(io_uring_cqring_wait, - TP_ARGS(ctx, min_events), - - TP_STRUCT__entry ( -- __field( void *, ctx ) -+ __field( void *, ctx ) - __field( int, min_events ) - ), - -@@ -272,7 +272,7 @@ TRACE_EVENT(io_uring_fail_link, - TP_ARGS(req, link), - - TP_STRUCT__entry ( -- __field( void *, req ) -+ __field( void *, req ) - __field( void *, link ) - ), - -@@ -290,38 +290,42 @@ TRACE_EVENT(io_uring_fail_link, - * @ctx: pointer to a ring context structure - * @user_data: user data associated with the request - * @res: result of the request -+ * @cflags: completion flags - * - */ - TRACE_EVENT(io_uring_complete, - -- TP_PROTO(void *ctx, u64 user_data, long res), -+ TP_PROTO(void *ctx, u64 user_data, int res, unsigned cflags), - -- TP_ARGS(ctx, user_data, res), -+ TP_ARGS(ctx, user_data, res, cflags), - - TP_STRUCT__entry ( - __field( void *, ctx ) - __field( u64, user_data ) -- __field( long, res ) -+ __field( int, res ) -+ __field( unsigned, cflags ) - ), - - TP_fast_assign( - __entry->ctx = ctx; - __entry->user_data = user_data; - __entry->res = res; -+ __entry->cflags = cflags; - ), - -- TP_printk("ring %p, user_data 0x%llx, result %ld", -+ TP_printk("ring %p, user_data 0x%llx, result %d, cflags %x", - __entry->ctx, (unsigned long long)__entry->user_data, -- __entry->res) -+ __entry->res, __entry->cflags) - ); - -- - /** - * io_uring_submit_sqe - called before submitting one SQE - * - * @ctx: pointer to a ring context structure -+ * @req: pointer to a submitted request - * @opcode: opcode of request - * @user_data: user data associated with the request -+ * @flags request flags - * @force_nonblock: whether a context blocking or not - * @sq_thread: true if sq_thread has submitted this SQE - * -@@ -330,41 +334,60 @@ TRACE_EVENT(io_uring_complete, - */ - TRACE_EVENT(io_uring_submit_sqe, - -- TP_PROTO(void *ctx, u8 opcode, u64 user_data, bool force_nonblock, -- bool sq_thread), -+ TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data, u32 flags, -+ bool force_nonblock, bool sq_thread), - -- TP_ARGS(ctx, opcode, user_data, force_nonblock, sq_thread), -+ TP_ARGS(ctx, req, opcode, user_data, flags, force_nonblock, sq_thread), - - TP_STRUCT__entry ( - __field( void *, ctx ) -+ __field( void *, req ) - __field( u8, opcode ) - __field( u64, user_data ) -+ __field( u32, flags ) - __field( bool, force_nonblock ) - __field( bool, sq_thread ) - ), - - TP_fast_assign( - __entry->ctx = ctx; -+ __entry->req = req; - __entry->opcode = opcode; - __entry->user_data = user_data; -+ __entry->flags = flags; - __entry->force_nonblock = force_nonblock; - __entry->sq_thread = sq_thread; - ), - -- TP_printk("ring %p, op %d, data 0x%llx, non block %d, sq_thread %d", -- __entry->ctx, __entry->opcode, -- (unsigned long long) __entry->user_data, -- __entry->force_nonblock, __entry->sq_thread) -+ TP_printk("ring %p, req %p, op %d, data 0x%llx, flags %u, " -+ "non block %d, sq_thread %d", __entry->ctx, __entry->req, -+ __entry->opcode, (unsigned long long)__entry->user_data, -+ __entry->flags, __entry->force_nonblock, __entry->sq_thread) - ); - -+/* -+ * io_uring_poll_arm - called after arming a poll wait if successful -+ * -+ * @ctx: pointer to a ring context structure -+ * @req: pointer to the armed request -+ * @opcode: opcode of request -+ * @user_data: user data associated with the request -+ * @mask: request poll events mask -+ * @events: registered events of interest -+ * -+ * Allows to track which fds are waiting for and what are the events of -+ * interest. -+ */ - TRACE_EVENT(io_uring_poll_arm, - -- TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask, int events), -+ TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data, -+ int mask, int events), - -- TP_ARGS(ctx, opcode, user_data, mask, events), -+ TP_ARGS(ctx, req, opcode, user_data, mask, events), - - TP_STRUCT__entry ( - __field( void *, ctx ) -+ __field( void *, req ) - __field( u8, opcode ) - __field( u64, user_data ) - __field( int, mask ) -@@ -373,16 +396,17 @@ TRACE_EVENT(io_uring_poll_arm, - - TP_fast_assign( - __entry->ctx = ctx; -+ __entry->req = req; - __entry->opcode = opcode; - __entry->user_data = user_data; - __entry->mask = mask; - __entry->events = events; - ), - -- TP_printk("ring %p, op %d, data 0x%llx, mask 0x%x, events 0x%x", -- __entry->ctx, __entry->opcode, -- (unsigned long long) __entry->user_data, -- __entry->mask, __entry->events) -+ TP_printk("ring %p, req %p, op %d, data 0x%llx, mask 0x%x, events 0x%x", -+ __entry->ctx, __entry->req, __entry->opcode, -+ (unsigned long long) __entry->user_data, -+ __entry->mask, __entry->events) - ); - - TRACE_EVENT(io_uring_poll_wake, -@@ -437,27 +461,40 @@ TRACE_EVENT(io_uring_task_add, - __entry->mask) - ); - -+/* -+ * io_uring_task_run - called when task_work_run() executes the poll events -+ * notification callbacks -+ * -+ * @ctx: pointer to a ring context structure -+ * @req: pointer to the armed request -+ * @opcode: opcode of request -+ * @user_data: user data associated with the request -+ * -+ * Allows to track when notified poll events are processed -+ */ - TRACE_EVENT(io_uring_task_run, - -- TP_PROTO(void *ctx, u8 opcode, u64 user_data), -+ TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data), - -- TP_ARGS(ctx, opcode, user_data), -+ TP_ARGS(ctx, req, opcode, user_data), - - TP_STRUCT__entry ( - __field( void *, ctx ) -+ __field( void *, req ) - __field( u8, opcode ) - __field( u64, user_data ) - ), - - TP_fast_assign( - __entry->ctx = ctx; -+ __entry->req = req; - __entry->opcode = opcode; - __entry->user_data = user_data; - ), - -- TP_printk("ring %p, op %d, data 0x%llx", -- __entry->ctx, __entry->opcode, -- (unsigned long long) __entry->user_data) -+ TP_printk("ring %p, req %p, op %d, data 0x%llx", -+ __entry->ctx, __entry->req, __entry->opcode, -+ (unsigned long long) __entry->user_data) - ); - - #endif /* _TRACE_IO_URING_H */ ---- a/include/uapi/linux/io_uring.h -+++ b/include/uapi/linux/io_uring.h -@@ -42,23 +42,25 @@ struct io_uring_sqe { - __u32 statx_flags; - __u32 fadvise_advice; - __u32 splice_flags; -+ __u32 rename_flags; -+ __u32 unlink_flags; -+ __u32 hardlink_flags; - }; - __u64 user_data; /* data to be passed back at completion time */ -+ /* pack this to avoid bogus arm OABI complaints */ - union { -- struct { -- /* pack this to avoid bogus arm OABI complaints */ -- union { -- /* index into fixed buffers, if used */ -- __u16 buf_index; -- /* for grouped buffer selection */ -- __u16 buf_group; -- } __attribute__((packed)); -- /* personality to use, if used */ -- __u16 personality; -- __s32 splice_fd_in; -- }; -- __u64 __pad2[3]; -+ /* index into fixed buffers, if used */ -+ __u16 buf_index; -+ /* for grouped buffer selection */ -+ __u16 buf_group; -+ } __attribute__((packed)); -+ /* personality to use, if used */ -+ __u16 personality; -+ union { -+ __s32 splice_fd_in; -+ __u32 file_index; - }; -+ __u64 __pad2[2]; - }; - - enum { -@@ -132,6 +134,9 @@ enum { - IORING_OP_PROVIDE_BUFFERS, - IORING_OP_REMOVE_BUFFERS, - IORING_OP_TEE, -+ IORING_OP_SHUTDOWN, -+ IORING_OP_RENAMEAT, -+ IORING_OP_UNLINKAT, - - /* this goes last, obviously */ - IORING_OP_LAST, -@@ -145,8 +150,13 @@ enum { - /* - * sqe->timeout_flags - */ --#define IORING_TIMEOUT_ABS (1U << 0) -- -+#define IORING_TIMEOUT_ABS (1U << 0) -+#define IORING_TIMEOUT_UPDATE (1U << 1) -+#define IORING_TIMEOUT_BOOTTIME (1U << 2) -+#define IORING_TIMEOUT_REALTIME (1U << 3) -+#define IORING_LINK_TIMEOUT_UPDATE (1U << 4) -+#define IORING_TIMEOUT_CLOCK_MASK (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME) -+#define IORING_TIMEOUT_UPDATE_MASK (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE) - /* - * sqe->splice_flags - * extends splice(2) flags -@@ -154,6 +164,21 @@ enum { - #define SPLICE_F_FD_IN_FIXED (1U << 31) /* the last bit of __u32 */ - - /* -+ * POLL_ADD flags. Note that since sqe->poll_events is the flag space, the -+ * command flags for POLL_ADD are stored in sqe->len. -+ * -+ * IORING_POLL_ADD_MULTI Multishot poll. Sets IORING_CQE_F_MORE if -+ * the poll handler will continue to report -+ * CQEs on behalf of the same SQE. -+ * -+ * IORING_POLL_UPDATE Update existing poll request, matching -+ * sqe->addr as the old user_data field. -+ */ -+#define IORING_POLL_ADD_MULTI (1U << 0) -+#define IORING_POLL_UPDATE_EVENTS (1U << 1) -+#define IORING_POLL_UPDATE_USER_DATA (1U << 2) -+ -+/* - * IO completion data structure (Completion Queue Entry) - */ - struct io_uring_cqe { -@@ -166,8 +191,10 @@ struct io_uring_cqe { - * cqe->flags - * - * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID -+ * IORING_CQE_F_MORE If set, parent SQE will generate more CQE entries - */ - #define IORING_CQE_F_BUFFER (1U << 0) -+#define IORING_CQE_F_MORE (1U << 1) - - enum { - IORING_CQE_BUFFER_SHIFT = 16, -@@ -226,6 +253,7 @@ struct io_cqring_offsets { - #define IORING_ENTER_GETEVENTS (1U << 0) - #define IORING_ENTER_SQ_WAKEUP (1U << 1) - #define IORING_ENTER_SQ_WAIT (1U << 2) -+#define IORING_ENTER_EXT_ARG (1U << 3) - - /* - * Passed in for io_uring_setup(2). Copied back with updated info on success -@@ -253,6 +281,10 @@ struct io_uring_params { - #define IORING_FEAT_CUR_PERSONALITY (1U << 4) - #define IORING_FEAT_FAST_POLL (1U << 5) - #define IORING_FEAT_POLL_32BITS (1U << 6) -+#define IORING_FEAT_SQPOLL_NONFIXED (1U << 7) -+#define IORING_FEAT_EXT_ARG (1U << 8) -+#define IORING_FEAT_NATIVE_WORKERS (1U << 9) -+#define IORING_FEAT_RSRC_TAGS (1U << 10) - - /* - * io_uring_register(2) opcodes and arguments -@@ -272,16 +304,62 @@ enum { - IORING_REGISTER_RESTRICTIONS = 11, - IORING_REGISTER_ENABLE_RINGS = 12, - -+ /* extended with tagging */ -+ IORING_REGISTER_FILES2 = 13, -+ IORING_REGISTER_FILES_UPDATE2 = 14, -+ IORING_REGISTER_BUFFERS2 = 15, -+ IORING_REGISTER_BUFFERS_UPDATE = 16, -+ -+ /* set/clear io-wq thread affinities */ -+ IORING_REGISTER_IOWQ_AFF = 17, -+ IORING_UNREGISTER_IOWQ_AFF = 18, -+ -+ /* set/get max number of io-wq workers */ -+ IORING_REGISTER_IOWQ_MAX_WORKERS = 19, -+ - /* this goes last */ - IORING_REGISTER_LAST - }; - -+/* io-wq worker categories */ -+enum { -+ IO_WQ_BOUND, -+ IO_WQ_UNBOUND, -+}; -+ -+/* deprecated, see struct io_uring_rsrc_update */ - struct io_uring_files_update { - __u32 offset; - __u32 resv; - __aligned_u64 /* __s32 * */ fds; - }; - -+struct io_uring_rsrc_register { -+ __u32 nr; -+ __u32 resv; -+ __u64 resv2; -+ __aligned_u64 data; -+ __aligned_u64 tags; -+}; -+ -+struct io_uring_rsrc_update { -+ __u32 offset; -+ __u32 resv; -+ __aligned_u64 data; -+}; -+ -+struct io_uring_rsrc_update2 { -+ __u32 offset; -+ __u32 resv; -+ __aligned_u64 data; -+ __aligned_u64 tags; -+ __u32 nr; -+ __u32 resv2; -+}; -+ -+/* Skip updating fd indexes set to this value in the fd table */ -+#define IORING_REGISTER_FILES_SKIP (-2) -+ - #define IO_URING_OP_SUPPORTED (1U << 0) - - struct io_uring_probe_op { -@@ -329,4 +407,11 @@ enum { - IORING_RESTRICTION_LAST - }; - -+struct io_uring_getevents_arg { -+ __u64 sigmask; -+ __u32 sigmask_sz; -+ __u32 pad; -+ __u64 ts; -+}; -+ - #endif ---- /dev/null -+++ b/io_uring/Makefile -@@ -0,0 +1,6 @@ -+# SPDX-License-Identifier: GPL-2.0 -+# -+# Makefile for io_uring -+ -+obj-$(CONFIG_IO_URING) += io_uring.o -+obj-$(CONFIG_IO_WQ) += io-wq.o ---- /dev/null -+++ b/io_uring/io-wq.c -@@ -0,0 +1,1398 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* -+ * Basic worker thread pool for io_uring -+ * -+ * Copyright (C) 2019 Jens Axboe -+ * -+ */ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "io-wq.h" -+ -+#define WORKER_IDLE_TIMEOUT (5 * HZ) -+ -+enum { -+ IO_WORKER_F_UP = 1, /* up and active */ -+ IO_WORKER_F_RUNNING = 2, /* account as running */ -+ IO_WORKER_F_FREE = 4, /* worker on free list */ -+ IO_WORKER_F_BOUND = 8, /* is doing bounded work */ -+}; -+ -+enum { -+ IO_WQ_BIT_EXIT = 0, /* wq exiting */ -+}; -+ -+enum { -+ IO_ACCT_STALLED_BIT = 0, /* stalled on hash */ -+}; -+ -+/* -+ * One for each thread in a wqe pool -+ */ -+struct io_worker { -+ refcount_t ref; -+ unsigned flags; -+ struct hlist_nulls_node nulls_node; -+ struct list_head all_list; -+ struct task_struct *task; -+ struct io_wqe *wqe; -+ -+ struct io_wq_work *cur_work; -+ spinlock_t lock; -+ -+ struct completion ref_done; -+ -+ unsigned long create_state; -+ struct callback_head create_work; -+ int create_index; -+ -+ union { -+ struct rcu_head rcu; -+ struct work_struct work; -+ }; -+}; -+ -+#if BITS_PER_LONG == 64 -+#define IO_WQ_HASH_ORDER 6 -+#else -+#define IO_WQ_HASH_ORDER 5 -+#endif -+ -+#define IO_WQ_NR_HASH_BUCKETS (1u << IO_WQ_HASH_ORDER) -+ -+struct io_wqe_acct { -+ unsigned nr_workers; -+ unsigned max_workers; -+ int index; -+ atomic_t nr_running; -+ struct io_wq_work_list work_list; -+ unsigned long flags; -+}; -+ -+enum { -+ IO_WQ_ACCT_BOUND, -+ IO_WQ_ACCT_UNBOUND, -+ IO_WQ_ACCT_NR, -+}; -+ -+/* -+ * Per-node worker thread pool -+ */ -+struct io_wqe { -+ raw_spinlock_t lock; -+ struct io_wqe_acct acct[2]; -+ -+ int node; -+ -+ struct hlist_nulls_head free_list; -+ struct list_head all_list; -+ -+ struct wait_queue_entry wait; -+ -+ struct io_wq *wq; -+ struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS]; -+ -+ cpumask_var_t cpu_mask; -+}; -+ -+/* -+ * Per io_wq state -+ */ -+struct io_wq { -+ unsigned long state; -+ -+ free_work_fn *free_work; -+ io_wq_work_fn *do_work; -+ -+ struct io_wq_hash *hash; -+ -+ atomic_t worker_refs; -+ struct completion worker_done; -+ -+ struct hlist_node cpuhp_node; -+ -+ struct task_struct *task; -+ -+ struct io_wqe *wqes[]; -+}; -+ -+static enum cpuhp_state io_wq_online; -+ -+struct io_cb_cancel_data { -+ work_cancel_fn *fn; -+ void *data; -+ int nr_running; -+ int nr_pending; -+ bool cancel_all; -+}; -+ -+static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index); -+static void io_wqe_dec_running(struct io_worker *worker); -+static bool io_acct_cancel_pending_work(struct io_wqe *wqe, -+ struct io_wqe_acct *acct, -+ struct io_cb_cancel_data *match); -+static void create_worker_cb(struct callback_head *cb); -+static void io_wq_cancel_tw_create(struct io_wq *wq); -+ -+static bool io_worker_get(struct io_worker *worker) -+{ -+ return refcount_inc_not_zero(&worker->ref); -+} -+ -+static void io_worker_release(struct io_worker *worker) -+{ -+ if (refcount_dec_and_test(&worker->ref)) -+ complete(&worker->ref_done); -+} -+ -+static inline struct io_wqe_acct *io_get_acct(struct io_wqe *wqe, bool bound) -+{ -+ return &wqe->acct[bound ? IO_WQ_ACCT_BOUND : IO_WQ_ACCT_UNBOUND]; -+} -+ -+static inline struct io_wqe_acct *io_work_get_acct(struct io_wqe *wqe, -+ struct io_wq_work *work) -+{ -+ return io_get_acct(wqe, !(work->flags & IO_WQ_WORK_UNBOUND)); -+} -+ -+static inline struct io_wqe_acct *io_wqe_get_acct(struct io_worker *worker) -+{ -+ return io_get_acct(worker->wqe, worker->flags & IO_WORKER_F_BOUND); -+} -+ -+static void io_worker_ref_put(struct io_wq *wq) -+{ -+ if (atomic_dec_and_test(&wq->worker_refs)) -+ complete(&wq->worker_done); -+} -+ -+static void io_worker_cancel_cb(struct io_worker *worker) -+{ -+ struct io_wqe_acct *acct = io_wqe_get_acct(worker); -+ struct io_wqe *wqe = worker->wqe; -+ struct io_wq *wq = wqe->wq; -+ -+ atomic_dec(&acct->nr_running); -+ raw_spin_lock(&worker->wqe->lock); -+ acct->nr_workers--; -+ raw_spin_unlock(&worker->wqe->lock); -+ io_worker_ref_put(wq); -+ clear_bit_unlock(0, &worker->create_state); -+ io_worker_release(worker); -+} -+ -+static bool io_task_worker_match(struct callback_head *cb, void *data) -+{ -+ struct io_worker *worker; -+ -+ if (cb->func != create_worker_cb) -+ return false; -+ worker = container_of(cb, struct io_worker, create_work); -+ return worker == data; -+} -+ -+static void io_worker_exit(struct io_worker *worker) -+{ -+ struct io_wqe *wqe = worker->wqe; -+ struct io_wq *wq = wqe->wq; -+ -+ while (1) { -+ struct callback_head *cb = task_work_cancel_match(wq->task, -+ io_task_worker_match, worker); -+ -+ if (!cb) -+ break; -+ io_worker_cancel_cb(worker); -+ } -+ -+ if (refcount_dec_and_test(&worker->ref)) -+ complete(&worker->ref_done); -+ wait_for_completion(&worker->ref_done); -+ -+ raw_spin_lock(&wqe->lock); -+ if (worker->flags & IO_WORKER_F_FREE) -+ hlist_nulls_del_rcu(&worker->nulls_node); -+ list_del_rcu(&worker->all_list); -+ preempt_disable(); -+ io_wqe_dec_running(worker); -+ worker->flags = 0; -+ current->flags &= ~PF_IO_WORKER; -+ preempt_enable(); -+ raw_spin_unlock(&wqe->lock); -+ -+ kfree_rcu(worker, rcu); -+ io_worker_ref_put(wqe->wq); -+ do_exit(0); -+} -+ -+static inline bool io_acct_run_queue(struct io_wqe_acct *acct) -+{ -+ if (!wq_list_empty(&acct->work_list) && -+ !test_bit(IO_ACCT_STALLED_BIT, &acct->flags)) -+ return true; -+ return false; -+} -+ -+/* -+ * Check head of free list for an available worker. If one isn't available, -+ * caller must create one. -+ */ -+static bool io_wqe_activate_free_worker(struct io_wqe *wqe, -+ struct io_wqe_acct *acct) -+ __must_hold(RCU) -+{ -+ struct hlist_nulls_node *n; -+ struct io_worker *worker; -+ -+ /* -+ * Iterate free_list and see if we can find an idle worker to -+ * activate. If a given worker is on the free_list but in the process -+ * of exiting, keep trying. -+ */ -+ hlist_nulls_for_each_entry_rcu(worker, n, &wqe->free_list, nulls_node) { -+ if (!io_worker_get(worker)) -+ continue; -+ if (io_wqe_get_acct(worker) != acct) { -+ io_worker_release(worker); -+ continue; -+ } -+ if (wake_up_process(worker->task)) { -+ io_worker_release(worker); -+ return true; -+ } -+ io_worker_release(worker); -+ } -+ -+ return false; -+} -+ -+/* -+ * We need a worker. If we find a free one, we're good. If not, and we're -+ * below the max number of workers, create one. -+ */ -+static bool io_wqe_create_worker(struct io_wqe *wqe, struct io_wqe_acct *acct) -+{ -+ /* -+ * Most likely an attempt to queue unbounded work on an io_wq that -+ * wasn't setup with any unbounded workers. -+ */ -+ if (unlikely(!acct->max_workers)) -+ pr_warn_once("io-wq is not configured for unbound workers"); -+ -+ raw_spin_lock(&wqe->lock); -+ if (acct->nr_workers >= acct->max_workers) { -+ raw_spin_unlock(&wqe->lock); -+ return true; -+ } -+ acct->nr_workers++; -+ raw_spin_unlock(&wqe->lock); -+ atomic_inc(&acct->nr_running); -+ atomic_inc(&wqe->wq->worker_refs); -+ return create_io_worker(wqe->wq, wqe, acct->index); -+} -+ -+static void io_wqe_inc_running(struct io_worker *worker) -+{ -+ struct io_wqe_acct *acct = io_wqe_get_acct(worker); -+ -+ atomic_inc(&acct->nr_running); -+} -+ -+static void create_worker_cb(struct callback_head *cb) -+{ -+ struct io_worker *worker; -+ struct io_wq *wq; -+ struct io_wqe *wqe; -+ struct io_wqe_acct *acct; -+ bool do_create = false; -+ -+ worker = container_of(cb, struct io_worker, create_work); -+ wqe = worker->wqe; -+ wq = wqe->wq; -+ acct = &wqe->acct[worker->create_index]; -+ raw_spin_lock(&wqe->lock); -+ if (acct->nr_workers < acct->max_workers) { -+ acct->nr_workers++; -+ do_create = true; -+ } -+ raw_spin_unlock(&wqe->lock); -+ if (do_create) { -+ create_io_worker(wq, wqe, worker->create_index); -+ } else { -+ atomic_dec(&acct->nr_running); -+ io_worker_ref_put(wq); -+ } -+ clear_bit_unlock(0, &worker->create_state); -+ io_worker_release(worker); -+} -+ -+static bool io_queue_worker_create(struct io_worker *worker, -+ struct io_wqe_acct *acct, -+ task_work_func_t func) -+{ -+ struct io_wqe *wqe = worker->wqe; -+ struct io_wq *wq = wqe->wq; -+ -+ /* raced with exit, just ignore create call */ -+ if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) -+ goto fail; -+ if (!io_worker_get(worker)) -+ goto fail; -+ /* -+ * create_state manages ownership of create_work/index. We should -+ * only need one entry per worker, as the worker going to sleep -+ * will trigger the condition, and waking will clear it once it -+ * runs the task_work. -+ */ -+ if (test_bit(0, &worker->create_state) || -+ test_and_set_bit_lock(0, &worker->create_state)) -+ goto fail_release; -+ -+ atomic_inc(&wq->worker_refs); -+ init_task_work(&worker->create_work, func); -+ worker->create_index = acct->index; -+ if (!task_work_add(wq->task, &worker->create_work, TWA_SIGNAL)) { -+ /* -+ * EXIT may have been set after checking it above, check after -+ * adding the task_work and remove any creation item if it is -+ * now set. wq exit does that too, but we can have added this -+ * work item after we canceled in io_wq_exit_workers(). -+ */ -+ if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) -+ io_wq_cancel_tw_create(wq); -+ io_worker_ref_put(wq); -+ return true; -+ } -+ io_worker_ref_put(wq); -+ clear_bit_unlock(0, &worker->create_state); -+fail_release: -+ io_worker_release(worker); -+fail: -+ atomic_dec(&acct->nr_running); -+ io_worker_ref_put(wq); -+ return false; -+} -+ -+static void io_wqe_dec_running(struct io_worker *worker) -+ __must_hold(wqe->lock) -+{ -+ struct io_wqe_acct *acct = io_wqe_get_acct(worker); -+ struct io_wqe *wqe = worker->wqe; -+ -+ if (!(worker->flags & IO_WORKER_F_UP)) -+ return; -+ -+ if (atomic_dec_and_test(&acct->nr_running) && io_acct_run_queue(acct)) { -+ atomic_inc(&acct->nr_running); -+ atomic_inc(&wqe->wq->worker_refs); -+ raw_spin_unlock(&wqe->lock); -+ io_queue_worker_create(worker, acct, create_worker_cb); -+ raw_spin_lock(&wqe->lock); -+ } -+} -+ -+/* -+ * Worker will start processing some work. Move it to the busy list, if -+ * it's currently on the freelist -+ */ -+static void __io_worker_busy(struct io_wqe *wqe, struct io_worker *worker, -+ struct io_wq_work *work) -+ __must_hold(wqe->lock) -+{ -+ if (worker->flags & IO_WORKER_F_FREE) { -+ worker->flags &= ~IO_WORKER_F_FREE; -+ hlist_nulls_del_init_rcu(&worker->nulls_node); -+ } -+} -+ -+/* -+ * No work, worker going to sleep. Move to freelist, and unuse mm if we -+ * have one attached. Dropping the mm may potentially sleep, so we drop -+ * the lock in that case and return success. Since the caller has to -+ * retry the loop in that case (we changed task state), we don't regrab -+ * the lock if we return success. -+ */ -+static void __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker) -+ __must_hold(wqe->lock) -+{ -+ if (!(worker->flags & IO_WORKER_F_FREE)) { -+ worker->flags |= IO_WORKER_F_FREE; -+ hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); -+ } -+} -+ -+static inline unsigned int io_get_work_hash(struct io_wq_work *work) -+{ -+ return work->flags >> IO_WQ_HASH_SHIFT; -+} -+ -+static bool io_wait_on_hash(struct io_wqe *wqe, unsigned int hash) -+{ -+ struct io_wq *wq = wqe->wq; -+ bool ret = false; -+ -+ spin_lock_irq(&wq->hash->wait.lock); -+ if (list_empty(&wqe->wait.entry)) { -+ __add_wait_queue(&wq->hash->wait, &wqe->wait); -+ if (!test_bit(hash, &wq->hash->map)) { -+ __set_current_state(TASK_RUNNING); -+ list_del_init(&wqe->wait.entry); -+ ret = true; -+ } -+ } -+ spin_unlock_irq(&wq->hash->wait.lock); -+ return ret; -+} -+ -+static struct io_wq_work *io_get_next_work(struct io_wqe_acct *acct, -+ struct io_worker *worker) -+ __must_hold(wqe->lock) -+{ -+ struct io_wq_work_node *node, *prev; -+ struct io_wq_work *work, *tail; -+ unsigned int stall_hash = -1U; -+ struct io_wqe *wqe = worker->wqe; -+ -+ wq_list_for_each(node, prev, &acct->work_list) { -+ unsigned int hash; -+ -+ work = container_of(node, struct io_wq_work, list); -+ -+ /* not hashed, can run anytime */ -+ if (!io_wq_is_hashed(work)) { -+ wq_list_del(&acct->work_list, node, prev); -+ return work; -+ } -+ -+ hash = io_get_work_hash(work); -+ /* all items with this hash lie in [work, tail] */ -+ tail = wqe->hash_tail[hash]; -+ -+ /* hashed, can run if not already running */ -+ if (!test_and_set_bit(hash, &wqe->wq->hash->map)) { -+ wqe->hash_tail[hash] = NULL; -+ wq_list_cut(&acct->work_list, &tail->list, prev); -+ return work; -+ } -+ if (stall_hash == -1U) -+ stall_hash = hash; -+ /* fast forward to a next hash, for-each will fix up @prev */ -+ node = &tail->list; -+ } -+ -+ if (stall_hash != -1U) { -+ bool unstalled; -+ -+ /* -+ * Set this before dropping the lock to avoid racing with new -+ * work being added and clearing the stalled bit. -+ */ -+ set_bit(IO_ACCT_STALLED_BIT, &acct->flags); -+ raw_spin_unlock(&wqe->lock); -+ unstalled = io_wait_on_hash(wqe, stall_hash); -+ raw_spin_lock(&wqe->lock); -+ if (unstalled) { -+ clear_bit(IO_ACCT_STALLED_BIT, &acct->flags); -+ if (wq_has_sleeper(&wqe->wq->hash->wait)) -+ wake_up(&wqe->wq->hash->wait); -+ } -+ } -+ -+ return NULL; -+} -+ -+static bool io_flush_signals(void) -+{ -+ if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL))) { -+ __set_current_state(TASK_RUNNING); -+ tracehook_notify_signal(); -+ return true; -+ } -+ return false; -+} -+ -+static void io_assign_current_work(struct io_worker *worker, -+ struct io_wq_work *work) -+{ -+ if (work) { -+ io_flush_signals(); -+ cond_resched(); -+ } -+ -+ spin_lock(&worker->lock); -+ worker->cur_work = work; -+ spin_unlock(&worker->lock); -+} -+ -+static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work); -+ -+static void io_worker_handle_work(struct io_worker *worker) -+ __releases(wqe->lock) -+{ -+ struct io_wqe_acct *acct = io_wqe_get_acct(worker); -+ struct io_wqe *wqe = worker->wqe; -+ struct io_wq *wq = wqe->wq; -+ bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state); -+ -+ do { -+ struct io_wq_work *work; -+get_next: -+ /* -+ * If we got some work, mark us as busy. If we didn't, but -+ * the list isn't empty, it means we stalled on hashed work. -+ * Mark us stalled so we don't keep looking for work when we -+ * can't make progress, any work completion or insertion will -+ * clear the stalled flag. -+ */ -+ work = io_get_next_work(acct, worker); -+ if (work) -+ __io_worker_busy(wqe, worker, work); -+ -+ raw_spin_unlock(&wqe->lock); -+ if (!work) -+ break; -+ io_assign_current_work(worker, work); -+ __set_current_state(TASK_RUNNING); -+ -+ /* handle a whole dependent link */ -+ do { -+ struct io_wq_work *next_hashed, *linked; -+ unsigned int hash = io_get_work_hash(work); -+ -+ next_hashed = wq_next_work(work); -+ -+ if (unlikely(do_kill) && (work->flags & IO_WQ_WORK_UNBOUND)) -+ work->flags |= IO_WQ_WORK_CANCEL; -+ wq->do_work(work); -+ io_assign_current_work(worker, NULL); -+ -+ linked = wq->free_work(work); -+ work = next_hashed; -+ if (!work && linked && !io_wq_is_hashed(linked)) { -+ work = linked; -+ linked = NULL; -+ } -+ io_assign_current_work(worker, work); -+ if (linked) -+ io_wqe_enqueue(wqe, linked); -+ -+ if (hash != -1U && !next_hashed) { -+ /* serialize hash clear with wake_up() */ -+ spin_lock_irq(&wq->hash->wait.lock); -+ clear_bit(hash, &wq->hash->map); -+ clear_bit(IO_ACCT_STALLED_BIT, &acct->flags); -+ spin_unlock_irq(&wq->hash->wait.lock); -+ if (wq_has_sleeper(&wq->hash->wait)) -+ wake_up(&wq->hash->wait); -+ raw_spin_lock(&wqe->lock); -+ /* skip unnecessary unlock-lock wqe->lock */ -+ if (!work) -+ goto get_next; -+ raw_spin_unlock(&wqe->lock); -+ } -+ } while (work); -+ -+ raw_spin_lock(&wqe->lock); -+ } while (1); -+} -+ -+static int io_wqe_worker(void *data) -+{ -+ struct io_worker *worker = data; -+ struct io_wqe_acct *acct = io_wqe_get_acct(worker); -+ struct io_wqe *wqe = worker->wqe; -+ struct io_wq *wq = wqe->wq; -+ bool last_timeout = false; -+ char buf[TASK_COMM_LEN]; -+ -+ worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); -+ -+ snprintf(buf, sizeof(buf), "iou-wrk-%d", wq->task->pid); -+ set_task_comm(current, buf); -+ -+ while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { -+ long ret; -+ -+ set_current_state(TASK_INTERRUPTIBLE); -+loop: -+ raw_spin_lock(&wqe->lock); -+ if (io_acct_run_queue(acct)) { -+ io_worker_handle_work(worker); -+ goto loop; -+ } -+ /* timed out, exit unless we're the last worker */ -+ if (last_timeout && acct->nr_workers > 1) { -+ acct->nr_workers--; -+ raw_spin_unlock(&wqe->lock); -+ __set_current_state(TASK_RUNNING); -+ break; -+ } -+ last_timeout = false; -+ __io_worker_idle(wqe, worker); -+ raw_spin_unlock(&wqe->lock); -+ if (io_flush_signals()) -+ continue; -+ ret = schedule_timeout(WORKER_IDLE_TIMEOUT); -+ if (signal_pending(current)) { -+ struct ksignal ksig; -+ -+ if (!get_signal(&ksig)) -+ continue; -+ break; -+ } -+ last_timeout = !ret; -+ } -+ -+ if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) { -+ raw_spin_lock(&wqe->lock); -+ io_worker_handle_work(worker); -+ } -+ -+ io_worker_exit(worker); -+ return 0; -+} -+ -+/* -+ * Called when a worker is scheduled in. Mark us as currently running. -+ */ -+void io_wq_worker_running(struct task_struct *tsk) -+{ -+ struct io_worker *worker = tsk->pf_io_worker; -+ -+ if (!worker) -+ return; -+ if (!(worker->flags & IO_WORKER_F_UP)) -+ return; -+ if (worker->flags & IO_WORKER_F_RUNNING) -+ return; -+ worker->flags |= IO_WORKER_F_RUNNING; -+ io_wqe_inc_running(worker); -+} -+ -+/* -+ * Called when worker is going to sleep. If there are no workers currently -+ * running and we have work pending, wake up a free one or create a new one. -+ */ -+void io_wq_worker_sleeping(struct task_struct *tsk) -+{ -+ struct io_worker *worker = tsk->pf_io_worker; -+ -+ if (!worker) -+ return; -+ if (!(worker->flags & IO_WORKER_F_UP)) -+ return; -+ if (!(worker->flags & IO_WORKER_F_RUNNING)) -+ return; -+ -+ worker->flags &= ~IO_WORKER_F_RUNNING; -+ -+ raw_spin_lock(&worker->wqe->lock); -+ io_wqe_dec_running(worker); -+ raw_spin_unlock(&worker->wqe->lock); -+} -+ -+static void io_init_new_worker(struct io_wqe *wqe, struct io_worker *worker, -+ struct task_struct *tsk) -+{ -+ tsk->pf_io_worker = worker; -+ worker->task = tsk; -+ set_cpus_allowed_ptr(tsk, wqe->cpu_mask); -+ tsk->flags |= PF_NO_SETAFFINITY; -+ -+ raw_spin_lock(&wqe->lock); -+ hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); -+ list_add_tail_rcu(&worker->all_list, &wqe->all_list); -+ worker->flags |= IO_WORKER_F_FREE; -+ raw_spin_unlock(&wqe->lock); -+ wake_up_new_task(tsk); -+} -+ -+static bool io_wq_work_match_all(struct io_wq_work *work, void *data) -+{ -+ return true; -+} -+ -+static inline bool io_should_retry_thread(long err) -+{ -+ /* -+ * Prevent perpetual task_work retry, if the task (or its group) is -+ * exiting. -+ */ -+ if (fatal_signal_pending(current)) -+ return false; -+ -+ switch (err) { -+ case -EAGAIN: -+ case -ERESTARTSYS: -+ case -ERESTARTNOINTR: -+ case -ERESTARTNOHAND: -+ return true; -+ default: -+ return false; -+ } -+} -+ -+static void create_worker_cont(struct callback_head *cb) -+{ -+ struct io_worker *worker; -+ struct task_struct *tsk; -+ struct io_wqe *wqe; -+ -+ worker = container_of(cb, struct io_worker, create_work); -+ clear_bit_unlock(0, &worker->create_state); -+ wqe = worker->wqe; -+ tsk = create_io_thread(io_wqe_worker, worker, wqe->node); -+ if (!IS_ERR(tsk)) { -+ io_init_new_worker(wqe, worker, tsk); -+ io_worker_release(worker); -+ return; -+ } else if (!io_should_retry_thread(PTR_ERR(tsk))) { -+ struct io_wqe_acct *acct = io_wqe_get_acct(worker); -+ -+ atomic_dec(&acct->nr_running); -+ raw_spin_lock(&wqe->lock); -+ acct->nr_workers--; -+ if (!acct->nr_workers) { -+ struct io_cb_cancel_data match = { -+ .fn = io_wq_work_match_all, -+ .cancel_all = true, -+ }; -+ -+ while (io_acct_cancel_pending_work(wqe, acct, &match)) -+ raw_spin_lock(&wqe->lock); -+ } -+ raw_spin_unlock(&wqe->lock); -+ io_worker_ref_put(wqe->wq); -+ kfree(worker); -+ return; -+ } -+ -+ /* re-create attempts grab a new worker ref, drop the existing one */ -+ io_worker_release(worker); -+ schedule_work(&worker->work); -+} -+ -+static void io_workqueue_create(struct work_struct *work) -+{ -+ struct io_worker *worker = container_of(work, struct io_worker, work); -+ struct io_wqe_acct *acct = io_wqe_get_acct(worker); -+ -+ if (!io_queue_worker_create(worker, acct, create_worker_cont)) -+ kfree(worker); -+} -+ -+static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) -+{ -+ struct io_wqe_acct *acct = &wqe->acct[index]; -+ struct io_worker *worker; -+ struct task_struct *tsk; -+ -+ __set_current_state(TASK_RUNNING); -+ -+ worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, wqe->node); -+ if (!worker) { -+fail: -+ atomic_dec(&acct->nr_running); -+ raw_spin_lock(&wqe->lock); -+ acct->nr_workers--; -+ raw_spin_unlock(&wqe->lock); -+ io_worker_ref_put(wq); -+ return false; -+ } -+ -+ refcount_set(&worker->ref, 1); -+ worker->wqe = wqe; -+ spin_lock_init(&worker->lock); -+ init_completion(&worker->ref_done); -+ -+ if (index == IO_WQ_ACCT_BOUND) -+ worker->flags |= IO_WORKER_F_BOUND; -+ -+ tsk = create_io_thread(io_wqe_worker, worker, wqe->node); -+ if (!IS_ERR(tsk)) { -+ io_init_new_worker(wqe, worker, tsk); -+ } else if (!io_should_retry_thread(PTR_ERR(tsk))) { -+ kfree(worker); -+ goto fail; -+ } else { -+ INIT_WORK(&worker->work, io_workqueue_create); -+ schedule_work(&worker->work); -+ } -+ -+ return true; -+} -+ -+/* -+ * Iterate the passed in list and call the specific function for each -+ * worker that isn't exiting -+ */ -+static bool io_wq_for_each_worker(struct io_wqe *wqe, -+ bool (*func)(struct io_worker *, void *), -+ void *data) -+{ -+ struct io_worker *worker; -+ bool ret = false; -+ -+ list_for_each_entry_rcu(worker, &wqe->all_list, all_list) { -+ if (io_worker_get(worker)) { -+ /* no task if node is/was offline */ -+ if (worker->task) -+ ret = func(worker, data); -+ io_worker_release(worker); -+ if (ret) -+ break; -+ } -+ } -+ -+ return ret; -+} -+ -+static bool io_wq_worker_wake(struct io_worker *worker, void *data) -+{ -+ set_notify_signal(worker->task); -+ wake_up_process(worker->task); -+ return false; -+} -+ -+static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe) -+{ -+ struct io_wq *wq = wqe->wq; -+ -+ do { -+ work->flags |= IO_WQ_WORK_CANCEL; -+ wq->do_work(work); -+ work = wq->free_work(work); -+ } while (work); -+} -+ -+static void io_wqe_insert_work(struct io_wqe *wqe, struct io_wq_work *work) -+{ -+ struct io_wqe_acct *acct = io_work_get_acct(wqe, work); -+ unsigned int hash; -+ struct io_wq_work *tail; -+ -+ if (!io_wq_is_hashed(work)) { -+append: -+ wq_list_add_tail(&work->list, &acct->work_list); -+ return; -+ } -+ -+ hash = io_get_work_hash(work); -+ tail = wqe->hash_tail[hash]; -+ wqe->hash_tail[hash] = work; -+ if (!tail) -+ goto append; -+ -+ wq_list_add_after(&work->list, &tail->list, &acct->work_list); -+} -+ -+static bool io_wq_work_match_item(struct io_wq_work *work, void *data) -+{ -+ return work == data; -+} -+ -+static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) -+{ -+ struct io_wqe_acct *acct = io_work_get_acct(wqe, work); -+ unsigned work_flags = work->flags; -+ bool do_create; -+ -+ /* -+ * If io-wq is exiting for this task, or if the request has explicitly -+ * been marked as one that should not get executed, cancel it here. -+ */ -+ if (test_bit(IO_WQ_BIT_EXIT, &wqe->wq->state) || -+ (work->flags & IO_WQ_WORK_CANCEL)) { -+ io_run_cancel(work, wqe); -+ return; -+ } -+ -+ raw_spin_lock(&wqe->lock); -+ io_wqe_insert_work(wqe, work); -+ clear_bit(IO_ACCT_STALLED_BIT, &acct->flags); -+ -+ rcu_read_lock(); -+ do_create = !io_wqe_activate_free_worker(wqe, acct); -+ rcu_read_unlock(); -+ -+ raw_spin_unlock(&wqe->lock); -+ -+ if (do_create && ((work_flags & IO_WQ_WORK_CONCURRENT) || -+ !atomic_read(&acct->nr_running))) { -+ bool did_create; -+ -+ did_create = io_wqe_create_worker(wqe, acct); -+ if (likely(did_create)) -+ return; -+ -+ raw_spin_lock(&wqe->lock); -+ /* fatal condition, failed to create the first worker */ -+ if (!acct->nr_workers) { -+ struct io_cb_cancel_data match = { -+ .fn = io_wq_work_match_item, -+ .data = work, -+ .cancel_all = false, -+ }; -+ -+ if (io_acct_cancel_pending_work(wqe, acct, &match)) -+ raw_spin_lock(&wqe->lock); -+ } -+ raw_spin_unlock(&wqe->lock); -+ } -+} -+ -+void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work) -+{ -+ struct io_wqe *wqe = wq->wqes[numa_node_id()]; -+ -+ io_wqe_enqueue(wqe, work); -+} -+ -+/* -+ * Work items that hash to the same value will not be done in parallel. -+ * Used to limit concurrent writes, generally hashed by inode. -+ */ -+void io_wq_hash_work(struct io_wq_work *work, void *val) -+{ -+ unsigned int bit; -+ -+ bit = hash_ptr(val, IO_WQ_HASH_ORDER); -+ work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT)); -+} -+ -+static bool io_wq_worker_cancel(struct io_worker *worker, void *data) -+{ -+ struct io_cb_cancel_data *match = data; -+ -+ /* -+ * Hold the lock to avoid ->cur_work going out of scope, caller -+ * may dereference the passed in work. -+ */ -+ spin_lock(&worker->lock); -+ if (worker->cur_work && -+ match->fn(worker->cur_work, match->data)) { -+ set_notify_signal(worker->task); -+ match->nr_running++; -+ } -+ spin_unlock(&worker->lock); -+ -+ return match->nr_running && !match->cancel_all; -+} -+ -+static inline void io_wqe_remove_pending(struct io_wqe *wqe, -+ struct io_wq_work *work, -+ struct io_wq_work_node *prev) -+{ -+ struct io_wqe_acct *acct = io_work_get_acct(wqe, work); -+ unsigned int hash = io_get_work_hash(work); -+ struct io_wq_work *prev_work = NULL; -+ -+ if (io_wq_is_hashed(work) && work == wqe->hash_tail[hash]) { -+ if (prev) -+ prev_work = container_of(prev, struct io_wq_work, list); -+ if (prev_work && io_get_work_hash(prev_work) == hash) -+ wqe->hash_tail[hash] = prev_work; -+ else -+ wqe->hash_tail[hash] = NULL; -+ } -+ wq_list_del(&acct->work_list, &work->list, prev); -+} -+ -+static bool io_acct_cancel_pending_work(struct io_wqe *wqe, -+ struct io_wqe_acct *acct, -+ struct io_cb_cancel_data *match) -+ __releases(wqe->lock) -+{ -+ struct io_wq_work_node *node, *prev; -+ struct io_wq_work *work; -+ -+ wq_list_for_each(node, prev, &acct->work_list) { -+ work = container_of(node, struct io_wq_work, list); -+ if (!match->fn(work, match->data)) -+ continue; -+ io_wqe_remove_pending(wqe, work, prev); -+ raw_spin_unlock(&wqe->lock); -+ io_run_cancel(work, wqe); -+ match->nr_pending++; -+ /* not safe to continue after unlock */ -+ return true; -+ } -+ -+ return false; -+} -+ -+static void io_wqe_cancel_pending_work(struct io_wqe *wqe, -+ struct io_cb_cancel_data *match) -+{ -+ int i; -+retry: -+ raw_spin_lock(&wqe->lock); -+ for (i = 0; i < IO_WQ_ACCT_NR; i++) { -+ struct io_wqe_acct *acct = io_get_acct(wqe, i == 0); -+ -+ if (io_acct_cancel_pending_work(wqe, acct, match)) { -+ if (match->cancel_all) -+ goto retry; -+ return; -+ } -+ } -+ raw_spin_unlock(&wqe->lock); -+} -+ -+static void io_wqe_cancel_running_work(struct io_wqe *wqe, -+ struct io_cb_cancel_data *match) -+{ -+ rcu_read_lock(); -+ io_wq_for_each_worker(wqe, io_wq_worker_cancel, match); -+ rcu_read_unlock(); -+} -+ -+enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, -+ void *data, bool cancel_all) -+{ -+ struct io_cb_cancel_data match = { -+ .fn = cancel, -+ .data = data, -+ .cancel_all = cancel_all, -+ }; -+ int node; -+ -+ /* -+ * First check pending list, if we're lucky we can just remove it -+ * from there. CANCEL_OK means that the work is returned as-new, -+ * no completion will be posted for it. -+ */ -+ for_each_node(node) { -+ struct io_wqe *wqe = wq->wqes[node]; -+ -+ io_wqe_cancel_pending_work(wqe, &match); -+ if (match.nr_pending && !match.cancel_all) -+ return IO_WQ_CANCEL_OK; -+ } -+ -+ /* -+ * Now check if a free (going busy) or busy worker has the work -+ * currently running. If we find it there, we'll return CANCEL_RUNNING -+ * as an indication that we attempt to signal cancellation. The -+ * completion will run normally in this case. -+ */ -+ for_each_node(node) { -+ struct io_wqe *wqe = wq->wqes[node]; -+ -+ io_wqe_cancel_running_work(wqe, &match); -+ if (match.nr_running && !match.cancel_all) -+ return IO_WQ_CANCEL_RUNNING; -+ } -+ -+ if (match.nr_running) -+ return IO_WQ_CANCEL_RUNNING; -+ if (match.nr_pending) -+ return IO_WQ_CANCEL_OK; -+ return IO_WQ_CANCEL_NOTFOUND; -+} -+ -+static int io_wqe_hash_wake(struct wait_queue_entry *wait, unsigned mode, -+ int sync, void *key) -+{ -+ struct io_wqe *wqe = container_of(wait, struct io_wqe, wait); -+ int i; -+ -+ list_del_init(&wait->entry); -+ -+ rcu_read_lock(); -+ for (i = 0; i < IO_WQ_ACCT_NR; i++) { -+ struct io_wqe_acct *acct = &wqe->acct[i]; -+ -+ if (test_and_clear_bit(IO_ACCT_STALLED_BIT, &acct->flags)) -+ io_wqe_activate_free_worker(wqe, acct); -+ } -+ rcu_read_unlock(); -+ return 1; -+} -+ -+struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) -+{ -+ int ret, node, i; -+ struct io_wq *wq; -+ -+ if (WARN_ON_ONCE(!data->free_work || !data->do_work)) -+ return ERR_PTR(-EINVAL); -+ if (WARN_ON_ONCE(!bounded)) -+ return ERR_PTR(-EINVAL); -+ -+ wq = kzalloc(struct_size(wq, wqes, nr_node_ids), GFP_KERNEL); -+ if (!wq) -+ return ERR_PTR(-ENOMEM); -+ ret = cpuhp_state_add_instance_nocalls(io_wq_online, &wq->cpuhp_node); -+ if (ret) -+ goto err_wq; -+ -+ refcount_inc(&data->hash->refs); -+ wq->hash = data->hash; -+ wq->free_work = data->free_work; -+ wq->do_work = data->do_work; -+ -+ ret = -ENOMEM; -+ for_each_node(node) { -+ struct io_wqe *wqe; -+ int alloc_node = node; -+ -+ if (!node_online(alloc_node)) -+ alloc_node = NUMA_NO_NODE; -+ wqe = kzalloc_node(sizeof(struct io_wqe), GFP_KERNEL, alloc_node); -+ if (!wqe) -+ goto err; -+ wq->wqes[node] = wqe; -+ if (!alloc_cpumask_var(&wqe->cpu_mask, GFP_KERNEL)) -+ goto err; -+ cpumask_copy(wqe->cpu_mask, cpumask_of_node(node)); -+ wqe->node = alloc_node; -+ wqe->acct[IO_WQ_ACCT_BOUND].max_workers = bounded; -+ wqe->acct[IO_WQ_ACCT_UNBOUND].max_workers = -+ task_rlimit(current, RLIMIT_NPROC); -+ INIT_LIST_HEAD(&wqe->wait.entry); -+ wqe->wait.func = io_wqe_hash_wake; -+ for (i = 0; i < IO_WQ_ACCT_NR; i++) { -+ struct io_wqe_acct *acct = &wqe->acct[i]; -+ -+ acct->index = i; -+ atomic_set(&acct->nr_running, 0); -+ INIT_WQ_LIST(&acct->work_list); -+ } -+ wqe->wq = wq; -+ raw_spin_lock_init(&wqe->lock); -+ INIT_HLIST_NULLS_HEAD(&wqe->free_list, 0); -+ INIT_LIST_HEAD(&wqe->all_list); -+ } -+ -+ wq->task = get_task_struct(data->task); -+ atomic_set(&wq->worker_refs, 1); -+ init_completion(&wq->worker_done); -+ return wq; -+err: -+ io_wq_put_hash(data->hash); -+ cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); -+ for_each_node(node) { -+ if (!wq->wqes[node]) -+ continue; -+ free_cpumask_var(wq->wqes[node]->cpu_mask); -+ kfree(wq->wqes[node]); -+ } -+err_wq: -+ kfree(wq); -+ return ERR_PTR(ret); -+} -+ -+static bool io_task_work_match(struct callback_head *cb, void *data) -+{ -+ struct io_worker *worker; -+ -+ if (cb->func != create_worker_cb && cb->func != create_worker_cont) -+ return false; -+ worker = container_of(cb, struct io_worker, create_work); -+ return worker->wqe->wq == data; -+} -+ -+void io_wq_exit_start(struct io_wq *wq) -+{ -+ set_bit(IO_WQ_BIT_EXIT, &wq->state); -+} -+ -+static void io_wq_cancel_tw_create(struct io_wq *wq) -+{ -+ struct callback_head *cb; -+ -+ while ((cb = task_work_cancel_match(wq->task, io_task_work_match, wq)) != NULL) { -+ struct io_worker *worker; -+ -+ worker = container_of(cb, struct io_worker, create_work); -+ io_worker_cancel_cb(worker); -+ } -+} -+ -+static void io_wq_exit_workers(struct io_wq *wq) -+{ -+ int node; -+ -+ if (!wq->task) -+ return; -+ -+ io_wq_cancel_tw_create(wq); -+ -+ rcu_read_lock(); -+ for_each_node(node) { -+ struct io_wqe *wqe = wq->wqes[node]; -+ -+ io_wq_for_each_worker(wqe, io_wq_worker_wake, NULL); -+ } -+ rcu_read_unlock(); -+ io_worker_ref_put(wq); -+ wait_for_completion(&wq->worker_done); -+ -+ for_each_node(node) { -+ spin_lock_irq(&wq->hash->wait.lock); -+ list_del_init(&wq->wqes[node]->wait.entry); -+ spin_unlock_irq(&wq->hash->wait.lock); -+ } -+ put_task_struct(wq->task); -+ wq->task = NULL; -+} -+ -+static void io_wq_destroy(struct io_wq *wq) -+{ -+ int node; -+ -+ cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); -+ -+ for_each_node(node) { -+ struct io_wqe *wqe = wq->wqes[node]; -+ struct io_cb_cancel_data match = { -+ .fn = io_wq_work_match_all, -+ .cancel_all = true, -+ }; -+ io_wqe_cancel_pending_work(wqe, &match); -+ free_cpumask_var(wqe->cpu_mask); -+ kfree(wqe); -+ } -+ io_wq_put_hash(wq->hash); -+ kfree(wq); -+} -+ -+void io_wq_put_and_exit(struct io_wq *wq) -+{ -+ WARN_ON_ONCE(!test_bit(IO_WQ_BIT_EXIT, &wq->state)); -+ -+ io_wq_exit_workers(wq); -+ io_wq_destroy(wq); -+} -+ -+struct online_data { -+ unsigned int cpu; -+ bool online; -+}; -+ -+static bool io_wq_worker_affinity(struct io_worker *worker, void *data) -+{ -+ struct online_data *od = data; -+ -+ if (od->online) -+ cpumask_set_cpu(od->cpu, worker->wqe->cpu_mask); -+ else -+ cpumask_clear_cpu(od->cpu, worker->wqe->cpu_mask); -+ return false; -+} -+ -+static int __io_wq_cpu_online(struct io_wq *wq, unsigned int cpu, bool online) -+{ -+ struct online_data od = { -+ .cpu = cpu, -+ .online = online -+ }; -+ int i; -+ -+ rcu_read_lock(); -+ for_each_node(i) -+ io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, &od); -+ rcu_read_unlock(); -+ return 0; -+} -+ -+static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node) -+{ -+ struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); -+ -+ return __io_wq_cpu_online(wq, cpu, true); -+} -+ -+static int io_wq_cpu_offline(unsigned int cpu, struct hlist_node *node) -+{ -+ struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); -+ -+ return __io_wq_cpu_online(wq, cpu, false); -+} -+ -+int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask) -+{ -+ int i; -+ -+ rcu_read_lock(); -+ for_each_node(i) { -+ struct io_wqe *wqe = wq->wqes[i]; -+ -+ if (mask) -+ cpumask_copy(wqe->cpu_mask, mask); -+ else -+ cpumask_copy(wqe->cpu_mask, cpumask_of_node(i)); -+ } -+ rcu_read_unlock(); -+ return 0; -+} -+ -+/* -+ * Set max number of unbounded workers, returns old value. If new_count is 0, -+ * then just return the old value. -+ */ -+int io_wq_max_workers(struct io_wq *wq, int *new_count) -+{ -+ int prev[IO_WQ_ACCT_NR]; -+ bool first_node = true; -+ int i, node; -+ -+ BUILD_BUG_ON((int) IO_WQ_ACCT_BOUND != (int) IO_WQ_BOUND); -+ BUILD_BUG_ON((int) IO_WQ_ACCT_UNBOUND != (int) IO_WQ_UNBOUND); -+ BUILD_BUG_ON((int) IO_WQ_ACCT_NR != 2); -+ -+ for (i = 0; i < 2; i++) { -+ if (new_count[i] > task_rlimit(current, RLIMIT_NPROC)) -+ new_count[i] = task_rlimit(current, RLIMIT_NPROC); -+ } -+ -+ for (i = 0; i < IO_WQ_ACCT_NR; i++) -+ prev[i] = 0; -+ -+ rcu_read_lock(); -+ for_each_node(node) { -+ struct io_wqe *wqe = wq->wqes[node]; -+ struct io_wqe_acct *acct; -+ -+ raw_spin_lock(&wqe->lock); -+ for (i = 0; i < IO_WQ_ACCT_NR; i++) { -+ acct = &wqe->acct[i]; -+ if (first_node) -+ prev[i] = max_t(int, acct->max_workers, prev[i]); -+ if (new_count[i]) -+ acct->max_workers = new_count[i]; -+ } -+ raw_spin_unlock(&wqe->lock); -+ first_node = false; -+ } -+ rcu_read_unlock(); -+ -+ for (i = 0; i < IO_WQ_ACCT_NR; i++) -+ new_count[i] = prev[i]; -+ -+ return 0; -+} -+ -+static __init int io_wq_init(void) -+{ -+ int ret; -+ -+ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online", -+ io_wq_cpu_online, io_wq_cpu_offline); -+ if (ret < 0) -+ return ret; -+ io_wq_online = ret; -+ return 0; -+} -+subsys_initcall(io_wq_init); ---- /dev/null -+++ b/io_uring/io-wq.h -@@ -0,0 +1,160 @@ -+#ifndef INTERNAL_IO_WQ_H -+#define INTERNAL_IO_WQ_H -+ -+#include -+ -+struct io_wq; -+ -+enum { -+ IO_WQ_WORK_CANCEL = 1, -+ IO_WQ_WORK_HASHED = 2, -+ IO_WQ_WORK_UNBOUND = 4, -+ IO_WQ_WORK_CONCURRENT = 16, -+ -+ IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */ -+}; -+ -+enum io_wq_cancel { -+ IO_WQ_CANCEL_OK, /* cancelled before started */ -+ IO_WQ_CANCEL_RUNNING, /* found, running, and attempted cancelled */ -+ IO_WQ_CANCEL_NOTFOUND, /* work not found */ -+}; -+ -+struct io_wq_work_node { -+ struct io_wq_work_node *next; -+}; -+ -+struct io_wq_work_list { -+ struct io_wq_work_node *first; -+ struct io_wq_work_node *last; -+}; -+ -+static inline void wq_list_add_after(struct io_wq_work_node *node, -+ struct io_wq_work_node *pos, -+ struct io_wq_work_list *list) -+{ -+ struct io_wq_work_node *next = pos->next; -+ -+ pos->next = node; -+ node->next = next; -+ if (!next) -+ list->last = node; -+} -+ -+static inline void wq_list_add_tail(struct io_wq_work_node *node, -+ struct io_wq_work_list *list) -+{ -+ node->next = NULL; -+ if (!list->first) { -+ list->last = node; -+ WRITE_ONCE(list->first, node); -+ } else { -+ list->last->next = node; -+ list->last = node; -+ } -+} -+ -+static inline void wq_list_cut(struct io_wq_work_list *list, -+ struct io_wq_work_node *last, -+ struct io_wq_work_node *prev) -+{ -+ /* first in the list, if prev==NULL */ -+ if (!prev) -+ WRITE_ONCE(list->first, last->next); -+ else -+ prev->next = last->next; -+ -+ if (last == list->last) -+ list->last = prev; -+ last->next = NULL; -+} -+ -+static inline void wq_list_del(struct io_wq_work_list *list, -+ struct io_wq_work_node *node, -+ struct io_wq_work_node *prev) -+{ -+ wq_list_cut(list, node, prev); -+} -+ -+#define wq_list_for_each(pos, prv, head) \ -+ for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next) -+ -+#define wq_list_empty(list) (READ_ONCE((list)->first) == NULL) -+#define INIT_WQ_LIST(list) do { \ -+ (list)->first = NULL; \ -+ (list)->last = NULL; \ -+} while (0) -+ -+struct io_wq_work { -+ struct io_wq_work_node list; -+ unsigned flags; -+}; -+ -+static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) -+{ -+ if (!work->list.next) -+ return NULL; -+ -+ return container_of(work->list.next, struct io_wq_work, list); -+} -+ -+typedef struct io_wq_work *(free_work_fn)(struct io_wq_work *); -+typedef void (io_wq_work_fn)(struct io_wq_work *); -+ -+struct io_wq_hash { -+ refcount_t refs; -+ unsigned long map; -+ struct wait_queue_head wait; -+}; -+ -+static inline void io_wq_put_hash(struct io_wq_hash *hash) -+{ -+ if (refcount_dec_and_test(&hash->refs)) -+ kfree(hash); -+} -+ -+struct io_wq_data { -+ struct io_wq_hash *hash; -+ struct task_struct *task; -+ io_wq_work_fn *do_work; -+ free_work_fn *free_work; -+}; -+ -+struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); -+void io_wq_exit_start(struct io_wq *wq); -+void io_wq_put_and_exit(struct io_wq *wq); -+ -+void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); -+void io_wq_hash_work(struct io_wq_work *work, void *val); -+ -+int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask); -+int io_wq_max_workers(struct io_wq *wq, int *new_count); -+ -+static inline bool io_wq_is_hashed(struct io_wq_work *work) -+{ -+ return work->flags & IO_WQ_WORK_HASHED; -+} -+ -+typedef bool (work_cancel_fn)(struct io_wq_work *, void *); -+ -+enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, -+ void *data, bool cancel_all); -+ -+#if defined(CONFIG_IO_WQ) -+extern void io_wq_worker_sleeping(struct task_struct *); -+extern void io_wq_worker_running(struct task_struct *); -+#else -+static inline void io_wq_worker_sleeping(struct task_struct *tsk) -+{ -+} -+static inline void io_wq_worker_running(struct task_struct *tsk) -+{ -+} -+#endif -+ -+static inline bool io_wq_current_is_worker(void) -+{ -+ return in_task() && (current->flags & PF_IO_WORKER) && -+ current->pf_io_worker; -+} -+#endif ---- /dev/null -+++ b/io_uring/io_uring.c -@@ -0,0 +1,10945 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* -+ * Shared application/kernel submission and completion ring pairs, for -+ * supporting fast/efficient IO. -+ * -+ * A note on the read/write ordering memory barriers that are matched between -+ * the application and kernel side. -+ * -+ * After the application reads the CQ ring tail, it must use an -+ * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses -+ * before writing the tail (using smp_load_acquire to read the tail will -+ * do). It also needs a smp_mb() before updating CQ head (ordering the -+ * entry load(s) with the head store), pairing with an implicit barrier -+ * through a control-dependency in io_get_cqe (smp_store_release to -+ * store head will do). Failure to do so could lead to reading invalid -+ * CQ entries. -+ * -+ * Likewise, the application must use an appropriate smp_wmb() before -+ * writing the SQ tail (ordering SQ entry stores with the tail store), -+ * which pairs with smp_load_acquire in io_get_sqring (smp_store_release -+ * to store the tail will do). And it needs a barrier ordering the SQ -+ * head load before writing new SQ entries (smp_load_acquire to read -+ * head will do). -+ * -+ * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application -+ * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after* -+ * updating the SQ tail; a full memory barrier smp_mb() is needed -+ * between. -+ * -+ * Also see the examples in the liburing library: -+ * -+ * git://git.kernel.dk/liburing -+ * -+ * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens -+ * from data shared between the kernel and application. This is done both -+ * for ordering purposes, but also to ensure that once a value is loaded from -+ * data that the application could potentially modify, it remains stable. -+ * -+ * Copyright (C) 2018-2019 Jens Axboe -+ * Copyright (c) 2018-2019 Christoph Hellwig -+ */ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define CREATE_TRACE_POINTS -+#include -+ -+#include -+ -+#include "../fs/internal.h" -+#include "io-wq.h" -+ -+#define IORING_MAX_ENTRIES 32768 -+#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) -+#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8 -+ -+/* only define max */ -+#define IORING_MAX_FIXED_FILES (1U << 15) -+#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ -+ IORING_REGISTER_LAST + IORING_OP_LAST) -+ -+#define IO_RSRC_TAG_TABLE_SHIFT (PAGE_SHIFT - 3) -+#define IO_RSRC_TAG_TABLE_MAX (1U << IO_RSRC_TAG_TABLE_SHIFT) -+#define IO_RSRC_TAG_TABLE_MASK (IO_RSRC_TAG_TABLE_MAX - 1) -+ -+#define IORING_MAX_REG_BUFFERS (1U << 14) -+ -+#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ -+ IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ -+ IOSQE_BUFFER_SELECT) -+#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \ -+ REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS) -+ -+#define IO_TCTX_REFS_CACHE_NR (1U << 10) -+ -+struct io_uring { -+ u32 head ____cacheline_aligned_in_smp; -+ u32 tail ____cacheline_aligned_in_smp; -+}; -+ -+/* -+ * This data is shared with the application through the mmap at offsets -+ * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING. -+ * -+ * The offsets to the member fields are published through struct -+ * io_sqring_offsets when calling io_uring_setup. -+ */ -+struct io_rings { -+ /* -+ * Head and tail offsets into the ring; the offsets need to be -+ * masked to get valid indices. -+ * -+ * The kernel controls head of the sq ring and the tail of the cq ring, -+ * and the application controls tail of the sq ring and the head of the -+ * cq ring. -+ */ -+ struct io_uring sq, cq; -+ /* -+ * Bitmasks to apply to head and tail offsets (constant, equals -+ * ring_entries - 1) -+ */ -+ u32 sq_ring_mask, cq_ring_mask; -+ /* Ring sizes (constant, power of 2) */ -+ u32 sq_ring_entries, cq_ring_entries; -+ /* -+ * Number of invalid entries dropped by the kernel due to -+ * invalid index stored in array -+ * -+ * Written by the kernel, shouldn't be modified by the -+ * application (i.e. get number of "new events" by comparing to -+ * cached value). -+ * -+ * After a new SQ head value was read by the application this -+ * counter includes all submissions that were dropped reaching -+ * the new SQ head (and possibly more). -+ */ -+ u32 sq_dropped; -+ /* -+ * Runtime SQ flags -+ * -+ * Written by the kernel, shouldn't be modified by the -+ * application. -+ * -+ * The application needs a full memory barrier before checking -+ * for IORING_SQ_NEED_WAKEUP after updating the sq tail. -+ */ -+ u32 sq_flags; -+ /* -+ * Runtime CQ flags -+ * -+ * Written by the application, shouldn't be modified by the -+ * kernel. -+ */ -+ u32 cq_flags; -+ /* -+ * Number of completion events lost because the queue was full; -+ * this should be avoided by the application by making sure -+ * there are not more requests pending than there is space in -+ * the completion queue. -+ * -+ * Written by the kernel, shouldn't be modified by the -+ * application (i.e. get number of "new events" by comparing to -+ * cached value). -+ * -+ * As completion events come in out of order this counter is not -+ * ordered with any other data. -+ */ -+ u32 cq_overflow; -+ /* -+ * Ring buffer of completion events. -+ * -+ * The kernel writes completion events fresh every time they are -+ * produced, so the application is allowed to modify pending -+ * entries. -+ */ -+ struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp; -+}; -+ -+enum io_uring_cmd_flags { -+ IO_URING_F_NONBLOCK = 1, -+ IO_URING_F_COMPLETE_DEFER = 2, -+}; -+ -+struct io_mapped_ubuf { -+ u64 ubuf; -+ u64 ubuf_end; -+ unsigned int nr_bvecs; -+ unsigned long acct_pages; -+ struct bio_vec bvec[]; -+}; -+ -+struct io_ring_ctx; -+ -+struct io_overflow_cqe { -+ struct io_uring_cqe cqe; -+ struct list_head list; -+}; -+ -+struct io_fixed_file { -+ /* file * with additional FFS_* flags */ -+ unsigned long file_ptr; -+}; -+ -+struct io_rsrc_put { -+ struct list_head list; -+ u64 tag; -+ union { -+ void *rsrc; -+ struct file *file; -+ struct io_mapped_ubuf *buf; -+ }; -+}; -+ -+struct io_file_table { -+ struct io_fixed_file *files; -+}; -+ -+struct io_rsrc_node { -+ struct percpu_ref refs; -+ struct list_head node; -+ struct list_head rsrc_list; -+ struct io_rsrc_data *rsrc_data; -+ struct llist_node llist; -+ bool done; -+}; -+ -+typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc); -+ -+struct io_rsrc_data { -+ struct io_ring_ctx *ctx; -+ -+ u64 **tags; -+ unsigned int nr; -+ rsrc_put_fn *do_put; -+ atomic_t refs; -+ struct completion done; -+ bool quiesce; -+}; -+ -+struct io_buffer { -+ struct list_head list; -+ __u64 addr; -+ __u32 len; -+ __u16 bid; -+}; -+ -+struct io_restriction { -+ DECLARE_BITMAP(register_op, IORING_REGISTER_LAST); -+ DECLARE_BITMAP(sqe_op, IORING_OP_LAST); -+ u8 sqe_flags_allowed; -+ u8 sqe_flags_required; -+ bool registered; -+}; -+ -+enum { -+ IO_SQ_THREAD_SHOULD_STOP = 0, -+ IO_SQ_THREAD_SHOULD_PARK, -+}; -+ -+struct io_sq_data { -+ refcount_t refs; -+ atomic_t park_pending; -+ struct mutex lock; -+ -+ /* ctx's that are using this sqd */ -+ struct list_head ctx_list; -+ -+ struct task_struct *thread; -+ struct wait_queue_head wait; -+ -+ unsigned sq_thread_idle; -+ int sq_cpu; -+ pid_t task_pid; -+ pid_t task_tgid; -+ -+ unsigned long state; -+ struct completion exited; -+}; -+ -+#define IO_COMPL_BATCH 32 -+#define IO_REQ_CACHE_SIZE 32 -+#define IO_REQ_ALLOC_BATCH 8 -+ -+struct io_submit_link { -+ struct io_kiocb *head; -+ struct io_kiocb *last; -+}; -+ -+struct io_submit_state { -+ struct blk_plug plug; -+ struct io_submit_link link; -+ -+ /* -+ * io_kiocb alloc cache -+ */ -+ void *reqs[IO_REQ_CACHE_SIZE]; -+ unsigned int free_reqs; -+ -+ bool plug_started; -+ -+ /* -+ * Batch completion logic -+ */ -+ struct io_kiocb *compl_reqs[IO_COMPL_BATCH]; -+ unsigned int compl_nr; -+ /* inline/task_work completion list, under ->uring_lock */ -+ struct list_head free_list; -+ -+ unsigned int ios_left; -+}; -+ -+struct io_ring_ctx { -+ /* const or read-mostly hot data */ -+ struct { -+ struct percpu_ref refs; -+ -+ struct io_rings *rings; -+ unsigned int flags; -+ unsigned int compat: 1; -+ unsigned int drain_next: 1; -+ unsigned int eventfd_async: 1; -+ unsigned int restricted: 1; -+ unsigned int off_timeout_used: 1; -+ unsigned int drain_active: 1; -+ } ____cacheline_aligned_in_smp; -+ -+ /* submission data */ -+ struct { -+ struct mutex uring_lock; -+ -+ /* -+ * Ring buffer of indices into array of io_uring_sqe, which is -+ * mmapped by the application using the IORING_OFF_SQES offset. -+ * -+ * This indirection could e.g. be used to assign fixed -+ * io_uring_sqe entries to operations and only submit them to -+ * the queue when needed. -+ * -+ * The kernel modifies neither the indices array nor the entries -+ * array. -+ */ -+ u32 *sq_array; -+ struct io_uring_sqe *sq_sqes; -+ unsigned cached_sq_head; -+ unsigned sq_entries; -+ struct list_head defer_list; -+ -+ /* -+ * Fixed resources fast path, should be accessed only under -+ * uring_lock, and updated through io_uring_register(2) -+ */ -+ struct io_rsrc_node *rsrc_node; -+ struct io_file_table file_table; -+ unsigned nr_user_files; -+ unsigned nr_user_bufs; -+ struct io_mapped_ubuf **user_bufs; -+ -+ struct io_submit_state submit_state; -+ struct list_head timeout_list; -+ struct list_head ltimeout_list; -+ struct list_head cq_overflow_list; -+ struct xarray io_buffers; -+ struct xarray personalities; -+ u32 pers_next; -+ unsigned sq_thread_idle; -+ } ____cacheline_aligned_in_smp; -+ -+ /* IRQ completion list, under ->completion_lock */ -+ struct list_head locked_free_list; -+ unsigned int locked_free_nr; -+ -+ const struct cred *sq_creds; /* cred used for __io_sq_thread() */ -+ struct io_sq_data *sq_data; /* if using sq thread polling */ -+ -+ struct wait_queue_head sqo_sq_wait; -+ struct list_head sqd_list; -+ -+ unsigned long check_cq_overflow; -+ -+ struct { -+ unsigned cached_cq_tail; -+ unsigned cq_entries; -+ struct eventfd_ctx *cq_ev_fd; -+ struct wait_queue_head poll_wait; -+ struct wait_queue_head cq_wait; -+ unsigned cq_extra; -+ atomic_t cq_timeouts; -+ unsigned cq_last_tm_flush; -+ } ____cacheline_aligned_in_smp; -+ -+ struct { -+ spinlock_t completion_lock; -+ -+ spinlock_t timeout_lock; -+ -+ /* -+ * ->iopoll_list is protected by the ctx->uring_lock for -+ * io_uring instances that don't use IORING_SETUP_SQPOLL. -+ * For SQPOLL, only the single threaded io_sq_thread() will -+ * manipulate the list, hence no extra locking is needed there. -+ */ -+ struct list_head iopoll_list; -+ struct hlist_head *cancel_hash; -+ unsigned cancel_hash_bits; -+ bool poll_multi_queue; -+ } ____cacheline_aligned_in_smp; -+ -+ struct io_restriction restrictions; -+ -+ /* slow path rsrc auxilary data, used by update/register */ -+ struct { -+ struct io_rsrc_node *rsrc_backup_node; -+ struct io_mapped_ubuf *dummy_ubuf; -+ struct io_rsrc_data *file_data; -+ struct io_rsrc_data *buf_data; -+ -+ struct delayed_work rsrc_put_work; -+ struct llist_head rsrc_put_llist; -+ struct list_head rsrc_ref_list; -+ spinlock_t rsrc_ref_lock; -+ }; -+ -+ /* Keep this last, we don't need it for the fast path */ -+ struct { -+ #if defined(CONFIG_UNIX) -+ struct socket *ring_sock; -+ #endif -+ /* hashed buffered write serialization */ -+ struct io_wq_hash *hash_map; -+ -+ /* Only used for accounting purposes */ -+ struct user_struct *user; -+ struct mm_struct *mm_account; -+ -+ /* ctx exit and cancelation */ -+ struct llist_head fallback_llist; -+ struct delayed_work fallback_work; -+ struct work_struct exit_work; -+ struct list_head tctx_list; -+ struct completion ref_comp; -+ u32 iowq_limits[2]; -+ bool iowq_limits_set; -+ }; -+}; -+ -+struct io_uring_task { -+ /* submission side */ -+ int cached_refs; -+ struct xarray xa; -+ struct wait_queue_head wait; -+ const struct io_ring_ctx *last; -+ struct io_wq *io_wq; -+ struct percpu_counter inflight; -+ atomic_t inflight_tracked; -+ atomic_t in_idle; -+ -+ spinlock_t task_lock; -+ struct io_wq_work_list task_list; -+ struct callback_head task_work; -+ bool task_running; -+}; -+ -+/* -+ * First field must be the file pointer in all the -+ * iocb unions! See also 'struct kiocb' in -+ */ -+struct io_poll_iocb { -+ struct file *file; -+ struct wait_queue_head *head; -+ __poll_t events; -+ struct wait_queue_entry wait; -+}; -+ -+struct io_poll_update { -+ struct file *file; -+ u64 old_user_data; -+ u64 new_user_data; -+ __poll_t events; -+ bool update_events; -+ bool update_user_data; -+}; -+ -+struct io_close { -+ struct file *file; -+ int fd; -+ u32 file_slot; -+}; -+ -+struct io_timeout_data { -+ struct io_kiocb *req; -+ struct hrtimer timer; -+ struct timespec64 ts; -+ enum hrtimer_mode mode; -+ u32 flags; -+}; -+ -+struct io_accept { -+ struct file *file; -+ struct sockaddr __user *addr; -+ int __user *addr_len; -+ int flags; -+ u32 file_slot; -+ unsigned long nofile; -+}; -+ -+struct io_sync { -+ struct file *file; -+ loff_t len; -+ loff_t off; -+ int flags; -+ int mode; -+}; -+ -+struct io_cancel { -+ struct file *file; -+ u64 addr; -+}; -+ -+struct io_timeout { -+ struct file *file; -+ u32 off; -+ u32 target_seq; -+ struct list_head list; -+ /* head of the link, used by linked timeouts only */ -+ struct io_kiocb *head; -+ /* for linked completions */ -+ struct io_kiocb *prev; -+}; -+ -+struct io_timeout_rem { -+ struct file *file; -+ u64 addr; -+ -+ /* timeout update */ -+ struct timespec64 ts; -+ u32 flags; -+ bool ltimeout; -+}; -+ -+struct io_rw { -+ /* NOTE: kiocb has the file as the first member, so don't do it here */ -+ struct kiocb kiocb; -+ u64 addr; -+ u64 len; -+}; -+ -+struct io_connect { -+ struct file *file; -+ struct sockaddr __user *addr; -+ int addr_len; -+}; -+ -+struct io_sr_msg { -+ struct file *file; -+ union { -+ struct compat_msghdr __user *umsg_compat; -+ struct user_msghdr __user *umsg; -+ void __user *buf; -+ }; -+ int msg_flags; -+ int bgid; -+ size_t len; -+ struct io_buffer *kbuf; -+}; -+ -+struct io_open { -+ struct file *file; -+ int dfd; -+ u32 file_slot; -+ struct filename *filename; -+ struct open_how how; -+ unsigned long nofile; -+}; -+ -+struct io_rsrc_update { -+ struct file *file; -+ u64 arg; -+ u32 nr_args; -+ u32 offset; -+}; -+ -+struct io_fadvise { -+ struct file *file; -+ u64 offset; -+ u32 len; -+ u32 advice; -+}; -+ -+struct io_madvise { -+ struct file *file; -+ u64 addr; -+ u32 len; -+ u32 advice; -+}; -+ -+struct io_epoll { -+ struct file *file; -+ int epfd; -+ int op; -+ int fd; -+ struct epoll_event event; -+}; -+ -+struct io_splice { -+ struct file *file_out; -+ loff_t off_out; -+ loff_t off_in; -+ u64 len; -+ int splice_fd_in; -+ unsigned int flags; -+}; -+ -+struct io_provide_buf { -+ struct file *file; -+ __u64 addr; -+ __u32 len; -+ __u32 bgid; -+ __u16 nbufs; -+ __u16 bid; -+}; -+ -+struct io_statx { -+ struct file *file; -+ int dfd; -+ unsigned int mask; -+ unsigned int flags; -+ const char __user *filename; -+ struct statx __user *buffer; -+}; -+ -+struct io_shutdown { -+ struct file *file; -+ int how; -+}; -+ -+struct io_rename { -+ struct file *file; -+ int old_dfd; -+ int new_dfd; -+ struct filename *oldpath; -+ struct filename *newpath; -+ int flags; -+}; -+ -+struct io_unlink { -+ struct file *file; -+ int dfd; -+ int flags; -+ struct filename *filename; -+}; -+ -+struct io_mkdir { -+ struct file *file; -+ int dfd; -+ umode_t mode; -+ struct filename *filename; -+}; -+ -+struct io_symlink { -+ struct file *file; -+ int new_dfd; -+ struct filename *oldpath; -+ struct filename *newpath; -+}; -+ -+struct io_hardlink { -+ struct file *file; -+ int old_dfd; -+ int new_dfd; -+ struct filename *oldpath; -+ struct filename *newpath; -+ int flags; -+}; -+ -+struct io_completion { -+ struct file *file; -+ u32 cflags; -+}; -+ -+struct io_async_connect { -+ struct sockaddr_storage address; -+}; -+ -+struct io_async_msghdr { -+ struct iovec fast_iov[UIO_FASTIOV]; -+ /* points to an allocated iov, if NULL we use fast_iov instead */ -+ struct iovec *free_iov; -+ struct sockaddr __user *uaddr; -+ struct msghdr msg; -+ struct sockaddr_storage addr; -+}; -+ -+struct io_async_rw { -+ struct iovec fast_iov[UIO_FASTIOV]; -+ const struct iovec *free_iovec; -+ struct iov_iter iter; -+ struct iov_iter_state iter_state; -+ size_t bytes_done; -+ struct wait_page_queue wpq; -+}; -+ -+enum { -+ REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT, -+ REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT, -+ REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT, -+ REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT, -+ REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT, -+ REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT, -+ -+ /* first byte is taken by user flags, shift it to not overlap */ -+ REQ_F_FAIL_BIT = 8, -+ REQ_F_INFLIGHT_BIT, -+ REQ_F_CUR_POS_BIT, -+ REQ_F_NOWAIT_BIT, -+ REQ_F_LINK_TIMEOUT_BIT, -+ REQ_F_NEED_CLEANUP_BIT, -+ REQ_F_POLLED_BIT, -+ REQ_F_BUFFER_SELECTED_BIT, -+ REQ_F_COMPLETE_INLINE_BIT, -+ REQ_F_REISSUE_BIT, -+ REQ_F_CREDS_BIT, -+ REQ_F_REFCOUNT_BIT, -+ REQ_F_ARM_LTIMEOUT_BIT, -+ /* keep async read/write and isreg together and in order */ -+ REQ_F_NOWAIT_READ_BIT, -+ REQ_F_NOWAIT_WRITE_BIT, -+ REQ_F_ISREG_BIT, -+ -+ /* not a real bit, just to check we're not overflowing the space */ -+ __REQ_F_LAST_BIT, -+}; -+ -+enum { -+ /* ctx owns file */ -+ REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT), -+ /* drain existing IO first */ -+ REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT), -+ /* linked sqes */ -+ REQ_F_LINK = BIT(REQ_F_LINK_BIT), -+ /* doesn't sever on completion < 0 */ -+ REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT), -+ /* IOSQE_ASYNC */ -+ REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT), -+ /* IOSQE_BUFFER_SELECT */ -+ REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT), -+ -+ /* fail rest of links */ -+ REQ_F_FAIL = BIT(REQ_F_FAIL_BIT), -+ /* on inflight list, should be cancelled and waited on exit reliably */ -+ REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT), -+ /* read/write uses file position */ -+ REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT), -+ /* must not punt to workers */ -+ REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT), -+ /* has or had linked timeout */ -+ REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT), -+ /* needs cleanup */ -+ REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT), -+ /* already went through poll handler */ -+ REQ_F_POLLED = BIT(REQ_F_POLLED_BIT), -+ /* buffer already selected */ -+ REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT), -+ /* completion is deferred through io_comp_state */ -+ REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT), -+ /* caller should reissue async */ -+ REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT), -+ /* supports async reads */ -+ REQ_F_NOWAIT_READ = BIT(REQ_F_NOWAIT_READ_BIT), -+ /* supports async writes */ -+ REQ_F_NOWAIT_WRITE = BIT(REQ_F_NOWAIT_WRITE_BIT), -+ /* regular file */ -+ REQ_F_ISREG = BIT(REQ_F_ISREG_BIT), -+ /* has creds assigned */ -+ REQ_F_CREDS = BIT(REQ_F_CREDS_BIT), -+ /* skip refcounting if not set */ -+ REQ_F_REFCOUNT = BIT(REQ_F_REFCOUNT_BIT), -+ /* there is a linked timeout that has to be armed */ -+ REQ_F_ARM_LTIMEOUT = BIT(REQ_F_ARM_LTIMEOUT_BIT), -+}; -+ -+struct async_poll { -+ struct io_poll_iocb poll; -+ struct io_poll_iocb *double_poll; -+}; -+ -+typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked); -+ -+struct io_task_work { -+ union { -+ struct io_wq_work_node node; -+ struct llist_node fallback_node; -+ }; -+ io_req_tw_func_t func; -+}; -+ -+enum { -+ IORING_RSRC_FILE = 0, -+ IORING_RSRC_BUFFER = 1, -+}; -+ -+/* -+ * NOTE! Each of the iocb union members has the file pointer -+ * as the first entry in their struct definition. So you can -+ * access the file pointer through any of the sub-structs, -+ * or directly as just 'ki_filp' in this struct. -+ */ -+struct io_kiocb { -+ union { -+ struct file *file; -+ struct io_rw rw; -+ struct io_poll_iocb poll; -+ struct io_poll_update poll_update; -+ struct io_accept accept; -+ struct io_sync sync; -+ struct io_cancel cancel; -+ struct io_timeout timeout; -+ struct io_timeout_rem timeout_rem; -+ struct io_connect connect; -+ struct io_sr_msg sr_msg; -+ struct io_open open; -+ struct io_close close; -+ struct io_rsrc_update rsrc_update; -+ struct io_fadvise fadvise; -+ struct io_madvise madvise; -+ struct io_epoll epoll; -+ struct io_splice splice; -+ struct io_provide_buf pbuf; -+ struct io_statx statx; -+ struct io_shutdown shutdown; -+ struct io_rename rename; -+ struct io_unlink unlink; -+ struct io_mkdir mkdir; -+ struct io_symlink symlink; -+ struct io_hardlink hardlink; -+ /* use only after cleaning per-op data, see io_clean_op() */ -+ struct io_completion compl; -+ }; -+ -+ /* opcode allocated if it needs to store data for async defer */ -+ void *async_data; -+ u8 opcode; -+ /* polled IO has completed */ -+ u8 iopoll_completed; -+ -+ u16 buf_index; -+ u32 result; -+ -+ struct io_ring_ctx *ctx; -+ unsigned int flags; -+ atomic_t refs; -+ struct task_struct *task; -+ u64 user_data; -+ -+ struct io_kiocb *link; -+ struct percpu_ref *fixed_rsrc_refs; -+ -+ /* used with ctx->iopoll_list with reads/writes */ -+ struct list_head inflight_entry; -+ struct io_task_work io_task_work; -+ /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ -+ struct hlist_node hash_node; -+ struct async_poll *apoll; -+ struct io_wq_work work; -+ const struct cred *creds; -+ -+ /* store used ubuf, so we can prevent reloading */ -+ struct io_mapped_ubuf *imu; -+ /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */ -+ struct io_buffer *kbuf; -+ atomic_t poll_refs; -+}; -+ -+struct io_tctx_node { -+ struct list_head ctx_node; -+ struct task_struct *task; -+ struct io_ring_ctx *ctx; -+}; -+ -+struct io_defer_entry { -+ struct list_head list; -+ struct io_kiocb *req; -+ u32 seq; -+}; -+ -+struct io_op_def { -+ /* needs req->file assigned */ -+ unsigned needs_file : 1; -+ /* hash wq insertion if file is a regular file */ -+ unsigned hash_reg_file : 1; -+ /* unbound wq insertion if file is a non-regular file */ -+ unsigned unbound_nonreg_file : 1; -+ /* opcode is not supported by this kernel */ -+ unsigned not_supported : 1; -+ /* set if opcode supports polled "wait" */ -+ unsigned pollin : 1; -+ unsigned pollout : 1; -+ /* op supports buffer selection */ -+ unsigned buffer_select : 1; -+ /* do prep async if is going to be punted */ -+ unsigned needs_async_setup : 1; -+ /* should block plug */ -+ unsigned plug : 1; -+ /* size of async data needed, if any */ -+ unsigned short async_size; -+}; -+ -+static const struct io_op_def io_op_defs[] = { -+ [IORING_OP_NOP] = {}, -+ [IORING_OP_READV] = { -+ .needs_file = 1, -+ .unbound_nonreg_file = 1, -+ .pollin = 1, -+ .buffer_select = 1, -+ .needs_async_setup = 1, -+ .plug = 1, -+ .async_size = sizeof(struct io_async_rw), -+ }, -+ [IORING_OP_WRITEV] = { -+ .needs_file = 1, -+ .hash_reg_file = 1, -+ .unbound_nonreg_file = 1, -+ .pollout = 1, -+ .needs_async_setup = 1, -+ .plug = 1, -+ .async_size = sizeof(struct io_async_rw), -+ }, -+ [IORING_OP_FSYNC] = { -+ .needs_file = 1, -+ }, -+ [IORING_OP_READ_FIXED] = { -+ .needs_file = 1, -+ .unbound_nonreg_file = 1, -+ .pollin = 1, -+ .plug = 1, -+ .async_size = sizeof(struct io_async_rw), -+ }, -+ [IORING_OP_WRITE_FIXED] = { -+ .needs_file = 1, -+ .hash_reg_file = 1, -+ .unbound_nonreg_file = 1, -+ .pollout = 1, -+ .plug = 1, -+ .async_size = sizeof(struct io_async_rw), -+ }, -+ [IORING_OP_POLL_ADD] = { -+ .needs_file = 1, -+ .unbound_nonreg_file = 1, -+ }, -+ [IORING_OP_POLL_REMOVE] = {}, -+ [IORING_OP_SYNC_FILE_RANGE] = { -+ .needs_file = 1, -+ }, -+ [IORING_OP_SENDMSG] = { -+ .needs_file = 1, -+ .unbound_nonreg_file = 1, -+ .pollout = 1, -+ .needs_async_setup = 1, -+ .async_size = sizeof(struct io_async_msghdr), -+ }, -+ [IORING_OP_RECVMSG] = { -+ .needs_file = 1, -+ .unbound_nonreg_file = 1, -+ .pollin = 1, -+ .buffer_select = 1, -+ .needs_async_setup = 1, -+ .async_size = sizeof(struct io_async_msghdr), -+ }, -+ [IORING_OP_TIMEOUT] = { -+ .async_size = sizeof(struct io_timeout_data), -+ }, -+ [IORING_OP_TIMEOUT_REMOVE] = { -+ /* used by timeout updates' prep() */ -+ }, -+ [IORING_OP_ACCEPT] = { -+ .needs_file = 1, -+ .unbound_nonreg_file = 1, -+ .pollin = 1, -+ }, -+ [IORING_OP_ASYNC_CANCEL] = {}, -+ [IORING_OP_LINK_TIMEOUT] = { -+ .async_size = sizeof(struct io_timeout_data), -+ }, -+ [IORING_OP_CONNECT] = { -+ .needs_file = 1, -+ .unbound_nonreg_file = 1, -+ .pollout = 1, -+ .needs_async_setup = 1, -+ .async_size = sizeof(struct io_async_connect), -+ }, -+ [IORING_OP_FALLOCATE] = { -+ .needs_file = 1, -+ }, -+ [IORING_OP_OPENAT] = {}, -+ [IORING_OP_CLOSE] = {}, -+ [IORING_OP_FILES_UPDATE] = {}, -+ [IORING_OP_STATX] = {}, -+ [IORING_OP_READ] = { -+ .needs_file = 1, -+ .unbound_nonreg_file = 1, -+ .pollin = 1, -+ .buffer_select = 1, -+ .plug = 1, -+ .async_size = sizeof(struct io_async_rw), -+ }, -+ [IORING_OP_WRITE] = { -+ .needs_file = 1, -+ .hash_reg_file = 1, -+ .unbound_nonreg_file = 1, -+ .pollout = 1, -+ .plug = 1, -+ .async_size = sizeof(struct io_async_rw), -+ }, -+ [IORING_OP_FADVISE] = { -+ .needs_file = 1, -+ }, -+ [IORING_OP_MADVISE] = {}, -+ [IORING_OP_SEND] = { -+ .needs_file = 1, -+ .unbound_nonreg_file = 1, -+ .pollout = 1, -+ }, -+ [IORING_OP_RECV] = { -+ .needs_file = 1, -+ .unbound_nonreg_file = 1, -+ .pollin = 1, -+ .buffer_select = 1, -+ }, -+ [IORING_OP_OPENAT2] = { -+ }, -+ [IORING_OP_EPOLL_CTL] = { -+ .unbound_nonreg_file = 1, -+ }, -+ [IORING_OP_SPLICE] = { -+ .needs_file = 1, -+ .hash_reg_file = 1, -+ .unbound_nonreg_file = 1, -+ }, -+ [IORING_OP_PROVIDE_BUFFERS] = {}, -+ [IORING_OP_REMOVE_BUFFERS] = {}, -+ [IORING_OP_TEE] = { -+ .needs_file = 1, -+ .hash_reg_file = 1, -+ .unbound_nonreg_file = 1, -+ }, -+ [IORING_OP_SHUTDOWN] = { -+ .needs_file = 1, -+ }, -+ [IORING_OP_RENAMEAT] = {}, -+ [IORING_OP_UNLINKAT] = {}, -+}; -+ -+/* requests with any of those set should undergo io_disarm_next() */ -+#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL) -+ -+static bool io_disarm_next(struct io_kiocb *req); -+static void io_uring_del_tctx_node(unsigned long index); -+static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, -+ struct task_struct *task, -+ bool cancel_all); -+static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd); -+ -+static void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags); -+ -+static void io_put_req(struct io_kiocb *req); -+static void io_put_req_deferred(struct io_kiocb *req); -+static void io_dismantle_req(struct io_kiocb *req); -+static void io_queue_linked_timeout(struct io_kiocb *req); -+static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, -+ struct io_uring_rsrc_update2 *up, -+ unsigned nr_args); -+static void io_clean_op(struct io_kiocb *req); -+static struct file *io_file_get(struct io_ring_ctx *ctx, -+ struct io_kiocb *req, int fd, bool fixed); -+static void __io_queue_sqe(struct io_kiocb *req); -+static void io_rsrc_put_work(struct work_struct *work); -+ -+static void io_req_task_queue(struct io_kiocb *req); -+static void io_submit_flush_completions(struct io_ring_ctx *ctx); -+static int io_req_prep_async(struct io_kiocb *req); -+ -+static int io_install_fixed_file(struct io_kiocb *req, struct file *file, -+ unsigned int issue_flags, u32 slot_index); -+static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags); -+ -+static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer); -+ -+static struct kmem_cache *req_cachep; -+ -+static const struct file_operations io_uring_fops; -+ -+struct sock *io_uring_get_socket(struct file *file) -+{ -+#if defined(CONFIG_UNIX) -+ if (file->f_op == &io_uring_fops) { -+ struct io_ring_ctx *ctx = file->private_data; -+ -+ return ctx->ring_sock->sk; -+ } -+#endif -+ return NULL; -+} -+EXPORT_SYMBOL(io_uring_get_socket); -+ -+static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked) -+{ -+ if (!*locked) { -+ mutex_lock(&ctx->uring_lock); -+ *locked = true; -+ } -+} -+ -+#define io_for_each_link(pos, head) \ -+ for (pos = (head); pos; pos = pos->link) -+ -+/* -+ * Shamelessly stolen from the mm implementation of page reference checking, -+ * see commit f958d7b528b1 for details. -+ */ -+#define req_ref_zero_or_close_to_overflow(req) \ -+ ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u) -+ -+static inline bool req_ref_inc_not_zero(struct io_kiocb *req) -+{ -+ WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT)); -+ return atomic_inc_not_zero(&req->refs); -+} -+ -+static inline bool req_ref_put_and_test(struct io_kiocb *req) -+{ -+ if (likely(!(req->flags & REQ_F_REFCOUNT))) -+ return true; -+ -+ WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req)); -+ return atomic_dec_and_test(&req->refs); -+} -+ -+static inline void req_ref_get(struct io_kiocb *req) -+{ -+ WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT)); -+ WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req)); -+ atomic_inc(&req->refs); -+} -+ -+static inline void __io_req_set_refcount(struct io_kiocb *req, int nr) -+{ -+ if (!(req->flags & REQ_F_REFCOUNT)) { -+ req->flags |= REQ_F_REFCOUNT; -+ atomic_set(&req->refs, nr); -+ } -+} -+ -+static inline void io_req_set_refcount(struct io_kiocb *req) -+{ -+ __io_req_set_refcount(req, 1); -+} -+ -+static inline void io_req_set_rsrc_node(struct io_kiocb *req) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ if (!req->fixed_rsrc_refs) { -+ req->fixed_rsrc_refs = &ctx->rsrc_node->refs; -+ percpu_ref_get(req->fixed_rsrc_refs); -+ } -+} -+ -+static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl) -+{ -+ bool got = percpu_ref_tryget(ref); -+ -+ /* already at zero, wait for ->release() */ -+ if (!got) -+ wait_for_completion(compl); -+ percpu_ref_resurrect(ref); -+ if (got) -+ percpu_ref_put(ref); -+} -+ -+static bool io_match_task(struct io_kiocb *head, struct task_struct *task, -+ bool cancel_all) -+ __must_hold(&req->ctx->timeout_lock) -+{ -+ struct io_kiocb *req; -+ -+ if (task && head->task != task) -+ return false; -+ if (cancel_all) -+ return true; -+ -+ io_for_each_link(req, head) { -+ if (req->flags & REQ_F_INFLIGHT) -+ return true; -+ } -+ return false; -+} -+ -+static bool io_match_linked(struct io_kiocb *head) -+{ -+ struct io_kiocb *req; -+ -+ io_for_each_link(req, head) { -+ if (req->flags & REQ_F_INFLIGHT) -+ return true; -+ } -+ return false; -+} -+ -+/* -+ * As io_match_task() but protected against racing with linked timeouts. -+ * User must not hold timeout_lock. -+ */ -+static bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task, -+ bool cancel_all) -+{ -+ bool matched; -+ -+ if (task && head->task != task) -+ return false; -+ if (cancel_all) -+ return true; -+ -+ if (head->flags & REQ_F_LINK_TIMEOUT) { -+ struct io_ring_ctx *ctx = head->ctx; -+ -+ /* protect against races with linked timeouts */ -+ spin_lock_irq(&ctx->timeout_lock); -+ matched = io_match_linked(head); -+ spin_unlock_irq(&ctx->timeout_lock); -+ } else { -+ matched = io_match_linked(head); -+ } -+ return matched; -+} -+ -+static inline void req_set_fail(struct io_kiocb *req) -+{ -+ req->flags |= REQ_F_FAIL; -+} -+ -+static inline void req_fail_link_node(struct io_kiocb *req, int res) -+{ -+ req_set_fail(req); -+ req->result = res; -+} -+ -+static void io_ring_ctx_ref_free(struct percpu_ref *ref) -+{ -+ struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); -+ -+ complete(&ctx->ref_comp); -+} -+ -+static inline bool io_is_timeout_noseq(struct io_kiocb *req) -+{ -+ return !req->timeout.off; -+} -+ -+static void io_fallback_req_func(struct work_struct *work) -+{ -+ struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, -+ fallback_work.work); -+ struct llist_node *node = llist_del_all(&ctx->fallback_llist); -+ struct io_kiocb *req, *tmp; -+ bool locked = false; -+ -+ percpu_ref_get(&ctx->refs); -+ llist_for_each_entry_safe(req, tmp, node, io_task_work.fallback_node) -+ req->io_task_work.func(req, &locked); -+ -+ if (locked) { -+ if (ctx->submit_state.compl_nr) -+ io_submit_flush_completions(ctx); -+ mutex_unlock(&ctx->uring_lock); -+ } -+ percpu_ref_put(&ctx->refs); -+ -+} -+ -+static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) -+{ -+ struct io_ring_ctx *ctx; -+ int hash_bits; -+ -+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); -+ if (!ctx) -+ return NULL; -+ -+ /* -+ * Use 5 bits less than the max cq entries, that should give us around -+ * 32 entries per hash list if totally full and uniformly spread. -+ */ -+ hash_bits = ilog2(p->cq_entries); -+ hash_bits -= 5; -+ if (hash_bits <= 0) -+ hash_bits = 1; -+ ctx->cancel_hash_bits = hash_bits; -+ ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head), -+ GFP_KERNEL); -+ if (!ctx->cancel_hash) -+ goto err; -+ __hash_init(ctx->cancel_hash, 1U << hash_bits); -+ -+ ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL); -+ if (!ctx->dummy_ubuf) -+ goto err; -+ /* set invalid range, so io_import_fixed() fails meeting it */ -+ ctx->dummy_ubuf->ubuf = -1UL; -+ -+ if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free, -+ PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) -+ goto err; -+ -+ ctx->flags = p->flags; -+ init_waitqueue_head(&ctx->sqo_sq_wait); -+ INIT_LIST_HEAD(&ctx->sqd_list); -+ init_waitqueue_head(&ctx->poll_wait); -+ INIT_LIST_HEAD(&ctx->cq_overflow_list); -+ init_completion(&ctx->ref_comp); -+ xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1); -+ xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1); -+ mutex_init(&ctx->uring_lock); -+ init_waitqueue_head(&ctx->cq_wait); -+ spin_lock_init(&ctx->completion_lock); -+ spin_lock_init(&ctx->timeout_lock); -+ INIT_LIST_HEAD(&ctx->iopoll_list); -+ INIT_LIST_HEAD(&ctx->defer_list); -+ INIT_LIST_HEAD(&ctx->timeout_list); -+ INIT_LIST_HEAD(&ctx->ltimeout_list); -+ spin_lock_init(&ctx->rsrc_ref_lock); -+ INIT_LIST_HEAD(&ctx->rsrc_ref_list); -+ INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); -+ init_llist_head(&ctx->rsrc_put_llist); -+ INIT_LIST_HEAD(&ctx->tctx_list); -+ INIT_LIST_HEAD(&ctx->submit_state.free_list); -+ INIT_LIST_HEAD(&ctx->locked_free_list); -+ INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func); -+ return ctx; -+err: -+ kfree(ctx->dummy_ubuf); -+ kfree(ctx->cancel_hash); -+ kfree(ctx); -+ return NULL; -+} -+ -+static void io_account_cq_overflow(struct io_ring_ctx *ctx) -+{ -+ struct io_rings *r = ctx->rings; -+ -+ WRITE_ONCE(r->cq_overflow, READ_ONCE(r->cq_overflow) + 1); -+ ctx->cq_extra--; -+} -+ -+static bool req_need_defer(struct io_kiocb *req, u32 seq) -+{ -+ if (unlikely(req->flags & REQ_F_IO_DRAIN)) { -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ return seq + READ_ONCE(ctx->cq_extra) != ctx->cached_cq_tail; -+ } -+ -+ return false; -+} -+ -+#define FFS_ASYNC_READ 0x1UL -+#define FFS_ASYNC_WRITE 0x2UL -+#ifdef CONFIG_64BIT -+#define FFS_ISREG 0x4UL -+#else -+#define FFS_ISREG 0x0UL -+#endif -+#define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG) -+ -+static inline bool io_req_ffs_set(struct io_kiocb *req) -+{ -+ return IS_ENABLED(CONFIG_64BIT) && (req->flags & REQ_F_FIXED_FILE); -+} -+ -+static void io_req_track_inflight(struct io_kiocb *req) -+{ -+ if (!(req->flags & REQ_F_INFLIGHT)) { -+ req->flags |= REQ_F_INFLIGHT; -+ atomic_inc(&req->task->io_uring->inflight_tracked); -+ } -+} -+ -+static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req) -+{ -+ if (WARN_ON_ONCE(!req->link)) -+ return NULL; -+ -+ req->flags &= ~REQ_F_ARM_LTIMEOUT; -+ req->flags |= REQ_F_LINK_TIMEOUT; -+ -+ /* linked timeouts should have two refs once prep'ed */ -+ io_req_set_refcount(req); -+ __io_req_set_refcount(req->link, 2); -+ return req->link; -+} -+ -+static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) -+{ -+ if (likely(!(req->flags & REQ_F_ARM_LTIMEOUT))) -+ return NULL; -+ return __io_prep_linked_timeout(req); -+} -+ -+static void io_prep_async_work(struct io_kiocb *req) -+{ -+ const struct io_op_def *def = &io_op_defs[req->opcode]; -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ if (!(req->flags & REQ_F_CREDS)) { -+ req->flags |= REQ_F_CREDS; -+ req->creds = get_current_cred(); -+ } -+ -+ req->work.list.next = NULL; -+ req->work.flags = 0; -+ if (req->flags & REQ_F_FORCE_ASYNC) -+ req->work.flags |= IO_WQ_WORK_CONCURRENT; -+ -+ if (req->flags & REQ_F_ISREG) { -+ if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL)) -+ io_wq_hash_work(&req->work, file_inode(req->file)); -+ } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) { -+ if (def->unbound_nonreg_file) -+ req->work.flags |= IO_WQ_WORK_UNBOUND; -+ } -+} -+ -+static void io_prep_async_link(struct io_kiocb *req) -+{ -+ struct io_kiocb *cur; -+ -+ if (req->flags & REQ_F_LINK_TIMEOUT) { -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ spin_lock_irq(&ctx->timeout_lock); -+ io_for_each_link(cur, req) -+ io_prep_async_work(cur); -+ spin_unlock_irq(&ctx->timeout_lock); -+ } else { -+ io_for_each_link(cur, req) -+ io_prep_async_work(cur); -+ } -+} -+ -+static void io_queue_async_work(struct io_kiocb *req, bool *locked) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ struct io_kiocb *link = io_prep_linked_timeout(req); -+ struct io_uring_task *tctx = req->task->io_uring; -+ -+ /* must not take the lock, NULL it as a precaution */ -+ locked = NULL; -+ -+ BUG_ON(!tctx); -+ BUG_ON(!tctx->io_wq); -+ -+ /* init ->work of the whole link before punting */ -+ io_prep_async_link(req); -+ -+ /* -+ * Not expected to happen, but if we do have a bug where this _can_ -+ * happen, catch it here and ensure the request is marked as -+ * canceled. That will make io-wq go through the usual work cancel -+ * procedure rather than attempt to run this request (or create a new -+ * worker for it). -+ */ -+ if (WARN_ON_ONCE(!same_thread_group(req->task, current))) -+ req->work.flags |= IO_WQ_WORK_CANCEL; -+ -+ trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req, -+ &req->work, req->flags); -+ io_wq_enqueue(tctx->io_wq, &req->work); -+ if (link) -+ io_queue_linked_timeout(link); -+} -+ -+static void io_kill_timeout(struct io_kiocb *req, int status) -+ __must_hold(&req->ctx->completion_lock) -+ __must_hold(&req->ctx->timeout_lock) -+{ -+ struct io_timeout_data *io = req->async_data; -+ -+ if (hrtimer_try_to_cancel(&io->timer) != -1) { -+ if (status) -+ req_set_fail(req); -+ atomic_set(&req->ctx->cq_timeouts, -+ atomic_read(&req->ctx->cq_timeouts) + 1); -+ list_del_init(&req->timeout.list); -+ io_fill_cqe_req(req, status, 0); -+ io_put_req_deferred(req); -+ } -+} -+ -+static void io_queue_deferred(struct io_ring_ctx *ctx) -+{ -+ while (!list_empty(&ctx->defer_list)) { -+ struct io_defer_entry *de = list_first_entry(&ctx->defer_list, -+ struct io_defer_entry, list); -+ -+ if (req_need_defer(de->req, de->seq)) -+ break; -+ list_del_init(&de->list); -+ io_req_task_queue(de->req); -+ kfree(de); -+ } -+} -+ -+static void io_flush_timeouts(struct io_ring_ctx *ctx) -+ __must_hold(&ctx->completion_lock) -+{ -+ u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); -+ struct io_kiocb *req, *tmp; -+ -+ spin_lock_irq(&ctx->timeout_lock); -+ list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { -+ u32 events_needed, events_got; -+ -+ if (io_is_timeout_noseq(req)) -+ break; -+ -+ /* -+ * Since seq can easily wrap around over time, subtract -+ * the last seq at which timeouts were flushed before comparing. -+ * Assuming not more than 2^31-1 events have happened since, -+ * these subtractions won't have wrapped, so we can check if -+ * target is in [last_seq, current_seq] by comparing the two. -+ */ -+ events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush; -+ events_got = seq - ctx->cq_last_tm_flush; -+ if (events_got < events_needed) -+ break; -+ -+ io_kill_timeout(req, 0); -+ } -+ ctx->cq_last_tm_flush = seq; -+ spin_unlock_irq(&ctx->timeout_lock); -+} -+ -+static void __io_commit_cqring_flush(struct io_ring_ctx *ctx) -+{ -+ if (ctx->off_timeout_used) -+ io_flush_timeouts(ctx); -+ if (ctx->drain_active) -+ io_queue_deferred(ctx); -+} -+ -+static inline void io_commit_cqring(struct io_ring_ctx *ctx) -+{ -+ if (unlikely(ctx->off_timeout_used || ctx->drain_active)) -+ __io_commit_cqring_flush(ctx); -+ /* order cqe stores with ring update */ -+ smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); -+} -+ -+static inline bool io_sqring_full(struct io_ring_ctx *ctx) -+{ -+ struct io_rings *r = ctx->rings; -+ -+ return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == ctx->sq_entries; -+} -+ -+static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) -+{ -+ return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); -+} -+ -+static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) -+{ -+ struct io_rings *rings = ctx->rings; -+ unsigned tail, mask = ctx->cq_entries - 1; -+ -+ /* -+ * writes to the cq entry need to come after reading head; the -+ * control dependency is enough as we're using WRITE_ONCE to -+ * fill the cq entry -+ */ -+ if (__io_cqring_events(ctx) == ctx->cq_entries) -+ return NULL; -+ -+ tail = ctx->cached_cq_tail++; -+ return &rings->cqes[tail & mask]; -+} -+ -+static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx) -+{ -+ if (likely(!ctx->cq_ev_fd)) -+ return false; -+ if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) -+ return false; -+ return !ctx->eventfd_async || io_wq_current_is_worker(); -+} -+ -+/* -+ * This should only get called when at least one event has been posted. -+ * Some applications rely on the eventfd notification count only changing -+ * IFF a new CQE has been added to the CQ ring. There's no depedency on -+ * 1:1 relationship between how many times this function is called (and -+ * hence the eventfd count) and number of CQEs posted to the CQ ring. -+ */ -+static void io_cqring_ev_posted(struct io_ring_ctx *ctx) -+{ -+ /* -+ * wake_up_all() may seem excessive, but io_wake_function() and -+ * io_should_wake() handle the termination of the loop and only -+ * wake as many waiters as we need to. -+ */ -+ if (wq_has_sleeper(&ctx->cq_wait)) -+ wake_up_all(&ctx->cq_wait); -+ if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait)) -+ wake_up(&ctx->sq_data->wait); -+ if (io_should_trigger_evfd(ctx)) -+ eventfd_signal(ctx->cq_ev_fd, 1); -+ if (waitqueue_active(&ctx->poll_wait)) -+ wake_up_interruptible(&ctx->poll_wait); -+} -+ -+static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) -+{ -+ /* see waitqueue_active() comment */ -+ smp_mb(); -+ -+ if (ctx->flags & IORING_SETUP_SQPOLL) { -+ if (waitqueue_active(&ctx->cq_wait)) -+ wake_up_all(&ctx->cq_wait); -+ } -+ if (io_should_trigger_evfd(ctx)) -+ eventfd_signal(ctx->cq_ev_fd, 1); -+ if (waitqueue_active(&ctx->poll_wait)) -+ wake_up_interruptible(&ctx->poll_wait); -+} -+ -+/* Returns true if there are no backlogged entries after the flush */ -+static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) -+{ -+ bool all_flushed, posted; -+ -+ if (!force && __io_cqring_events(ctx) == ctx->cq_entries) -+ return false; -+ -+ posted = false; -+ spin_lock(&ctx->completion_lock); -+ while (!list_empty(&ctx->cq_overflow_list)) { -+ struct io_uring_cqe *cqe = io_get_cqe(ctx); -+ struct io_overflow_cqe *ocqe; -+ -+ if (!cqe && !force) -+ break; -+ ocqe = list_first_entry(&ctx->cq_overflow_list, -+ struct io_overflow_cqe, list); -+ if (cqe) -+ memcpy(cqe, &ocqe->cqe, sizeof(*cqe)); -+ else -+ io_account_cq_overflow(ctx); -+ -+ posted = true; -+ list_del(&ocqe->list); -+ kfree(ocqe); -+ } -+ -+ all_flushed = list_empty(&ctx->cq_overflow_list); -+ if (all_flushed) { -+ clear_bit(0, &ctx->check_cq_overflow); -+ WRITE_ONCE(ctx->rings->sq_flags, -+ ctx->rings->sq_flags & ~IORING_SQ_CQ_OVERFLOW); -+ } -+ -+ if (posted) -+ io_commit_cqring(ctx); -+ spin_unlock(&ctx->completion_lock); -+ if (posted) -+ io_cqring_ev_posted(ctx); -+ return all_flushed; -+} -+ -+static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx) -+{ -+ bool ret = true; -+ -+ if (test_bit(0, &ctx->check_cq_overflow)) { -+ /* iopoll syncs against uring_lock, not completion_lock */ -+ if (ctx->flags & IORING_SETUP_IOPOLL) -+ mutex_lock(&ctx->uring_lock); -+ ret = __io_cqring_overflow_flush(ctx, false); -+ if (ctx->flags & IORING_SETUP_IOPOLL) -+ mutex_unlock(&ctx->uring_lock); -+ } -+ -+ return ret; -+} -+ -+/* must to be called somewhat shortly after putting a request */ -+static inline void io_put_task(struct task_struct *task, int nr) -+{ -+ struct io_uring_task *tctx = task->io_uring; -+ -+ if (likely(task == current)) { -+ tctx->cached_refs += nr; -+ } else { -+ percpu_counter_sub(&tctx->inflight, nr); -+ if (unlikely(atomic_read(&tctx->in_idle))) -+ wake_up(&tctx->wait); -+ put_task_struct_many(task, nr); -+ } -+} -+ -+static void io_task_refs_refill(struct io_uring_task *tctx) -+{ -+ unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR; -+ -+ percpu_counter_add(&tctx->inflight, refill); -+ refcount_add(refill, ¤t->usage); -+ tctx->cached_refs += refill; -+} -+ -+static inline void io_get_task_refs(int nr) -+{ -+ struct io_uring_task *tctx = current->io_uring; -+ -+ tctx->cached_refs -= nr; -+ if (unlikely(tctx->cached_refs < 0)) -+ io_task_refs_refill(tctx); -+} -+ -+static __cold void io_uring_drop_tctx_refs(struct task_struct *task) -+{ -+ struct io_uring_task *tctx = task->io_uring; -+ unsigned int refs = tctx->cached_refs; -+ -+ if (refs) { -+ tctx->cached_refs = 0; -+ percpu_counter_sub(&tctx->inflight, refs); -+ put_task_struct_many(task, refs); -+ } -+} -+ -+static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, -+ s32 res, u32 cflags) -+{ -+ struct io_overflow_cqe *ocqe; -+ -+ ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT); -+ if (!ocqe) { -+ /* -+ * If we're in ring overflow flush mode, or in task cancel mode, -+ * or cannot allocate an overflow entry, then we need to drop it -+ * on the floor. -+ */ -+ io_account_cq_overflow(ctx); -+ return false; -+ } -+ if (list_empty(&ctx->cq_overflow_list)) { -+ set_bit(0, &ctx->check_cq_overflow); -+ WRITE_ONCE(ctx->rings->sq_flags, -+ ctx->rings->sq_flags | IORING_SQ_CQ_OVERFLOW); -+ -+ } -+ ocqe->cqe.user_data = user_data; -+ ocqe->cqe.res = res; -+ ocqe->cqe.flags = cflags; -+ list_add_tail(&ocqe->list, &ctx->cq_overflow_list); -+ return true; -+} -+ -+static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data, -+ s32 res, u32 cflags) -+{ -+ struct io_uring_cqe *cqe; -+ -+ trace_io_uring_complete(ctx, user_data, res, cflags); -+ -+ /* -+ * If we can't get a cq entry, userspace overflowed the -+ * submission (by quite a lot). Increment the overflow count in -+ * the ring. -+ */ -+ cqe = io_get_cqe(ctx); -+ if (likely(cqe)) { -+ WRITE_ONCE(cqe->user_data, user_data); -+ WRITE_ONCE(cqe->res, res); -+ WRITE_ONCE(cqe->flags, cflags); -+ return true; -+ } -+ return io_cqring_event_overflow(ctx, user_data, res, cflags); -+} -+ -+static noinline void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags) -+{ -+ __io_fill_cqe(req->ctx, req->user_data, res, cflags); -+} -+ -+static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, -+ s32 res, u32 cflags) -+{ -+ ctx->cq_extra++; -+ return __io_fill_cqe(ctx, user_data, res, cflags); -+} -+ -+static void io_req_complete_post(struct io_kiocb *req, s32 res, -+ u32 cflags) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ spin_lock(&ctx->completion_lock); -+ __io_fill_cqe(ctx, req->user_data, res, cflags); -+ /* -+ * If we're the last reference to this request, add to our locked -+ * free_list cache. -+ */ -+ if (req_ref_put_and_test(req)) { -+ if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { -+ if (req->flags & IO_DISARM_MASK) -+ io_disarm_next(req); -+ if (req->link) { -+ io_req_task_queue(req->link); -+ req->link = NULL; -+ } -+ } -+ io_dismantle_req(req); -+ io_put_task(req->task, 1); -+ list_add(&req->inflight_entry, &ctx->locked_free_list); -+ ctx->locked_free_nr++; -+ } else { -+ if (!percpu_ref_tryget(&ctx->refs)) -+ req = NULL; -+ } -+ io_commit_cqring(ctx); -+ spin_unlock(&ctx->completion_lock); -+ -+ if (req) { -+ io_cqring_ev_posted(ctx); -+ percpu_ref_put(&ctx->refs); -+ } -+} -+ -+static inline bool io_req_needs_clean(struct io_kiocb *req) -+{ -+ return req->flags & IO_REQ_CLEAN_FLAGS; -+} -+ -+static inline void io_req_complete_state(struct io_kiocb *req, s32 res, -+ u32 cflags) -+{ -+ if (io_req_needs_clean(req)) -+ io_clean_op(req); -+ req->result = res; -+ req->compl.cflags = cflags; -+ req->flags |= REQ_F_COMPLETE_INLINE; -+} -+ -+static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags, -+ s32 res, u32 cflags) -+{ -+ if (issue_flags & IO_URING_F_COMPLETE_DEFER) -+ io_req_complete_state(req, res, cflags); -+ else -+ io_req_complete_post(req, res, cflags); -+} -+ -+static inline void io_req_complete(struct io_kiocb *req, s32 res) -+{ -+ __io_req_complete(req, 0, res, 0); -+} -+ -+static void io_req_complete_failed(struct io_kiocb *req, s32 res) -+{ -+ req_set_fail(req); -+ io_req_complete_post(req, res, 0); -+} -+ -+static void io_req_complete_fail_submit(struct io_kiocb *req) -+{ -+ /* -+ * We don't submit, fail them all, for that replace hardlinks with -+ * normal links. Extra REQ_F_LINK is tolerated. -+ */ -+ req->flags &= ~REQ_F_HARDLINK; -+ req->flags |= REQ_F_LINK; -+ io_req_complete_failed(req, req->result); -+} -+ -+/* -+ * Don't initialise the fields below on every allocation, but do that in -+ * advance and keep them valid across allocations. -+ */ -+static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx) -+{ -+ req->ctx = ctx; -+ req->link = NULL; -+ req->async_data = NULL; -+ /* not necessary, but safer to zero */ -+ req->result = 0; -+} -+ -+static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx, -+ struct io_submit_state *state) -+{ -+ spin_lock(&ctx->completion_lock); -+ list_splice_init(&ctx->locked_free_list, &state->free_list); -+ ctx->locked_free_nr = 0; -+ spin_unlock(&ctx->completion_lock); -+} -+ -+/* Returns true IFF there are requests in the cache */ -+static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) -+{ -+ struct io_submit_state *state = &ctx->submit_state; -+ int nr; -+ -+ /* -+ * If we have more than a batch's worth of requests in our IRQ side -+ * locked cache, grab the lock and move them over to our submission -+ * side cache. -+ */ -+ if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH) -+ io_flush_cached_locked_reqs(ctx, state); -+ -+ nr = state->free_reqs; -+ while (!list_empty(&state->free_list)) { -+ struct io_kiocb *req = list_first_entry(&state->free_list, -+ struct io_kiocb, inflight_entry); -+ -+ list_del(&req->inflight_entry); -+ state->reqs[nr++] = req; -+ if (nr == ARRAY_SIZE(state->reqs)) -+ break; -+ } -+ -+ state->free_reqs = nr; -+ return nr != 0; -+} -+ -+/* -+ * A request might get retired back into the request caches even before opcode -+ * handlers and io_issue_sqe() are done with it, e.g. inline completion path. -+ * Because of that, io_alloc_req() should be called only under ->uring_lock -+ * and with extra caution to not get a request that is still worked on. -+ */ -+static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) -+ __must_hold(&ctx->uring_lock) -+{ -+ struct io_submit_state *state = &ctx->submit_state; -+ gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; -+ int ret, i; -+ -+ BUILD_BUG_ON(ARRAY_SIZE(state->reqs) < IO_REQ_ALLOC_BATCH); -+ -+ if (likely(state->free_reqs || io_flush_cached_reqs(ctx))) -+ goto got_req; -+ -+ ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH, -+ state->reqs); -+ -+ /* -+ * Bulk alloc is all-or-nothing. If we fail to get a batch, -+ * retry single alloc to be on the safe side. -+ */ -+ if (unlikely(ret <= 0)) { -+ state->reqs[0] = kmem_cache_alloc(req_cachep, gfp); -+ if (!state->reqs[0]) -+ return NULL; -+ ret = 1; -+ } -+ -+ for (i = 0; i < ret; i++) -+ io_preinit_req(state->reqs[i], ctx); -+ state->free_reqs = ret; -+got_req: -+ state->free_reqs--; -+ return state->reqs[state->free_reqs]; -+} -+ -+static inline void io_put_file(struct file *file) -+{ -+ if (file) -+ fput(file); -+} -+ -+static void io_dismantle_req(struct io_kiocb *req) -+{ -+ unsigned int flags = req->flags; -+ -+ if (io_req_needs_clean(req)) -+ io_clean_op(req); -+ if (!(flags & REQ_F_FIXED_FILE)) -+ io_put_file(req->file); -+ if (req->fixed_rsrc_refs) -+ percpu_ref_put(req->fixed_rsrc_refs); -+ if (req->async_data) { -+ kfree(req->async_data); -+ req->async_data = NULL; -+ } -+} -+ -+static void __io_free_req(struct io_kiocb *req) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ io_dismantle_req(req); -+ io_put_task(req->task, 1); -+ -+ spin_lock(&ctx->completion_lock); -+ list_add(&req->inflight_entry, &ctx->locked_free_list); -+ ctx->locked_free_nr++; -+ spin_unlock(&ctx->completion_lock); -+ -+ percpu_ref_put(&ctx->refs); -+} -+ -+static inline void io_remove_next_linked(struct io_kiocb *req) -+{ -+ struct io_kiocb *nxt = req->link; -+ -+ req->link = nxt->link; -+ nxt->link = NULL; -+} -+ -+static bool io_kill_linked_timeout(struct io_kiocb *req) -+ __must_hold(&req->ctx->completion_lock) -+ __must_hold(&req->ctx->timeout_lock) -+{ -+ struct io_kiocb *link = req->link; -+ -+ if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { -+ struct io_timeout_data *io = link->async_data; -+ -+ io_remove_next_linked(req); -+ link->timeout.head = NULL; -+ if (hrtimer_try_to_cancel(&io->timer) != -1) { -+ list_del(&link->timeout.list); -+ io_fill_cqe_req(link, -ECANCELED, 0); -+ io_put_req_deferred(link); -+ return true; -+ } -+ } -+ return false; -+} -+ -+static void io_fail_links(struct io_kiocb *req) -+ __must_hold(&req->ctx->completion_lock) -+{ -+ struct io_kiocb *nxt, *link = req->link; -+ -+ req->link = NULL; -+ while (link) { -+ long res = -ECANCELED; -+ -+ if (link->flags & REQ_F_FAIL) -+ res = link->result; -+ -+ nxt = link->link; -+ link->link = NULL; -+ -+ trace_io_uring_fail_link(req, link); -+ io_fill_cqe_req(link, res, 0); -+ io_put_req_deferred(link); -+ link = nxt; -+ } -+} -+ -+static bool io_disarm_next(struct io_kiocb *req) -+ __must_hold(&req->ctx->completion_lock) -+{ -+ bool posted = false; -+ -+ if (req->flags & REQ_F_ARM_LTIMEOUT) { -+ struct io_kiocb *link = req->link; -+ -+ req->flags &= ~REQ_F_ARM_LTIMEOUT; -+ if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { -+ io_remove_next_linked(req); -+ io_fill_cqe_req(link, -ECANCELED, 0); -+ io_put_req_deferred(link); -+ posted = true; -+ } -+ } else if (req->flags & REQ_F_LINK_TIMEOUT) { -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ spin_lock_irq(&ctx->timeout_lock); -+ posted = io_kill_linked_timeout(req); -+ spin_unlock_irq(&ctx->timeout_lock); -+ } -+ if (unlikely((req->flags & REQ_F_FAIL) && -+ !(req->flags & REQ_F_HARDLINK))) { -+ posted |= (req->link != NULL); -+ io_fail_links(req); -+ } -+ return posted; -+} -+ -+static struct io_kiocb *__io_req_find_next(struct io_kiocb *req) -+{ -+ struct io_kiocb *nxt; -+ -+ /* -+ * If LINK is set, we have dependent requests in this chain. If we -+ * didn't fail this request, queue the first one up, moving any other -+ * dependencies to the next request. In case of failure, fail the rest -+ * of the chain. -+ */ -+ if (req->flags & IO_DISARM_MASK) { -+ struct io_ring_ctx *ctx = req->ctx; -+ bool posted; -+ -+ spin_lock(&ctx->completion_lock); -+ posted = io_disarm_next(req); -+ if (posted) -+ io_commit_cqring(req->ctx); -+ spin_unlock(&ctx->completion_lock); -+ if (posted) -+ io_cqring_ev_posted(ctx); -+ } -+ nxt = req->link; -+ req->link = NULL; -+ return nxt; -+} -+ -+static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) -+{ -+ if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK)))) -+ return NULL; -+ return __io_req_find_next(req); -+} -+ -+static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked) -+{ -+ if (!ctx) -+ return; -+ if (*locked) { -+ if (ctx->submit_state.compl_nr) -+ io_submit_flush_completions(ctx); -+ mutex_unlock(&ctx->uring_lock); -+ *locked = false; -+ } -+ percpu_ref_put(&ctx->refs); -+} -+ -+static void tctx_task_work(struct callback_head *cb) -+{ -+ bool locked = false; -+ struct io_ring_ctx *ctx = NULL; -+ struct io_uring_task *tctx = container_of(cb, struct io_uring_task, -+ task_work); -+ -+ while (1) { -+ struct io_wq_work_node *node; -+ -+ if (!tctx->task_list.first && locked && ctx->submit_state.compl_nr) -+ io_submit_flush_completions(ctx); -+ -+ spin_lock_irq(&tctx->task_lock); -+ node = tctx->task_list.first; -+ INIT_WQ_LIST(&tctx->task_list); -+ if (!node) -+ tctx->task_running = false; -+ spin_unlock_irq(&tctx->task_lock); -+ if (!node) -+ break; -+ -+ do { -+ struct io_wq_work_node *next = node->next; -+ struct io_kiocb *req = container_of(node, struct io_kiocb, -+ io_task_work.node); -+ -+ if (req->ctx != ctx) { -+ ctx_flush_and_put(ctx, &locked); -+ ctx = req->ctx; -+ /* if not contended, grab and improve batching */ -+ locked = mutex_trylock(&ctx->uring_lock); -+ percpu_ref_get(&ctx->refs); -+ } -+ req->io_task_work.func(req, &locked); -+ node = next; -+ } while (node); -+ -+ cond_resched(); -+ } -+ -+ ctx_flush_and_put(ctx, &locked); -+ -+ /* relaxed read is enough as only the task itself sets ->in_idle */ -+ if (unlikely(atomic_read(&tctx->in_idle))) -+ io_uring_drop_tctx_refs(current); -+} -+ -+static void io_req_task_work_add(struct io_kiocb *req) -+{ -+ struct task_struct *tsk = req->task; -+ struct io_uring_task *tctx = tsk->io_uring; -+ enum task_work_notify_mode notify; -+ struct io_wq_work_node *node; -+ unsigned long flags; -+ bool running; -+ -+ WARN_ON_ONCE(!tctx); -+ -+ spin_lock_irqsave(&tctx->task_lock, flags); -+ wq_list_add_tail(&req->io_task_work.node, &tctx->task_list); -+ running = tctx->task_running; -+ if (!running) -+ tctx->task_running = true; -+ spin_unlock_irqrestore(&tctx->task_lock, flags); -+ -+ /* task_work already pending, we're done */ -+ if (running) -+ return; -+ -+ /* -+ * SQPOLL kernel thread doesn't need notification, just a wakeup. For -+ * all other cases, use TWA_SIGNAL unconditionally to ensure we're -+ * processing task_work. There's no reliable way to tell if TWA_RESUME -+ * will do the job. -+ */ -+ notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL; -+ if (!task_work_add(tsk, &tctx->task_work, notify)) { -+ wake_up_process(tsk); -+ return; -+ } -+ -+ spin_lock_irqsave(&tctx->task_lock, flags); -+ tctx->task_running = false; -+ node = tctx->task_list.first; -+ INIT_WQ_LIST(&tctx->task_list); -+ spin_unlock_irqrestore(&tctx->task_lock, flags); -+ -+ while (node) { -+ req = container_of(node, struct io_kiocb, io_task_work.node); -+ node = node->next; -+ if (llist_add(&req->io_task_work.fallback_node, -+ &req->ctx->fallback_llist)) -+ schedule_delayed_work(&req->ctx->fallback_work, 1); -+ } -+} -+ -+static void io_req_task_cancel(struct io_kiocb *req, bool *locked) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ /* not needed for normal modes, but SQPOLL depends on it */ -+ io_tw_lock(ctx, locked); -+ io_req_complete_failed(req, req->result); -+} -+ -+static void io_req_task_submit(struct io_kiocb *req, bool *locked) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ io_tw_lock(ctx, locked); -+ /* req->task == current here, checking PF_EXITING is safe */ -+ if (likely(!(req->task->flags & PF_EXITING))) -+ __io_queue_sqe(req); -+ else -+ io_req_complete_failed(req, -EFAULT); -+} -+ -+static void io_req_task_queue_fail(struct io_kiocb *req, int ret) -+{ -+ req->result = ret; -+ req->io_task_work.func = io_req_task_cancel; -+ io_req_task_work_add(req); -+} -+ -+static void io_req_task_queue(struct io_kiocb *req) -+{ -+ req->io_task_work.func = io_req_task_submit; -+ io_req_task_work_add(req); -+} -+ -+static void io_req_task_queue_reissue(struct io_kiocb *req) -+{ -+ req->io_task_work.func = io_queue_async_work; -+ io_req_task_work_add(req); -+} -+ -+static inline void io_queue_next(struct io_kiocb *req) -+{ -+ struct io_kiocb *nxt = io_req_find_next(req); -+ -+ if (nxt) -+ io_req_task_queue(nxt); -+} -+ -+static void io_free_req(struct io_kiocb *req) -+{ -+ io_queue_next(req); -+ __io_free_req(req); -+} -+ -+static void io_free_req_work(struct io_kiocb *req, bool *locked) -+{ -+ io_free_req(req); -+} -+ -+struct req_batch { -+ struct task_struct *task; -+ int task_refs; -+ int ctx_refs; -+}; -+ -+static inline void io_init_req_batch(struct req_batch *rb) -+{ -+ rb->task_refs = 0; -+ rb->ctx_refs = 0; -+ rb->task = NULL; -+} -+ -+static void io_req_free_batch_finish(struct io_ring_ctx *ctx, -+ struct req_batch *rb) -+{ -+ if (rb->ctx_refs) -+ percpu_ref_put_many(&ctx->refs, rb->ctx_refs); -+ if (rb->task) -+ io_put_task(rb->task, rb->task_refs); -+} -+ -+static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, -+ struct io_submit_state *state) -+{ -+ io_queue_next(req); -+ io_dismantle_req(req); -+ -+ if (req->task != rb->task) { -+ if (rb->task) -+ io_put_task(rb->task, rb->task_refs); -+ rb->task = req->task; -+ rb->task_refs = 0; -+ } -+ rb->task_refs++; -+ rb->ctx_refs++; -+ -+ if (state->free_reqs != ARRAY_SIZE(state->reqs)) -+ state->reqs[state->free_reqs++] = req; -+ else -+ list_add(&req->inflight_entry, &state->free_list); -+} -+ -+static void io_submit_flush_completions(struct io_ring_ctx *ctx) -+ __must_hold(&ctx->uring_lock) -+{ -+ struct io_submit_state *state = &ctx->submit_state; -+ int i, nr = state->compl_nr; -+ struct req_batch rb; -+ -+ spin_lock(&ctx->completion_lock); -+ for (i = 0; i < nr; i++) { -+ struct io_kiocb *req = state->compl_reqs[i]; -+ -+ __io_fill_cqe(ctx, req->user_data, req->result, -+ req->compl.cflags); -+ } -+ io_commit_cqring(ctx); -+ spin_unlock(&ctx->completion_lock); -+ io_cqring_ev_posted(ctx); -+ -+ io_init_req_batch(&rb); -+ for (i = 0; i < nr; i++) { -+ struct io_kiocb *req = state->compl_reqs[i]; -+ -+ if (req_ref_put_and_test(req)) -+ io_req_free_batch(&rb, req, &ctx->submit_state); -+ } -+ -+ io_req_free_batch_finish(ctx, &rb); -+ state->compl_nr = 0; -+} -+ -+/* -+ * Drop reference to request, return next in chain (if there is one) if this -+ * was the last reference to this request. -+ */ -+static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req) -+{ -+ struct io_kiocb *nxt = NULL; -+ -+ if (req_ref_put_and_test(req)) { -+ nxt = io_req_find_next(req); -+ __io_free_req(req); -+ } -+ return nxt; -+} -+ -+static inline void io_put_req(struct io_kiocb *req) -+{ -+ if (req_ref_put_and_test(req)) -+ io_free_req(req); -+} -+ -+static inline void io_put_req_deferred(struct io_kiocb *req) -+{ -+ if (req_ref_put_and_test(req)) { -+ req->io_task_work.func = io_free_req_work; -+ io_req_task_work_add(req); -+ } -+} -+ -+static unsigned io_cqring_events(struct io_ring_ctx *ctx) -+{ -+ /* See comment at the top of this file */ -+ smp_rmb(); -+ return __io_cqring_events(ctx); -+} -+ -+static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) -+{ -+ struct io_rings *rings = ctx->rings; -+ -+ /* make sure SQ entry isn't read before tail */ -+ return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; -+} -+ -+static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf) -+{ -+ unsigned int cflags; -+ -+ cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT; -+ cflags |= IORING_CQE_F_BUFFER; -+ req->flags &= ~REQ_F_BUFFER_SELECTED; -+ kfree(kbuf); -+ return cflags; -+} -+ -+static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req) -+{ -+ struct io_buffer *kbuf; -+ -+ if (likely(!(req->flags & REQ_F_BUFFER_SELECTED))) -+ return 0; -+ kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; -+ return io_put_kbuf(req, kbuf); -+} -+ -+static inline bool io_run_task_work(void) -+{ -+ if (test_thread_flag(TIF_NOTIFY_SIGNAL) || current->task_works) { -+ __set_current_state(TASK_RUNNING); -+ tracehook_notify_signal(); -+ return true; -+ } -+ -+ return false; -+} -+ -+/* -+ * Find and free completed poll iocbs -+ */ -+static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, -+ struct list_head *done) -+{ -+ struct req_batch rb; -+ struct io_kiocb *req; -+ -+ /* order with ->result store in io_complete_rw_iopoll() */ -+ smp_rmb(); -+ -+ io_init_req_batch(&rb); -+ while (!list_empty(done)) { -+ req = list_first_entry(done, struct io_kiocb, inflight_entry); -+ list_del(&req->inflight_entry); -+ -+ io_fill_cqe_req(req, req->result, io_put_rw_kbuf(req)); -+ (*nr_events)++; -+ -+ if (req_ref_put_and_test(req)) -+ io_req_free_batch(&rb, req, &ctx->submit_state); -+ } -+ -+ io_commit_cqring(ctx); -+ io_cqring_ev_posted_iopoll(ctx); -+ io_req_free_batch_finish(ctx, &rb); -+} -+ -+static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, -+ long min) -+{ -+ struct io_kiocb *req, *tmp; -+ LIST_HEAD(done); -+ bool spin; -+ -+ /* -+ * Only spin for completions if we don't have multiple devices hanging -+ * off our complete list, and we're under the requested amount. -+ */ -+ spin = !ctx->poll_multi_queue && *nr_events < min; -+ -+ list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) { -+ struct kiocb *kiocb = &req->rw.kiocb; -+ int ret; -+ -+ /* -+ * Move completed and retryable entries to our local lists. -+ * If we find a request that requires polling, break out -+ * and complete those lists first, if we have entries there. -+ */ -+ if (READ_ONCE(req->iopoll_completed)) { -+ list_move_tail(&req->inflight_entry, &done); -+ continue; -+ } -+ if (!list_empty(&done)) -+ break; -+ -+ ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin); -+ if (unlikely(ret < 0)) -+ return ret; -+ else if (ret) -+ spin = false; -+ -+ /* iopoll may have completed current req */ -+ if (READ_ONCE(req->iopoll_completed)) -+ list_move_tail(&req->inflight_entry, &done); -+ } -+ -+ if (!list_empty(&done)) -+ io_iopoll_complete(ctx, nr_events, &done); -+ -+ return 0; -+} -+ -+/* -+ * We can't just wait for polled events to come to us, we have to actively -+ * find and complete them. -+ */ -+static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx) -+{ -+ if (!(ctx->flags & IORING_SETUP_IOPOLL)) -+ return; -+ -+ mutex_lock(&ctx->uring_lock); -+ while (!list_empty(&ctx->iopoll_list)) { -+ unsigned int nr_events = 0; -+ -+ io_do_iopoll(ctx, &nr_events, 0); -+ -+ /* let it sleep and repeat later if can't complete a request */ -+ if (nr_events == 0) -+ break; -+ /* -+ * Ensure we allow local-to-the-cpu processing to take place, -+ * in this case we need to ensure that we reap all events. -+ * Also let task_work, etc. to progress by releasing the mutex -+ */ -+ if (need_resched()) { -+ mutex_unlock(&ctx->uring_lock); -+ cond_resched(); -+ mutex_lock(&ctx->uring_lock); -+ } -+ } -+ mutex_unlock(&ctx->uring_lock); -+} -+ -+static int io_iopoll_check(struct io_ring_ctx *ctx, long min) -+{ -+ unsigned int nr_events = 0; -+ int ret = 0; -+ -+ /* -+ * We disallow the app entering submit/complete with polling, but we -+ * still need to lock the ring to prevent racing with polled issue -+ * that got punted to a workqueue. -+ */ -+ mutex_lock(&ctx->uring_lock); -+ /* -+ * Don't enter poll loop if we already have events pending. -+ * If we do, we can potentially be spinning for commands that -+ * already triggered a CQE (eg in error). -+ */ -+ if (test_bit(0, &ctx->check_cq_overflow)) -+ __io_cqring_overflow_flush(ctx, false); -+ if (io_cqring_events(ctx)) -+ goto out; -+ do { -+ /* -+ * If a submit got punted to a workqueue, we can have the -+ * application entering polling for a command before it gets -+ * issued. That app will hold the uring_lock for the duration -+ * of the poll right here, so we need to take a breather every -+ * now and then to ensure that the issue has a chance to add -+ * the poll to the issued list. Otherwise we can spin here -+ * forever, while the workqueue is stuck trying to acquire the -+ * very same mutex. -+ */ -+ if (list_empty(&ctx->iopoll_list)) { -+ u32 tail = ctx->cached_cq_tail; -+ -+ mutex_unlock(&ctx->uring_lock); -+ io_run_task_work(); -+ mutex_lock(&ctx->uring_lock); -+ -+ /* some requests don't go through iopoll_list */ -+ if (tail != ctx->cached_cq_tail || -+ list_empty(&ctx->iopoll_list)) -+ break; -+ } -+ ret = io_do_iopoll(ctx, &nr_events, min); -+ } while (!ret && nr_events < min && !need_resched()); -+out: -+ mutex_unlock(&ctx->uring_lock); -+ return ret; -+} -+ -+static void kiocb_end_write(struct io_kiocb *req) -+{ -+ /* -+ * Tell lockdep we inherited freeze protection from submission -+ * thread. -+ */ -+ if (req->flags & REQ_F_ISREG) { -+ struct super_block *sb = file_inode(req->file)->i_sb; -+ -+ __sb_writers_acquired(sb, SB_FREEZE_WRITE); -+ sb_end_write(sb); -+ } -+} -+ -+#ifdef CONFIG_BLOCK -+static bool io_resubmit_prep(struct io_kiocb *req) -+{ -+ struct io_async_rw *rw = req->async_data; -+ -+ if (!rw) -+ return !io_req_prep_async(req); -+ iov_iter_restore(&rw->iter, &rw->iter_state); -+ return true; -+} -+ -+static bool io_rw_should_reissue(struct io_kiocb *req) -+{ -+ umode_t mode = file_inode(req->file)->i_mode; -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ if (!S_ISBLK(mode) && !S_ISREG(mode)) -+ return false; -+ if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() && -+ !(ctx->flags & IORING_SETUP_IOPOLL))) -+ return false; -+ /* -+ * If ref is dying, we might be running poll reap from the exit work. -+ * Don't attempt to reissue from that path, just let it fail with -+ * -EAGAIN. -+ */ -+ if (percpu_ref_is_dying(&ctx->refs)) -+ return false; -+ /* -+ * Play it safe and assume not safe to re-import and reissue if we're -+ * not in the original thread group (or in task context). -+ */ -+ if (!same_thread_group(req->task, current) || !in_task()) -+ return false; -+ return true; -+} -+#else -+static bool io_resubmit_prep(struct io_kiocb *req) -+{ -+ return false; -+} -+static bool io_rw_should_reissue(struct io_kiocb *req) -+{ -+ return false; -+} -+#endif -+ -+static bool __io_complete_rw_common(struct io_kiocb *req, long res) -+{ -+ if (req->rw.kiocb.ki_flags & IOCB_WRITE) { -+ kiocb_end_write(req); -+ fsnotify_modify(req->file); -+ } else { -+ fsnotify_access(req->file); -+ } -+ if (res != req->result) { -+ if ((res == -EAGAIN || res == -EOPNOTSUPP) && -+ io_rw_should_reissue(req)) { -+ req->flags |= REQ_F_REISSUE; -+ return true; -+ } -+ req_set_fail(req); -+ req->result = res; -+ } -+ return false; -+} -+ -+static inline int io_fixup_rw_res(struct io_kiocb *req, unsigned res) -+{ -+ struct io_async_rw *io = req->async_data; -+ -+ /* add previously done IO, if any */ -+ if (io && io->bytes_done > 0) { -+ if (res < 0) -+ res = io->bytes_done; -+ else -+ res += io->bytes_done; -+ } -+ return res; -+} -+ -+static void io_req_task_complete(struct io_kiocb *req, bool *locked) -+{ -+ unsigned int cflags = io_put_rw_kbuf(req); -+ int res = req->result; -+ -+ if (*locked) { -+ struct io_ring_ctx *ctx = req->ctx; -+ struct io_submit_state *state = &ctx->submit_state; -+ -+ io_req_complete_state(req, res, cflags); -+ state->compl_reqs[state->compl_nr++] = req; -+ if (state->compl_nr == ARRAY_SIZE(state->compl_reqs)) -+ io_submit_flush_completions(ctx); -+ } else { -+ io_req_complete_post(req, res, cflags); -+ } -+} -+ -+static void __io_complete_rw(struct io_kiocb *req, long res, long res2, -+ unsigned int issue_flags) -+{ -+ if (__io_complete_rw_common(req, res)) -+ return; -+ __io_req_complete(req, issue_flags, io_fixup_rw_res(req, res), io_put_rw_kbuf(req)); -+} -+ -+static void io_complete_rw(struct kiocb *kiocb, long res, long res2) -+{ -+ struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); -+ -+ if (__io_complete_rw_common(req, res)) -+ return; -+ req->result = io_fixup_rw_res(req, res); -+ req->io_task_work.func = io_req_task_complete; -+ io_req_task_work_add(req); -+} -+ -+static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) -+{ -+ struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); -+ -+ if (kiocb->ki_flags & IOCB_WRITE) -+ kiocb_end_write(req); -+ if (unlikely(res != req->result)) { -+ if (res == -EAGAIN && io_rw_should_reissue(req)) { -+ req->flags |= REQ_F_REISSUE; -+ return; -+ } -+ } -+ -+ WRITE_ONCE(req->result, res); -+ /* order with io_iopoll_complete() checking ->result */ -+ smp_wmb(); -+ WRITE_ONCE(req->iopoll_completed, 1); -+} -+ -+/* -+ * After the iocb has been issued, it's safe to be found on the poll list. -+ * Adding the kiocb to the list AFTER submission ensures that we don't -+ * find it from a io_do_iopoll() thread before the issuer is done -+ * accessing the kiocb cookie. -+ */ -+static void io_iopoll_req_issued(struct io_kiocb *req) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ const bool in_async = io_wq_current_is_worker(); -+ -+ /* workqueue context doesn't hold uring_lock, grab it now */ -+ if (unlikely(in_async)) -+ mutex_lock(&ctx->uring_lock); -+ -+ /* -+ * Track whether we have multiple files in our lists. This will impact -+ * how we do polling eventually, not spinning if we're on potentially -+ * different devices. -+ */ -+ if (list_empty(&ctx->iopoll_list)) { -+ ctx->poll_multi_queue = false; -+ } else if (!ctx->poll_multi_queue) { -+ struct io_kiocb *list_req; -+ unsigned int queue_num0, queue_num1; -+ -+ list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb, -+ inflight_entry); -+ -+ if (list_req->file != req->file) { -+ ctx->poll_multi_queue = true; -+ } else { -+ queue_num0 = blk_qc_t_to_queue_num(list_req->rw.kiocb.ki_cookie); -+ queue_num1 = blk_qc_t_to_queue_num(req->rw.kiocb.ki_cookie); -+ if (queue_num0 != queue_num1) -+ ctx->poll_multi_queue = true; -+ } -+ } -+ -+ /* -+ * For fast devices, IO may have already completed. If it has, add -+ * it to the front so we find it first. -+ */ -+ if (READ_ONCE(req->iopoll_completed)) -+ list_add(&req->inflight_entry, &ctx->iopoll_list); -+ else -+ list_add_tail(&req->inflight_entry, &ctx->iopoll_list); -+ -+ if (unlikely(in_async)) { -+ /* -+ * If IORING_SETUP_SQPOLL is enabled, sqes are either handle -+ * in sq thread task context or in io worker task context. If -+ * current task context is sq thread, we don't need to check -+ * whether should wake up sq thread. -+ */ -+ if ((ctx->flags & IORING_SETUP_SQPOLL) && -+ wq_has_sleeper(&ctx->sq_data->wait)) -+ wake_up(&ctx->sq_data->wait); -+ -+ mutex_unlock(&ctx->uring_lock); -+ } -+} -+ -+static bool io_bdev_nowait(struct block_device *bdev) -+{ -+ return !bdev || blk_queue_nowait(bdev_get_queue(bdev)); -+} -+ -+/* -+ * If we tracked the file through the SCM inflight mechanism, we could support -+ * any file. For now, just ensure that anything potentially problematic is done -+ * inline. -+ */ -+static bool __io_file_supports_nowait(struct file *file, int rw) -+{ -+ umode_t mode = file_inode(file)->i_mode; -+ -+ if (S_ISBLK(mode)) { -+ if (IS_ENABLED(CONFIG_BLOCK) && -+ io_bdev_nowait(I_BDEV(file->f_mapping->host))) -+ return true; -+ return false; -+ } -+ if (S_ISSOCK(mode)) -+ return true; -+ if (S_ISREG(mode)) { -+ if (IS_ENABLED(CONFIG_BLOCK) && -+ io_bdev_nowait(file->f_inode->i_sb->s_bdev) && -+ file->f_op != &io_uring_fops) -+ return true; -+ return false; -+ } -+ -+ /* any ->read/write should understand O_NONBLOCK */ -+ if (file->f_flags & O_NONBLOCK) -+ return true; -+ -+ if (!(file->f_mode & FMODE_NOWAIT)) -+ return false; -+ -+ if (rw == READ) -+ return file->f_op->read_iter != NULL; -+ -+ return file->f_op->write_iter != NULL; -+} -+ -+static bool io_file_supports_nowait(struct io_kiocb *req, int rw) -+{ -+ if (rw == READ && (req->flags & REQ_F_NOWAIT_READ)) -+ return true; -+ else if (rw == WRITE && (req->flags & REQ_F_NOWAIT_WRITE)) -+ return true; -+ -+ return __io_file_supports_nowait(req->file, rw); -+} -+ -+static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, -+ int rw) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ struct kiocb *kiocb = &req->rw.kiocb; -+ struct file *file = req->file; -+ unsigned ioprio; -+ int ret; -+ -+ if (!io_req_ffs_set(req) && S_ISREG(file_inode(file)->i_mode)) -+ req->flags |= REQ_F_ISREG; -+ -+ kiocb->ki_pos = READ_ONCE(sqe->off); -+ if (kiocb->ki_pos == -1) { -+ if (!(file->f_mode & FMODE_STREAM)) { -+ req->flags |= REQ_F_CUR_POS; -+ kiocb->ki_pos = file->f_pos; -+ } else { -+ kiocb->ki_pos = 0; -+ } -+ } -+ kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp)); -+ kiocb->ki_flags = iocb_flags(kiocb->ki_filp); -+ ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags)); -+ if (unlikely(ret)) -+ return ret; -+ -+ /* -+ * If the file is marked O_NONBLOCK, still allow retry for it if it -+ * supports async. Otherwise it's impossible to use O_NONBLOCK files -+ * reliably. If not, or it IOCB_NOWAIT is set, don't retry. -+ */ -+ if ((kiocb->ki_flags & IOCB_NOWAIT) || -+ ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req, rw))) -+ req->flags |= REQ_F_NOWAIT; -+ -+ ioprio = READ_ONCE(sqe->ioprio); -+ if (ioprio) { -+ ret = ioprio_check_cap(ioprio); -+ if (ret) -+ return ret; -+ -+ kiocb->ki_ioprio = ioprio; -+ } else -+ kiocb->ki_ioprio = get_current_ioprio(); -+ -+ if (ctx->flags & IORING_SETUP_IOPOLL) { -+ if (!(kiocb->ki_flags & IOCB_DIRECT) || -+ !kiocb->ki_filp->f_op->iopoll) -+ return -EOPNOTSUPP; -+ -+ kiocb->ki_flags |= IOCB_HIPRI; -+ kiocb->ki_complete = io_complete_rw_iopoll; -+ req->iopoll_completed = 0; -+ } else { -+ if (kiocb->ki_flags & IOCB_HIPRI) -+ return -EINVAL; -+ kiocb->ki_complete = io_complete_rw; -+ } -+ -+ /* used for fixed read/write too - just read unconditionally */ -+ req->buf_index = READ_ONCE(sqe->buf_index); -+ req->imu = NULL; -+ -+ if (req->opcode == IORING_OP_READ_FIXED || -+ req->opcode == IORING_OP_WRITE_FIXED) { -+ struct io_ring_ctx *ctx = req->ctx; -+ u16 index; -+ -+ if (unlikely(req->buf_index >= ctx->nr_user_bufs)) -+ return -EFAULT; -+ index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); -+ req->imu = ctx->user_bufs[index]; -+ io_req_set_rsrc_node(req); -+ } -+ -+ req->rw.addr = READ_ONCE(sqe->addr); -+ req->rw.len = READ_ONCE(sqe->len); -+ return 0; -+} -+ -+static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) -+{ -+ switch (ret) { -+ case -EIOCBQUEUED: -+ break; -+ case -ERESTARTSYS: -+ case -ERESTARTNOINTR: -+ case -ERESTARTNOHAND: -+ case -ERESTART_RESTARTBLOCK: -+ /* -+ * We can't just restart the syscall, since previously -+ * submitted sqes may already be in progress. Just fail this -+ * IO with EINTR. -+ */ -+ ret = -EINTR; -+ fallthrough; -+ default: -+ kiocb->ki_complete(kiocb, ret, 0); -+ } -+} -+ -+static void kiocb_done(struct kiocb *kiocb, ssize_t ret, -+ unsigned int issue_flags) -+{ -+ struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); -+ -+ if (req->flags & REQ_F_CUR_POS) -+ req->file->f_pos = kiocb->ki_pos; -+ if (ret >= 0 && (kiocb->ki_complete == io_complete_rw)) -+ __io_complete_rw(req, ret, 0, issue_flags); -+ else -+ io_rw_done(kiocb, ret); -+ -+ if (req->flags & REQ_F_REISSUE) { -+ req->flags &= ~REQ_F_REISSUE; -+ if (io_resubmit_prep(req)) { -+ io_req_task_queue_reissue(req); -+ } else { -+ unsigned int cflags = io_put_rw_kbuf(req); -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ ret = io_fixup_rw_res(req, ret); -+ req_set_fail(req); -+ if (!(issue_flags & IO_URING_F_NONBLOCK)) { -+ mutex_lock(&ctx->uring_lock); -+ __io_req_complete(req, issue_flags, ret, cflags); -+ mutex_unlock(&ctx->uring_lock); -+ } else { -+ __io_req_complete(req, issue_flags, ret, cflags); -+ } -+ } -+ } -+} -+ -+static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter, -+ struct io_mapped_ubuf *imu) -+{ -+ size_t len = req->rw.len; -+ u64 buf_end, buf_addr = req->rw.addr; -+ size_t offset; -+ -+ if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end))) -+ return -EFAULT; -+ /* not inside the mapped region */ -+ if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end)) -+ return -EFAULT; -+ -+ /* -+ * May not be a start of buffer, set size appropriately -+ * and advance us to the beginning. -+ */ -+ offset = buf_addr - imu->ubuf; -+ iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len); -+ -+ if (offset) { -+ /* -+ * Don't use iov_iter_advance() here, as it's really slow for -+ * using the latter parts of a big fixed buffer - it iterates -+ * over each segment manually. We can cheat a bit here, because -+ * we know that: -+ * -+ * 1) it's a BVEC iter, we set it up -+ * 2) all bvecs are PAGE_SIZE in size, except potentially the -+ * first and last bvec -+ * -+ * So just find our index, and adjust the iterator afterwards. -+ * If the offset is within the first bvec (or the whole first -+ * bvec, just use iov_iter_advance(). This makes it easier -+ * since we can just skip the first segment, which may not -+ * be PAGE_SIZE aligned. -+ */ -+ const struct bio_vec *bvec = imu->bvec; -+ -+ if (offset <= bvec->bv_len) { -+ iov_iter_advance(iter, offset); -+ } else { -+ unsigned long seg_skip; -+ -+ /* skip first vec */ -+ offset -= bvec->bv_len; -+ seg_skip = 1 + (offset >> PAGE_SHIFT); -+ -+ iter->bvec = bvec + seg_skip; -+ iter->nr_segs -= seg_skip; -+ iter->count -= bvec->bv_len + offset; -+ iter->iov_offset = offset & ~PAGE_MASK; -+ } -+ } -+ -+ return 0; -+} -+ -+static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter) -+{ -+ if (WARN_ON_ONCE(!req->imu)) -+ return -EFAULT; -+ return __io_import_fixed(req, rw, iter, req->imu); -+} -+ -+static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock) -+{ -+ if (needs_lock) -+ mutex_unlock(&ctx->uring_lock); -+} -+ -+static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock) -+{ -+ /* -+ * "Normal" inline submissions always hold the uring_lock, since we -+ * grab it from the system call. Same is true for the SQPOLL offload. -+ * The only exception is when we've detached the request and issue it -+ * from an async worker thread, grab the lock for that case. -+ */ -+ if (needs_lock) -+ mutex_lock(&ctx->uring_lock); -+} -+ -+static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len, -+ int bgid, struct io_buffer *kbuf, -+ bool needs_lock) -+{ -+ struct io_buffer *head; -+ -+ if (req->flags & REQ_F_BUFFER_SELECTED) -+ return kbuf; -+ -+ io_ring_submit_lock(req->ctx, needs_lock); -+ -+ lockdep_assert_held(&req->ctx->uring_lock); -+ -+ head = xa_load(&req->ctx->io_buffers, bgid); -+ if (head) { -+ if (!list_empty(&head->list)) { -+ kbuf = list_last_entry(&head->list, struct io_buffer, -+ list); -+ list_del(&kbuf->list); -+ } else { -+ kbuf = head; -+ xa_erase(&req->ctx->io_buffers, bgid); -+ } -+ if (*len > kbuf->len) -+ *len = kbuf->len; -+ } else { -+ kbuf = ERR_PTR(-ENOBUFS); -+ } -+ -+ io_ring_submit_unlock(req->ctx, needs_lock); -+ -+ return kbuf; -+} -+ -+static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len, -+ bool needs_lock) -+{ -+ struct io_buffer *kbuf; -+ u16 bgid; -+ -+ kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; -+ bgid = req->buf_index; -+ kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock); -+ if (IS_ERR(kbuf)) -+ return kbuf; -+ req->rw.addr = (u64) (unsigned long) kbuf; -+ req->flags |= REQ_F_BUFFER_SELECTED; -+ return u64_to_user_ptr(kbuf->addr); -+} -+ -+#ifdef CONFIG_COMPAT -+static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, -+ bool needs_lock) -+{ -+ struct compat_iovec __user *uiov; -+ compat_ssize_t clen; -+ void __user *buf; -+ ssize_t len; -+ -+ uiov = u64_to_user_ptr(req->rw.addr); -+ if (!access_ok(uiov, sizeof(*uiov))) -+ return -EFAULT; -+ if (__get_user(clen, &uiov->iov_len)) -+ return -EFAULT; -+ if (clen < 0) -+ return -EINVAL; -+ -+ len = clen; -+ buf = io_rw_buffer_select(req, &len, needs_lock); -+ if (IS_ERR(buf)) -+ return PTR_ERR(buf); -+ iov[0].iov_base = buf; -+ iov[0].iov_len = (compat_size_t) len; -+ return 0; -+} -+#endif -+ -+static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, -+ bool needs_lock) -+{ -+ struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr); -+ void __user *buf; -+ ssize_t len; -+ -+ if (copy_from_user(iov, uiov, sizeof(*uiov))) -+ return -EFAULT; -+ -+ len = iov[0].iov_len; -+ if (len < 0) -+ return -EINVAL; -+ buf = io_rw_buffer_select(req, &len, needs_lock); -+ if (IS_ERR(buf)) -+ return PTR_ERR(buf); -+ iov[0].iov_base = buf; -+ iov[0].iov_len = len; -+ return 0; -+} -+ -+static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, -+ bool needs_lock) -+{ -+ if (req->flags & REQ_F_BUFFER_SELECTED) { -+ struct io_buffer *kbuf; -+ -+ kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; -+ iov[0].iov_base = u64_to_user_ptr(kbuf->addr); -+ iov[0].iov_len = kbuf->len; -+ return 0; -+ } -+ if (req->rw.len != 1) -+ return -EINVAL; -+ -+#ifdef CONFIG_COMPAT -+ if (req->ctx->compat) -+ return io_compat_import(req, iov, needs_lock); -+#endif -+ -+ return __io_iov_buffer_select(req, iov, needs_lock); -+} -+ -+static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, -+ struct iov_iter *iter, bool needs_lock) -+{ -+ void __user *buf = u64_to_user_ptr(req->rw.addr); -+ size_t sqe_len = req->rw.len; -+ u8 opcode = req->opcode; -+ ssize_t ret; -+ -+ if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) { -+ *iovec = NULL; -+ return io_import_fixed(req, rw, iter); -+ } -+ -+ /* buffer index only valid with fixed read/write, or buffer select */ -+ if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT)) -+ return -EINVAL; -+ -+ if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) { -+ if (req->flags & REQ_F_BUFFER_SELECT) { -+ buf = io_rw_buffer_select(req, &sqe_len, needs_lock); -+ if (IS_ERR(buf)) -+ return PTR_ERR(buf); -+ req->rw.len = sqe_len; -+ } -+ -+ ret = import_single_range(rw, buf, sqe_len, *iovec, iter); -+ *iovec = NULL; -+ return ret; -+ } -+ -+ if (req->flags & REQ_F_BUFFER_SELECT) { -+ ret = io_iov_buffer_select(req, *iovec, needs_lock); -+ if (!ret) -+ iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len); -+ *iovec = NULL; -+ return ret; -+ } -+ -+ return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter, -+ req->ctx->compat); -+} -+ -+static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb) -+{ -+ return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos; -+} -+ -+/* -+ * For files that don't have ->read_iter() and ->write_iter(), handle them -+ * by looping over ->read() or ->write() manually. -+ */ -+static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) -+{ -+ struct kiocb *kiocb = &req->rw.kiocb; -+ struct file *file = req->file; -+ ssize_t ret = 0; -+ -+ /* -+ * Don't support polled IO through this interface, and we can't -+ * support non-blocking either. For the latter, this just causes -+ * the kiocb to be handled from an async context. -+ */ -+ if (kiocb->ki_flags & IOCB_HIPRI) -+ return -EOPNOTSUPP; -+ if (kiocb->ki_flags & IOCB_NOWAIT) -+ return -EAGAIN; -+ -+ while (iov_iter_count(iter)) { -+ struct iovec iovec; -+ ssize_t nr; -+ -+ if (!iov_iter_is_bvec(iter)) { -+ iovec = iov_iter_iovec(iter); -+ } else { -+ iovec.iov_base = u64_to_user_ptr(req->rw.addr); -+ iovec.iov_len = req->rw.len; -+ } -+ -+ if (rw == READ) { -+ nr = file->f_op->read(file, iovec.iov_base, -+ iovec.iov_len, io_kiocb_ppos(kiocb)); -+ } else { -+ nr = file->f_op->write(file, iovec.iov_base, -+ iovec.iov_len, io_kiocb_ppos(kiocb)); -+ } -+ -+ if (nr < 0) { -+ if (!ret) -+ ret = nr; -+ break; -+ } -+ ret += nr; -+ if (!iov_iter_is_bvec(iter)) { -+ iov_iter_advance(iter, nr); -+ } else { -+ req->rw.addr += nr; -+ req->rw.len -= nr; -+ if (!req->rw.len) -+ break; -+ } -+ if (nr != iovec.iov_len) -+ break; -+ } -+ -+ return ret; -+} -+ -+static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, -+ const struct iovec *fast_iov, struct iov_iter *iter) -+{ -+ struct io_async_rw *rw = req->async_data; -+ -+ memcpy(&rw->iter, iter, sizeof(*iter)); -+ rw->free_iovec = iovec; -+ rw->bytes_done = 0; -+ /* can only be fixed buffers, no need to do anything */ -+ if (iov_iter_is_bvec(iter)) -+ return; -+ if (!iovec) { -+ unsigned iov_off = 0; -+ -+ rw->iter.iov = rw->fast_iov; -+ if (iter->iov != fast_iov) { -+ iov_off = iter->iov - fast_iov; -+ rw->iter.iov += iov_off; -+ } -+ if (rw->fast_iov != fast_iov) -+ memcpy(rw->fast_iov + iov_off, fast_iov + iov_off, -+ sizeof(struct iovec) * iter->nr_segs); -+ } else { -+ req->flags |= REQ_F_NEED_CLEANUP; -+ } -+} -+ -+static inline int io_alloc_async_data(struct io_kiocb *req) -+{ -+ WARN_ON_ONCE(!io_op_defs[req->opcode].async_size); -+ req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL); -+ return req->async_data == NULL; -+} -+ -+static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, -+ const struct iovec *fast_iov, -+ struct iov_iter *iter, bool force) -+{ -+ if (!force && !io_op_defs[req->opcode].needs_async_setup) -+ return 0; -+ if (!req->async_data) { -+ struct io_async_rw *iorw; -+ -+ if (io_alloc_async_data(req)) { -+ kfree(iovec); -+ return -ENOMEM; -+ } -+ -+ io_req_map_rw(req, iovec, fast_iov, iter); -+ iorw = req->async_data; -+ /* we've copied and mapped the iter, ensure state is saved */ -+ iov_iter_save_state(&iorw->iter, &iorw->iter_state); -+ } -+ return 0; -+} -+ -+static inline int io_rw_prep_async(struct io_kiocb *req, int rw) -+{ -+ struct io_async_rw *iorw = req->async_data; -+ struct iovec *iov = iorw->fast_iov; -+ int ret; -+ -+ ret = io_import_iovec(rw, req, &iov, &iorw->iter, false); -+ if (unlikely(ret < 0)) -+ return ret; -+ -+ iorw->bytes_done = 0; -+ iorw->free_iovec = iov; -+ if (iov) -+ req->flags |= REQ_F_NEED_CLEANUP; -+ iov_iter_save_state(&iorw->iter, &iorw->iter_state); -+ return 0; -+} -+ -+static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ if (unlikely(!(req->file->f_mode & FMODE_READ))) -+ return -EBADF; -+ return io_prep_rw(req, sqe, READ); -+} -+ -+/* -+ * This is our waitqueue callback handler, registered through lock_page_async() -+ * when we initially tried to do the IO with the iocb armed our waitqueue. -+ * This gets called when the page is unlocked, and we generally expect that to -+ * happen when the page IO is completed and the page is now uptodate. This will -+ * queue a task_work based retry of the operation, attempting to copy the data -+ * again. If the latter fails because the page was NOT uptodate, then we will -+ * do a thread based blocking retry of the operation. That's the unexpected -+ * slow path. -+ */ -+static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, -+ int sync, void *arg) -+{ -+ struct wait_page_queue *wpq; -+ struct io_kiocb *req = wait->private; -+ struct wait_page_key *key = arg; -+ -+ wpq = container_of(wait, struct wait_page_queue, wait); -+ -+ if (!wake_page_match(wpq, key)) -+ return 0; -+ -+ req->rw.kiocb.ki_flags &= ~IOCB_WAITQ; -+ list_del_init(&wait->entry); -+ io_req_task_queue(req); -+ return 1; -+} -+ -+/* -+ * This controls whether a given IO request should be armed for async page -+ * based retry. If we return false here, the request is handed to the async -+ * worker threads for retry. If we're doing buffered reads on a regular file, -+ * we prepare a private wait_page_queue entry and retry the operation. This -+ * will either succeed because the page is now uptodate and unlocked, or it -+ * will register a callback when the page is unlocked at IO completion. Through -+ * that callback, io_uring uses task_work to setup a retry of the operation. -+ * That retry will attempt the buffered read again. The retry will generally -+ * succeed, or in rare cases where it fails, we then fall back to using the -+ * async worker threads for a blocking retry. -+ */ -+static bool io_rw_should_retry(struct io_kiocb *req) -+{ -+ struct io_async_rw *rw = req->async_data; -+ struct wait_page_queue *wait = &rw->wpq; -+ struct kiocb *kiocb = &req->rw.kiocb; -+ -+ /* never retry for NOWAIT, we just complete with -EAGAIN */ -+ if (req->flags & REQ_F_NOWAIT) -+ return false; -+ -+ /* Only for buffered IO */ -+ if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI)) -+ return false; -+ -+ /* -+ * just use poll if we can, and don't attempt if the fs doesn't -+ * support callback based unlocks -+ */ -+ if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC)) -+ return false; -+ -+ wait->wait.func = io_async_buf_func; -+ wait->wait.private = req; -+ wait->wait.flags = 0; -+ INIT_LIST_HEAD(&wait->wait.entry); -+ kiocb->ki_flags |= IOCB_WAITQ; -+ kiocb->ki_flags &= ~IOCB_NOWAIT; -+ kiocb->ki_waitq = wait; -+ return true; -+} -+ -+static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter) -+{ -+ if (req->file->f_op->read_iter) -+ return call_read_iter(req->file, &req->rw.kiocb, iter); -+ else if (req->file->f_op->read) -+ return loop_rw_iter(READ, req, iter); -+ else -+ return -EINVAL; -+} -+ -+static bool need_read_all(struct io_kiocb *req) -+{ -+ return req->flags & REQ_F_ISREG || -+ S_ISBLK(file_inode(req->file)->i_mode); -+} -+ -+static int io_read(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; -+ struct kiocb *kiocb = &req->rw.kiocb; -+ struct iov_iter __iter, *iter = &__iter; -+ struct io_async_rw *rw = req->async_data; -+ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; -+ struct iov_iter_state __state, *state; -+ ssize_t ret, ret2; -+ -+ if (rw) { -+ iter = &rw->iter; -+ state = &rw->iter_state; -+ /* -+ * We come here from an earlier attempt, restore our state to -+ * match in case it doesn't. It's cheap enough that we don't -+ * need to make this conditional. -+ */ -+ iov_iter_restore(iter, state); -+ iovec = NULL; -+ } else { -+ ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock); -+ if (ret < 0) -+ return ret; -+ state = &__state; -+ iov_iter_save_state(iter, state); -+ } -+ req->result = iov_iter_count(iter); -+ -+ /* Ensure we clear previously set non-block flag */ -+ if (!force_nonblock) -+ kiocb->ki_flags &= ~IOCB_NOWAIT; -+ else -+ kiocb->ki_flags |= IOCB_NOWAIT; -+ -+ /* If the file doesn't support async, just async punt */ -+ if (force_nonblock && !io_file_supports_nowait(req, READ)) { -+ ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true); -+ return ret ?: -EAGAIN; -+ } -+ -+ ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), req->result); -+ if (unlikely(ret)) { -+ kfree(iovec); -+ return ret; -+ } -+ -+ ret = io_iter_do_read(req, iter); -+ -+ if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { -+ req->flags &= ~REQ_F_REISSUE; -+ /* IOPOLL retry should happen for io-wq threads */ -+ if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ goto done; -+ /* no retry on NONBLOCK nor RWF_NOWAIT */ -+ if (req->flags & REQ_F_NOWAIT) -+ goto done; -+ ret = 0; -+ } else if (ret == -EIOCBQUEUED) { -+ goto out_free; -+ } else if (ret <= 0 || ret == req->result || !force_nonblock || -+ (req->flags & REQ_F_NOWAIT) || !need_read_all(req)) { -+ /* read all, failed, already did sync or don't want to retry */ -+ goto done; -+ } -+ -+ /* -+ * Don't depend on the iter state matching what was consumed, or being -+ * untouched in case of error. Restore it and we'll advance it -+ * manually if we need to. -+ */ -+ iov_iter_restore(iter, state); -+ -+ ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true); -+ if (ret2) -+ return ret2; -+ -+ iovec = NULL; -+ rw = req->async_data; -+ /* -+ * Now use our persistent iterator and state, if we aren't already. -+ * We've restored and mapped the iter to match. -+ */ -+ if (iter != &rw->iter) { -+ iter = &rw->iter; -+ state = &rw->iter_state; -+ } -+ -+ do { -+ /* -+ * We end up here because of a partial read, either from -+ * above or inside this loop. Advance the iter by the bytes -+ * that were consumed. -+ */ -+ iov_iter_advance(iter, ret); -+ if (!iov_iter_count(iter)) -+ break; -+ rw->bytes_done += ret; -+ iov_iter_save_state(iter, state); -+ -+ /* if we can retry, do so with the callbacks armed */ -+ if (!io_rw_should_retry(req)) { -+ kiocb->ki_flags &= ~IOCB_WAITQ; -+ return -EAGAIN; -+ } -+ -+ req->result = iov_iter_count(iter); -+ /* -+ * Now retry read with the IOCB_WAITQ parts set in the iocb. If -+ * we get -EIOCBQUEUED, then we'll get a notification when the -+ * desired page gets unlocked. We can also get a partial read -+ * here, and if we do, then just retry at the new offset. -+ */ -+ ret = io_iter_do_read(req, iter); -+ if (ret == -EIOCBQUEUED) -+ return 0; -+ /* we got some bytes, but not all. retry. */ -+ kiocb->ki_flags &= ~IOCB_WAITQ; -+ iov_iter_restore(iter, state); -+ } while (ret > 0); -+done: -+ kiocb_done(kiocb, ret, issue_flags); -+out_free: -+ /* it's faster to check here then delegate to kfree */ -+ if (iovec) -+ kfree(iovec); -+ return 0; -+} -+ -+static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ if (unlikely(!(req->file->f_mode & FMODE_WRITE))) -+ return -EBADF; -+ return io_prep_rw(req, sqe, WRITE); -+} -+ -+static int io_write(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; -+ struct kiocb *kiocb = &req->rw.kiocb; -+ struct iov_iter __iter, *iter = &__iter; -+ struct io_async_rw *rw = req->async_data; -+ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; -+ struct iov_iter_state __state, *state; -+ ssize_t ret, ret2; -+ -+ if (rw) { -+ iter = &rw->iter; -+ state = &rw->iter_state; -+ iov_iter_restore(iter, state); -+ iovec = NULL; -+ } else { -+ ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock); -+ if (ret < 0) -+ return ret; -+ state = &__state; -+ iov_iter_save_state(iter, state); -+ } -+ req->result = iov_iter_count(iter); -+ -+ /* Ensure we clear previously set non-block flag */ -+ if (!force_nonblock) -+ kiocb->ki_flags &= ~IOCB_NOWAIT; -+ else -+ kiocb->ki_flags |= IOCB_NOWAIT; -+ -+ /* If the file doesn't support async, just async punt */ -+ if (force_nonblock && !io_file_supports_nowait(req, WRITE)) -+ goto copy_iov; -+ -+ /* file path doesn't support NOWAIT for non-direct_IO */ -+ if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) && -+ (req->flags & REQ_F_ISREG)) -+ goto copy_iov; -+ -+ ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), req->result); -+ if (unlikely(ret)) -+ goto out_free; -+ -+ /* -+ * Open-code file_start_write here to grab freeze protection, -+ * which will be released by another thread in -+ * io_complete_rw(). Fool lockdep by telling it the lock got -+ * released so that it doesn't complain about the held lock when -+ * we return to userspace. -+ */ -+ if (req->flags & REQ_F_ISREG) { -+ sb_start_write(file_inode(req->file)->i_sb); -+ __sb_writers_release(file_inode(req->file)->i_sb, -+ SB_FREEZE_WRITE); -+ } -+ kiocb->ki_flags |= IOCB_WRITE; -+ -+ if (req->file->f_op->write_iter) -+ ret2 = call_write_iter(req->file, kiocb, iter); -+ else if (req->file->f_op->write) -+ ret2 = loop_rw_iter(WRITE, req, iter); -+ else -+ ret2 = -EINVAL; -+ -+ if (req->flags & REQ_F_REISSUE) { -+ req->flags &= ~REQ_F_REISSUE; -+ ret2 = -EAGAIN; -+ } -+ -+ /* -+ * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just -+ * retry them without IOCB_NOWAIT. -+ */ -+ if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) -+ ret2 = -EAGAIN; -+ /* no retry on NONBLOCK nor RWF_NOWAIT */ -+ if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT)) -+ goto done; -+ if (!force_nonblock || ret2 != -EAGAIN) { -+ /* IOPOLL retry should happen for io-wq threads */ -+ if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN) -+ goto copy_iov; -+done: -+ kiocb_done(kiocb, ret2, issue_flags); -+ } else { -+copy_iov: -+ iov_iter_restore(iter, state); -+ ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false); -+ if (!ret) { -+ if (kiocb->ki_flags & IOCB_WRITE) -+ kiocb_end_write(req); -+ return -EAGAIN; -+ } -+ return ret; -+ } -+out_free: -+ /* it's reportedly faster than delegating the null check to kfree() */ -+ if (iovec) -+ kfree(iovec); -+ return ret; -+} -+ -+static int io_renameat_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+ struct io_rename *ren = &req->rename; -+ const char __user *oldf, *newf; -+ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) -+ return -EINVAL; -+ if (unlikely(req->flags & REQ_F_FIXED_FILE)) -+ return -EBADF; -+ -+ ren->old_dfd = READ_ONCE(sqe->fd); -+ oldf = u64_to_user_ptr(READ_ONCE(sqe->addr)); -+ newf = u64_to_user_ptr(READ_ONCE(sqe->addr2)); -+ ren->new_dfd = READ_ONCE(sqe->len); -+ ren->flags = READ_ONCE(sqe->rename_flags); -+ -+ ren->oldpath = getname(oldf); -+ if (IS_ERR(ren->oldpath)) -+ return PTR_ERR(ren->oldpath); -+ -+ ren->newpath = getname(newf); -+ if (IS_ERR(ren->newpath)) { -+ putname(ren->oldpath); -+ return PTR_ERR(ren->newpath); -+ } -+ -+ req->flags |= REQ_F_NEED_CLEANUP; -+ return 0; -+} -+ -+static int io_renameat(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_rename *ren = &req->rename; -+ int ret; -+ -+ if (issue_flags & IO_URING_F_NONBLOCK) -+ return -EAGAIN; -+ -+ ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd, -+ ren->newpath, ren->flags); -+ -+ req->flags &= ~REQ_F_NEED_CLEANUP; -+ if (ret < 0) -+ req_set_fail(req); -+ io_req_complete(req, ret); -+ return 0; -+} -+ -+static int io_unlinkat_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+ struct io_unlink *un = &req->unlink; -+ const char __user *fname; -+ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index || -+ sqe->splice_fd_in) -+ return -EINVAL; -+ if (unlikely(req->flags & REQ_F_FIXED_FILE)) -+ return -EBADF; -+ -+ un->dfd = READ_ONCE(sqe->fd); -+ -+ un->flags = READ_ONCE(sqe->unlink_flags); -+ if (un->flags & ~AT_REMOVEDIR) -+ return -EINVAL; -+ -+ fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); -+ un->filename = getname(fname); -+ if (IS_ERR(un->filename)) -+ return PTR_ERR(un->filename); -+ -+ req->flags |= REQ_F_NEED_CLEANUP; -+ return 0; -+} -+ -+static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_unlink *un = &req->unlink; -+ int ret; -+ -+ if (issue_flags & IO_URING_F_NONBLOCK) -+ return -EAGAIN; -+ -+ if (un->flags & AT_REMOVEDIR) -+ ret = do_rmdir(un->dfd, un->filename); -+ else -+ ret = do_unlinkat(un->dfd, un->filename); -+ -+ req->flags &= ~REQ_F_NEED_CLEANUP; -+ if (ret < 0) -+ req_set_fail(req); -+ io_req_complete(req, ret); -+ return 0; -+} -+ -+static int io_shutdown_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+#if defined(CONFIG_NET) -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (unlikely(sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags || -+ sqe->buf_index || sqe->splice_fd_in)) -+ return -EINVAL; -+ -+ req->shutdown.how = READ_ONCE(sqe->len); -+ return 0; -+#else -+ return -EOPNOTSUPP; -+#endif -+} -+ -+static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) -+{ -+#if defined(CONFIG_NET) -+ struct socket *sock; -+ int ret; -+ -+ if (issue_flags & IO_URING_F_NONBLOCK) -+ return -EAGAIN; -+ -+ sock = sock_from_file(req->file, &ret); -+ if (unlikely(!sock)) -+ return ret; -+ -+ ret = __sys_shutdown_sock(sock, req->shutdown.how); -+ if (ret < 0) -+ req_set_fail(req); -+ io_req_complete(req, ret); -+ return 0; -+#else -+ return -EOPNOTSUPP; -+#endif -+} -+ -+static int __io_splice_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+ struct io_splice *sp = &req->splice; -+ unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL; -+ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ -+ sp->len = READ_ONCE(sqe->len); -+ sp->flags = READ_ONCE(sqe->splice_flags); -+ if (unlikely(sp->flags & ~valid_flags)) -+ return -EINVAL; -+ sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in); -+ return 0; -+} -+ -+static int io_tee_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+ if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off)) -+ return -EINVAL; -+ return __io_splice_prep(req, sqe); -+} -+ -+static int io_tee(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_splice *sp = &req->splice; -+ struct file *out = sp->file_out; -+ unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; -+ struct file *in; -+ long ret = 0; -+ -+ if (issue_flags & IO_URING_F_NONBLOCK) -+ return -EAGAIN; -+ -+ in = io_file_get(req->ctx, req, sp->splice_fd_in, -+ (sp->flags & SPLICE_F_FD_IN_FIXED)); -+ if (!in) { -+ ret = -EBADF; -+ goto done; -+ } -+ -+ if (sp->len) -+ ret = do_tee(in, out, sp->len, flags); -+ -+ if (!(sp->flags & SPLICE_F_FD_IN_FIXED)) -+ io_put_file(in); -+done: -+ if (ret != sp->len) -+ req_set_fail(req); -+ io_req_complete(req, ret); -+ return 0; -+} -+ -+static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ struct io_splice *sp = &req->splice; -+ -+ sp->off_in = READ_ONCE(sqe->splice_off_in); -+ sp->off_out = READ_ONCE(sqe->off); -+ return __io_splice_prep(req, sqe); -+} -+ -+static int io_splice(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_splice *sp = &req->splice; -+ struct file *out = sp->file_out; -+ unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; -+ loff_t *poff_in, *poff_out; -+ struct file *in; -+ long ret = 0; -+ -+ if (issue_flags & IO_URING_F_NONBLOCK) -+ return -EAGAIN; -+ -+ in = io_file_get(req->ctx, req, sp->splice_fd_in, -+ (sp->flags & SPLICE_F_FD_IN_FIXED)); -+ if (!in) { -+ ret = -EBADF; -+ goto done; -+ } -+ -+ poff_in = (sp->off_in == -1) ? NULL : &sp->off_in; -+ poff_out = (sp->off_out == -1) ? NULL : &sp->off_out; -+ -+ if (sp->len) -+ ret = do_splice(in, poff_in, out, poff_out, sp->len, flags); -+ -+ if (!(sp->flags & SPLICE_F_FD_IN_FIXED)) -+ io_put_file(in); -+done: -+ if (ret != sp->len) -+ req_set_fail(req); -+ io_req_complete(req, ret); -+ return 0; -+} -+ -+/* -+ * IORING_OP_NOP just posts a completion event, nothing else. -+ */ -+static int io_nop(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ -+ __io_req_complete(req, issue_flags, 0, 0); -+ return 0; -+} -+ -+static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index || -+ sqe->splice_fd_in)) -+ return -EINVAL; -+ -+ req->sync.flags = READ_ONCE(sqe->fsync_flags); -+ if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC)) -+ return -EINVAL; -+ -+ req->sync.off = READ_ONCE(sqe->off); -+ req->sync.len = READ_ONCE(sqe->len); -+ return 0; -+} -+ -+static int io_fsync(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ loff_t end = req->sync.off + req->sync.len; -+ int ret; -+ -+ /* fsync always requires a blocking context */ -+ if (issue_flags & IO_URING_F_NONBLOCK) -+ return -EAGAIN; -+ -+ ret = vfs_fsync_range(req->file, req->sync.off, -+ end > 0 ? end : LLONG_MAX, -+ req->sync.flags & IORING_FSYNC_DATASYNC); -+ if (ret < 0) -+ req_set_fail(req); -+ io_req_complete(req, ret); -+ return 0; -+} -+ -+static int io_fallocate_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+ if (sqe->ioprio || sqe->buf_index || sqe->rw_flags || -+ sqe->splice_fd_in) -+ return -EINVAL; -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ -+ req->sync.off = READ_ONCE(sqe->off); -+ req->sync.len = READ_ONCE(sqe->addr); -+ req->sync.mode = READ_ONCE(sqe->len); -+ return 0; -+} -+ -+static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ int ret; -+ -+ /* fallocate always requiring blocking context */ -+ if (issue_flags & IO_URING_F_NONBLOCK) -+ return -EAGAIN; -+ ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off, -+ req->sync.len); -+ if (ret < 0) -+ req_set_fail(req); -+ else -+ fsnotify_modify(req->file); -+ io_req_complete(req, ret); -+ return 0; -+} -+ -+static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ const char __user *fname; -+ int ret; -+ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (unlikely(sqe->ioprio || sqe->buf_index)) -+ return -EINVAL; -+ if (unlikely(req->flags & REQ_F_FIXED_FILE)) -+ return -EBADF; -+ -+ /* open.how should be already initialised */ -+ if (!(req->open.how.flags & O_PATH) && force_o_largefile()) -+ req->open.how.flags |= O_LARGEFILE; -+ -+ req->open.dfd = READ_ONCE(sqe->fd); -+ fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); -+ req->open.filename = getname(fname); -+ if (IS_ERR(req->open.filename)) { -+ ret = PTR_ERR(req->open.filename); -+ req->open.filename = NULL; -+ return ret; -+ } -+ -+ req->open.file_slot = READ_ONCE(sqe->file_index); -+ if (req->open.file_slot && (req->open.how.flags & O_CLOEXEC)) -+ return -EINVAL; -+ -+ req->open.nofile = rlimit(RLIMIT_NOFILE); -+ req->flags |= REQ_F_NEED_CLEANUP; -+ return 0; -+} -+ -+static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ u64 mode = READ_ONCE(sqe->len); -+ u64 flags = READ_ONCE(sqe->open_flags); -+ -+ req->open.how = build_open_how(flags, mode); -+ return __io_openat_prep(req, sqe); -+} -+ -+static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ struct open_how __user *how; -+ size_t len; -+ int ret; -+ -+ how = u64_to_user_ptr(READ_ONCE(sqe->addr2)); -+ len = READ_ONCE(sqe->len); -+ if (len < OPEN_HOW_SIZE_VER0) -+ return -EINVAL; -+ -+ ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how, -+ len); -+ if (ret) -+ return ret; -+ -+ return __io_openat_prep(req, sqe); -+} -+ -+static int io_openat2(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct open_flags op; -+ struct file *file; -+ bool resolve_nonblock, nonblock_set; -+ bool fixed = !!req->open.file_slot; -+ int ret; -+ -+ ret = build_open_flags(&req->open.how, &op); -+ if (ret) -+ goto err; -+ nonblock_set = op.open_flag & O_NONBLOCK; -+ resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED; -+ if (issue_flags & IO_URING_F_NONBLOCK) { -+ /* -+ * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open, -+ * it'll always -EAGAIN -+ */ -+ if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE)) -+ return -EAGAIN; -+ op.lookup_flags |= LOOKUP_CACHED; -+ op.open_flag |= O_NONBLOCK; -+ } -+ -+ if (!fixed) { -+ ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile); -+ if (ret < 0) -+ goto err; -+ } -+ -+ file = do_filp_open(req->open.dfd, req->open.filename, &op); -+ if (IS_ERR(file)) { -+ /* -+ * We could hang on to this 'fd' on retrying, but seems like -+ * marginal gain for something that is now known to be a slower -+ * path. So just put it, and we'll get a new one when we retry. -+ */ -+ if (!fixed) -+ put_unused_fd(ret); -+ -+ ret = PTR_ERR(file); -+ /* only retry if RESOLVE_CACHED wasn't already set by application */ -+ if (ret == -EAGAIN && -+ (!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK))) -+ return -EAGAIN; -+ goto err; -+ } -+ -+ if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set) -+ file->f_flags &= ~O_NONBLOCK; -+ fsnotify_open(file); -+ -+ if (!fixed) -+ fd_install(ret, file); -+ else -+ ret = io_install_fixed_file(req, file, issue_flags, -+ req->open.file_slot - 1); -+err: -+ putname(req->open.filename); -+ req->flags &= ~REQ_F_NEED_CLEANUP; -+ if (ret < 0) -+ req_set_fail(req); -+ __io_req_complete(req, issue_flags, ret, 0); -+ return 0; -+} -+ -+static int io_openat(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ return io_openat2(req, issue_flags); -+} -+ -+static int io_remove_buffers_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+ struct io_provide_buf *p = &req->pbuf; -+ u64 tmp; -+ -+ if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off || -+ sqe->splice_fd_in) -+ return -EINVAL; -+ -+ tmp = READ_ONCE(sqe->fd); -+ if (!tmp || tmp > USHRT_MAX) -+ return -EINVAL; -+ -+ memset(p, 0, sizeof(*p)); -+ p->nbufs = tmp; -+ p->bgid = READ_ONCE(sqe->buf_group); -+ return 0; -+} -+ -+static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf, -+ int bgid, unsigned nbufs) -+{ -+ unsigned i = 0; -+ -+ /* shouldn't happen */ -+ if (!nbufs) -+ return 0; -+ -+ /* the head kbuf is the list itself */ -+ while (!list_empty(&buf->list)) { -+ struct io_buffer *nxt; -+ -+ nxt = list_first_entry(&buf->list, struct io_buffer, list); -+ list_del(&nxt->list); -+ kfree(nxt); -+ if (++i == nbufs) -+ return i; -+ cond_resched(); -+ } -+ i++; -+ kfree(buf); -+ xa_erase(&ctx->io_buffers, bgid); -+ -+ return i; -+} -+ -+static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_provide_buf *p = &req->pbuf; -+ struct io_ring_ctx *ctx = req->ctx; -+ struct io_buffer *head; -+ int ret = 0; -+ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; -+ -+ io_ring_submit_lock(ctx, !force_nonblock); -+ -+ lockdep_assert_held(&ctx->uring_lock); -+ -+ ret = -ENOENT; -+ head = xa_load(&ctx->io_buffers, p->bgid); -+ if (head) -+ ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs); -+ if (ret < 0) -+ req_set_fail(req); -+ -+ /* complete before unlock, IOPOLL may need the lock */ -+ __io_req_complete(req, issue_flags, ret, 0); -+ io_ring_submit_unlock(ctx, !force_nonblock); -+ return 0; -+} -+ -+static int io_provide_buffers_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+ unsigned long size, tmp_check; -+ struct io_provide_buf *p = &req->pbuf; -+ u64 tmp; -+ -+ if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in) -+ return -EINVAL; -+ -+ tmp = READ_ONCE(sqe->fd); -+ if (!tmp || tmp > USHRT_MAX) -+ return -E2BIG; -+ p->nbufs = tmp; -+ p->addr = READ_ONCE(sqe->addr); -+ p->len = READ_ONCE(sqe->len); -+ -+ if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs, -+ &size)) -+ return -EOVERFLOW; -+ if (check_add_overflow((unsigned long)p->addr, size, &tmp_check)) -+ return -EOVERFLOW; -+ -+ size = (unsigned long)p->len * p->nbufs; -+ if (!access_ok(u64_to_user_ptr(p->addr), size)) -+ return -EFAULT; -+ -+ p->bgid = READ_ONCE(sqe->buf_group); -+ tmp = READ_ONCE(sqe->off); -+ if (tmp > USHRT_MAX) -+ return -E2BIG; -+ p->bid = tmp; -+ return 0; -+} -+ -+static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head) -+{ -+ struct io_buffer *buf; -+ u64 addr = pbuf->addr; -+ int i, bid = pbuf->bid; -+ -+ for (i = 0; i < pbuf->nbufs; i++) { -+ buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT); -+ if (!buf) -+ break; -+ -+ buf->addr = addr; -+ buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT); -+ buf->bid = bid; -+ addr += pbuf->len; -+ bid++; -+ if (!*head) { -+ INIT_LIST_HEAD(&buf->list); -+ *head = buf; -+ } else { -+ list_add_tail(&buf->list, &(*head)->list); -+ } -+ cond_resched(); -+ } -+ -+ return i ? i : -ENOMEM; -+} -+ -+static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_provide_buf *p = &req->pbuf; -+ struct io_ring_ctx *ctx = req->ctx; -+ struct io_buffer *head, *list; -+ int ret = 0; -+ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; -+ -+ io_ring_submit_lock(ctx, !force_nonblock); -+ -+ lockdep_assert_held(&ctx->uring_lock); -+ -+ list = head = xa_load(&ctx->io_buffers, p->bgid); -+ -+ ret = io_add_buffers(p, &head); -+ if (ret >= 0 && !list) { -+ ret = xa_insert(&ctx->io_buffers, p->bgid, head, -+ GFP_KERNEL_ACCOUNT); -+ if (ret < 0) -+ __io_remove_buffers(ctx, head, p->bgid, -1U); -+ } -+ if (ret < 0) -+ req_set_fail(req); -+ /* complete before unlock, IOPOLL may need the lock */ -+ __io_req_complete(req, issue_flags, ret, 0); -+ io_ring_submit_unlock(ctx, !force_nonblock); -+ return 0; -+} -+ -+static int io_epoll_ctl_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+#if defined(CONFIG_EPOLL) -+ if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) -+ return -EINVAL; -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ -+ req->epoll.epfd = READ_ONCE(sqe->fd); -+ req->epoll.op = READ_ONCE(sqe->len); -+ req->epoll.fd = READ_ONCE(sqe->off); -+ -+ if (ep_op_has_event(req->epoll.op)) { -+ struct epoll_event __user *ev; -+ -+ ev = u64_to_user_ptr(READ_ONCE(sqe->addr)); -+ if (copy_from_user(&req->epoll.event, ev, sizeof(*ev))) -+ return -EFAULT; -+ } -+ -+ return 0; -+#else -+ return -EOPNOTSUPP; -+#endif -+} -+ -+static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags) -+{ -+#if defined(CONFIG_EPOLL) -+ struct io_epoll *ie = &req->epoll; -+ int ret; -+ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; -+ -+ ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock); -+ if (force_nonblock && ret == -EAGAIN) -+ return -EAGAIN; -+ -+ if (ret < 0) -+ req_set_fail(req); -+ __io_req_complete(req, issue_flags, ret, 0); -+ return 0; -+#else -+ return -EOPNOTSUPP; -+#endif -+} -+ -+static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) -+ if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->splice_fd_in) -+ return -EINVAL; -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ -+ req->madvise.addr = READ_ONCE(sqe->addr); -+ req->madvise.len = READ_ONCE(sqe->len); -+ req->madvise.advice = READ_ONCE(sqe->fadvise_advice); -+ return 0; -+#else -+ return -EOPNOTSUPP; -+#endif -+} -+ -+static int io_madvise(struct io_kiocb *req, unsigned int issue_flags) -+{ -+#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) -+ struct io_madvise *ma = &req->madvise; -+ int ret; -+ -+ if (issue_flags & IO_URING_F_NONBLOCK) -+ return -EAGAIN; -+ -+ ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice); -+ if (ret < 0) -+ req_set_fail(req); -+ io_req_complete(req, ret); -+ return 0; -+#else -+ return -EOPNOTSUPP; -+#endif -+} -+ -+static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ if (sqe->ioprio || sqe->buf_index || sqe->addr || sqe->splice_fd_in) -+ return -EINVAL; -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ -+ req->fadvise.offset = READ_ONCE(sqe->off); -+ req->fadvise.len = READ_ONCE(sqe->len); -+ req->fadvise.advice = READ_ONCE(sqe->fadvise_advice); -+ return 0; -+} -+ -+static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_fadvise *fa = &req->fadvise; -+ int ret; -+ -+ if (issue_flags & IO_URING_F_NONBLOCK) { -+ switch (fa->advice) { -+ case POSIX_FADV_NORMAL: -+ case POSIX_FADV_RANDOM: -+ case POSIX_FADV_SEQUENTIAL: -+ break; -+ default: -+ return -EAGAIN; -+ } -+ } -+ -+ ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice); -+ if (ret < 0) -+ req_set_fail(req); -+ __io_req_complete(req, issue_flags, ret, 0); -+ return 0; -+} -+ -+static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) -+ return -EINVAL; -+ if (req->flags & REQ_F_FIXED_FILE) -+ return -EBADF; -+ -+ req->statx.dfd = READ_ONCE(sqe->fd); -+ req->statx.mask = READ_ONCE(sqe->len); -+ req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr)); -+ req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2)); -+ req->statx.flags = READ_ONCE(sqe->statx_flags); -+ -+ return 0; -+} -+ -+static int io_statx(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_statx *ctx = &req->statx; -+ int ret; -+ -+ if (issue_flags & IO_URING_F_NONBLOCK) -+ return -EAGAIN; -+ -+ ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask, -+ ctx->buffer); -+ -+ if (ret < 0) -+ req_set_fail(req); -+ io_req_complete(req, ret); -+ return 0; -+} -+ -+static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (sqe->ioprio || sqe->off || sqe->addr || sqe->len || -+ sqe->rw_flags || sqe->buf_index) -+ return -EINVAL; -+ if (req->flags & REQ_F_FIXED_FILE) -+ return -EBADF; -+ -+ req->close.fd = READ_ONCE(sqe->fd); -+ req->close.file_slot = READ_ONCE(sqe->file_index); -+ if (req->close.file_slot && req->close.fd) -+ return -EINVAL; -+ -+ return 0; -+} -+ -+static int io_close(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct files_struct *files = current->files; -+ struct io_close *close = &req->close; -+ struct fdtable *fdt; -+ struct file *file = NULL; -+ int ret = -EBADF; -+ -+ if (req->close.file_slot) { -+ ret = io_close_fixed(req, issue_flags); -+ goto err; -+ } -+ -+ spin_lock(&files->file_lock); -+ fdt = files_fdtable(files); -+ if (close->fd >= fdt->max_fds) { -+ spin_unlock(&files->file_lock); -+ goto err; -+ } -+ file = fdt->fd[close->fd]; -+ if (!file || file->f_op == &io_uring_fops) { -+ spin_unlock(&files->file_lock); -+ file = NULL; -+ goto err; -+ } -+ -+ /* if the file has a flush method, be safe and punt to async */ -+ if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) { -+ spin_unlock(&files->file_lock); -+ return -EAGAIN; -+ } -+ -+ ret = __close_fd_get_file(close->fd, &file); -+ spin_unlock(&files->file_lock); -+ if (ret < 0) { -+ if (ret == -ENOENT) -+ ret = -EBADF; -+ goto err; -+ } -+ -+ /* No ->flush() or already async, safely close from here */ -+ ret = filp_close(file, current->files); -+err: -+ if (ret < 0) -+ req_set_fail(req); -+ if (file) -+ fput(file); -+ __io_req_complete(req, issue_flags, ret, 0); -+ return 0; -+} -+ -+static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index || -+ sqe->splice_fd_in)) -+ return -EINVAL; -+ -+ req->sync.off = READ_ONCE(sqe->off); -+ req->sync.len = READ_ONCE(sqe->len); -+ req->sync.flags = READ_ONCE(sqe->sync_range_flags); -+ return 0; -+} -+ -+static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ int ret; -+ -+ /* sync_file_range always requires a blocking context */ -+ if (issue_flags & IO_URING_F_NONBLOCK) -+ return -EAGAIN; -+ -+ ret = sync_file_range(req->file, req->sync.off, req->sync.len, -+ req->sync.flags); -+ if (ret < 0) -+ req_set_fail(req); -+ io_req_complete(req, ret); -+ return 0; -+} -+ -+#if defined(CONFIG_NET) -+static int io_setup_async_msg(struct io_kiocb *req, -+ struct io_async_msghdr *kmsg) -+{ -+ struct io_async_msghdr *async_msg = req->async_data; -+ -+ if (async_msg) -+ return -EAGAIN; -+ if (io_alloc_async_data(req)) { -+ kfree(kmsg->free_iov); -+ return -ENOMEM; -+ } -+ async_msg = req->async_data; -+ req->flags |= REQ_F_NEED_CLEANUP; -+ memcpy(async_msg, kmsg, sizeof(*kmsg)); -+ if (async_msg->msg.msg_name) -+ async_msg->msg.msg_name = &async_msg->addr; -+ /* if were using fast_iov, set it to the new one */ -+ if (!async_msg->free_iov) -+ async_msg->msg.msg_iter.iov = async_msg->fast_iov; -+ -+ return -EAGAIN; -+} -+ -+static int io_sendmsg_copy_hdr(struct io_kiocb *req, -+ struct io_async_msghdr *iomsg) -+{ -+ iomsg->msg.msg_name = &iomsg->addr; -+ iomsg->free_iov = iomsg->fast_iov; -+ return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg, -+ req->sr_msg.msg_flags, &iomsg->free_iov); -+} -+ -+static int io_sendmsg_prep_async(struct io_kiocb *req) -+{ -+ int ret; -+ -+ ret = io_sendmsg_copy_hdr(req, req->async_data); -+ if (!ret) -+ req->flags |= REQ_F_NEED_CLEANUP; -+ return ret; -+} -+ -+static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ struct io_sr_msg *sr = &req->sr_msg; -+ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (unlikely(sqe->addr2 || sqe->file_index)) -+ return -EINVAL; -+ if (unlikely(sqe->addr2 || sqe->file_index || sqe->ioprio)) -+ return -EINVAL; -+ -+ sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); -+ sr->len = READ_ONCE(sqe->len); -+ sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; -+ if (sr->msg_flags & MSG_DONTWAIT) -+ req->flags |= REQ_F_NOWAIT; -+ -+#ifdef CONFIG_COMPAT -+ if (req->ctx->compat) -+ sr->msg_flags |= MSG_CMSG_COMPAT; -+#endif -+ return 0; -+} -+ -+static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_async_msghdr iomsg, *kmsg; -+ struct socket *sock; -+ unsigned flags; -+ int min_ret = 0; -+ int ret; -+ -+ sock = sock_from_file(req->file, &ret); -+ if (unlikely(!sock)) -+ return ret; -+ -+ kmsg = req->async_data; -+ if (!kmsg) { -+ ret = io_sendmsg_copy_hdr(req, &iomsg); -+ if (ret) -+ return ret; -+ kmsg = &iomsg; -+ } -+ -+ flags = req->sr_msg.msg_flags; -+ if (issue_flags & IO_URING_F_NONBLOCK) -+ flags |= MSG_DONTWAIT; -+ if (flags & MSG_WAITALL) -+ min_ret = iov_iter_count(&kmsg->msg.msg_iter); -+ -+ ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); -+ if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN) -+ return io_setup_async_msg(req, kmsg); -+ if (ret == -ERESTARTSYS) -+ ret = -EINTR; -+ -+ /* fast path, check for non-NULL to avoid function call */ -+ if (kmsg->free_iov) -+ kfree(kmsg->free_iov); -+ req->flags &= ~REQ_F_NEED_CLEANUP; -+ if (ret < min_ret) -+ req_set_fail(req); -+ __io_req_complete(req, issue_flags, ret, 0); -+ return 0; -+} -+ -+static int io_send(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_sr_msg *sr = &req->sr_msg; -+ struct msghdr msg; -+ struct iovec iov; -+ struct socket *sock; -+ unsigned flags; -+ int min_ret = 0; -+ int ret; -+ -+ sock = sock_from_file(req->file, &ret); -+ if (unlikely(!sock)) -+ return ret; -+ -+ ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter); -+ if (unlikely(ret)) -+ return ret; -+ -+ msg.msg_name = NULL; -+ msg.msg_control = NULL; -+ msg.msg_controllen = 0; -+ msg.msg_namelen = 0; -+ -+ flags = req->sr_msg.msg_flags; -+ if (issue_flags & IO_URING_F_NONBLOCK) -+ flags |= MSG_DONTWAIT; -+ if (flags & MSG_WAITALL) -+ min_ret = iov_iter_count(&msg.msg_iter); -+ -+ msg.msg_flags = flags; -+ ret = sock_sendmsg(sock, &msg); -+ if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN) -+ return -EAGAIN; -+ if (ret == -ERESTARTSYS) -+ ret = -EINTR; -+ -+ if (ret < min_ret) -+ req_set_fail(req); -+ __io_req_complete(req, issue_flags, ret, 0); -+ return 0; -+} -+ -+static int __io_recvmsg_copy_hdr(struct io_kiocb *req, -+ struct io_async_msghdr *iomsg) -+{ -+ struct io_sr_msg *sr = &req->sr_msg; -+ struct iovec __user *uiov; -+ size_t iov_len; -+ int ret; -+ -+ ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg, -+ &iomsg->uaddr, &uiov, &iov_len); -+ if (ret) -+ return ret; -+ -+ if (req->flags & REQ_F_BUFFER_SELECT) { -+ if (iov_len > 1) -+ return -EINVAL; -+ if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov))) -+ return -EFAULT; -+ sr->len = iomsg->fast_iov[0].iov_len; -+ iomsg->free_iov = NULL; -+ } else { -+ iomsg->free_iov = iomsg->fast_iov; -+ ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV, -+ &iomsg->free_iov, &iomsg->msg.msg_iter, -+ false); -+ if (ret > 0) -+ ret = 0; -+ } -+ -+ return ret; -+} -+ -+#ifdef CONFIG_COMPAT -+static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, -+ struct io_async_msghdr *iomsg) -+{ -+ struct io_sr_msg *sr = &req->sr_msg; -+ struct compat_iovec __user *uiov; -+ compat_uptr_t ptr; -+ compat_size_t len; -+ int ret; -+ -+ ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr, -+ &ptr, &len); -+ if (ret) -+ return ret; -+ -+ uiov = compat_ptr(ptr); -+ if (req->flags & REQ_F_BUFFER_SELECT) { -+ compat_ssize_t clen; -+ -+ if (len > 1) -+ return -EINVAL; -+ if (!access_ok(uiov, sizeof(*uiov))) -+ return -EFAULT; -+ if (__get_user(clen, &uiov->iov_len)) -+ return -EFAULT; -+ if (clen < 0) -+ return -EINVAL; -+ sr->len = clen; -+ iomsg->free_iov = NULL; -+ } else { -+ iomsg->free_iov = iomsg->fast_iov; -+ ret = __import_iovec(READ, (struct iovec __user *)uiov, len, -+ UIO_FASTIOV, &iomsg->free_iov, -+ &iomsg->msg.msg_iter, true); -+ if (ret < 0) -+ return ret; -+ } -+ -+ return 0; -+} -+#endif -+ -+static int io_recvmsg_copy_hdr(struct io_kiocb *req, -+ struct io_async_msghdr *iomsg) -+{ -+ iomsg->msg.msg_name = &iomsg->addr; -+ -+#ifdef CONFIG_COMPAT -+ if (req->ctx->compat) -+ return __io_compat_recvmsg_copy_hdr(req, iomsg); -+#endif -+ -+ return __io_recvmsg_copy_hdr(req, iomsg); -+} -+ -+static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req, -+ bool needs_lock) -+{ -+ struct io_sr_msg *sr = &req->sr_msg; -+ struct io_buffer *kbuf; -+ -+ kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock); -+ if (IS_ERR(kbuf)) -+ return kbuf; -+ -+ sr->kbuf = kbuf; -+ req->flags |= REQ_F_BUFFER_SELECTED; -+ return kbuf; -+} -+ -+static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req) -+{ -+ return io_put_kbuf(req, req->sr_msg.kbuf); -+} -+ -+static int io_recvmsg_prep_async(struct io_kiocb *req) -+{ -+ int ret; -+ -+ ret = io_recvmsg_copy_hdr(req, req->async_data); -+ if (!ret) -+ req->flags |= REQ_F_NEED_CLEANUP; -+ return ret; -+} -+ -+static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ struct io_sr_msg *sr = &req->sr_msg; -+ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (unlikely(sqe->addr2 || sqe->file_index)) -+ return -EINVAL; -+ if (unlikely(sqe->addr2 || sqe->file_index || sqe->ioprio)) -+ return -EINVAL; -+ -+ sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); -+ sr->len = READ_ONCE(sqe->len); -+ sr->bgid = READ_ONCE(sqe->buf_group); -+ sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; -+ if (sr->msg_flags & MSG_DONTWAIT) -+ req->flags |= REQ_F_NOWAIT; -+ -+#ifdef CONFIG_COMPAT -+ if (req->ctx->compat) -+ sr->msg_flags |= MSG_CMSG_COMPAT; -+#endif -+ return 0; -+} -+ -+static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_async_msghdr iomsg, *kmsg; -+ struct socket *sock; -+ struct io_buffer *kbuf; -+ unsigned flags; -+ int min_ret = 0; -+ int ret, cflags = 0; -+ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; -+ -+ sock = sock_from_file(req->file, &ret); -+ if (unlikely(!sock)) -+ return ret; -+ -+ kmsg = req->async_data; -+ if (!kmsg) { -+ ret = io_recvmsg_copy_hdr(req, &iomsg); -+ if (ret) -+ return ret; -+ kmsg = &iomsg; -+ } -+ -+ if (req->flags & REQ_F_BUFFER_SELECT) { -+ kbuf = io_recv_buffer_select(req, !force_nonblock); -+ if (IS_ERR(kbuf)) -+ return PTR_ERR(kbuf); -+ kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr); -+ kmsg->fast_iov[0].iov_len = req->sr_msg.len; -+ iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, -+ 1, req->sr_msg.len); -+ } -+ -+ flags = req->sr_msg.msg_flags; -+ if (force_nonblock) -+ flags |= MSG_DONTWAIT; -+ if (flags & MSG_WAITALL) -+ min_ret = iov_iter_count(&kmsg->msg.msg_iter); -+ -+ ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg, -+ kmsg->uaddr, flags); -+ if (force_nonblock && ret == -EAGAIN) -+ return io_setup_async_msg(req, kmsg); -+ if (ret == -ERESTARTSYS) -+ ret = -EINTR; -+ -+ if (req->flags & REQ_F_BUFFER_SELECTED) -+ cflags = io_put_recv_kbuf(req); -+ /* fast path, check for non-NULL to avoid function call */ -+ if (kmsg->free_iov) -+ kfree(kmsg->free_iov); -+ req->flags &= ~REQ_F_NEED_CLEANUP; -+ if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)))) -+ req_set_fail(req); -+ __io_req_complete(req, issue_flags, ret, cflags); -+ return 0; -+} -+ -+static int io_recv(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_buffer *kbuf; -+ struct io_sr_msg *sr = &req->sr_msg; -+ struct msghdr msg; -+ void __user *buf = sr->buf; -+ struct socket *sock; -+ struct iovec iov; -+ unsigned flags; -+ int min_ret = 0; -+ int ret, cflags = 0; -+ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; -+ -+ sock = sock_from_file(req->file, &ret); -+ if (unlikely(!sock)) -+ return ret; -+ -+ if (req->flags & REQ_F_BUFFER_SELECT) { -+ kbuf = io_recv_buffer_select(req, !force_nonblock); -+ if (IS_ERR(kbuf)) -+ return PTR_ERR(kbuf); -+ buf = u64_to_user_ptr(kbuf->addr); -+ } -+ -+ ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter); -+ if (unlikely(ret)) -+ goto out_free; -+ -+ msg.msg_name = NULL; -+ msg.msg_control = NULL; -+ msg.msg_controllen = 0; -+ msg.msg_namelen = 0; -+ msg.msg_iocb = NULL; -+ msg.msg_flags = 0; -+ -+ flags = req->sr_msg.msg_flags; -+ if (force_nonblock) -+ flags |= MSG_DONTWAIT; -+ if (flags & MSG_WAITALL) -+ min_ret = iov_iter_count(&msg.msg_iter); -+ -+ ret = sock_recvmsg(sock, &msg, flags); -+ if (force_nonblock && ret == -EAGAIN) -+ return -EAGAIN; -+ if (ret == -ERESTARTSYS) -+ ret = -EINTR; -+out_free: -+ if (req->flags & REQ_F_BUFFER_SELECTED) -+ cflags = io_put_recv_kbuf(req); -+ if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)))) -+ req_set_fail(req); -+ __io_req_complete(req, issue_flags, ret, cflags); -+ return 0; -+} -+ -+static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ struct io_accept *accept = &req->accept; -+ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (sqe->ioprio || sqe->len || sqe->buf_index) -+ return -EINVAL; -+ -+ accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); -+ accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); -+ accept->flags = READ_ONCE(sqe->accept_flags); -+ accept->nofile = rlimit(RLIMIT_NOFILE); -+ -+ accept->file_slot = READ_ONCE(sqe->file_index); -+ if (accept->file_slot && (accept->flags & SOCK_CLOEXEC)) -+ return -EINVAL; -+ if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) -+ return -EINVAL; -+ if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK)) -+ accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK; -+ return 0; -+} -+ -+static int io_accept(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_accept *accept = &req->accept; -+ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; -+ unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; -+ bool fixed = !!accept->file_slot; -+ struct file *file; -+ int ret, fd; -+ -+ if (req->file->f_flags & O_NONBLOCK) -+ req->flags |= REQ_F_NOWAIT; -+ -+ if (!fixed) { -+ fd = __get_unused_fd_flags(accept->flags, accept->nofile); -+ if (unlikely(fd < 0)) -+ return fd; -+ } -+ file = do_accept(req->file, file_flags, accept->addr, accept->addr_len, -+ accept->flags); -+ -+ if (IS_ERR(file)) { -+ if (!fixed) -+ put_unused_fd(fd); -+ ret = PTR_ERR(file); -+ if (ret == -EAGAIN && force_nonblock) -+ return -EAGAIN; -+ if (ret == -ERESTARTSYS) -+ ret = -EINTR; -+ req_set_fail(req); -+ } else if (!fixed) { -+ fd_install(fd, file); -+ ret = fd; -+ } else { -+ ret = io_install_fixed_file(req, file, issue_flags, -+ accept->file_slot - 1); -+ } -+ __io_req_complete(req, issue_flags, ret, 0); -+ return 0; -+} -+ -+static int io_connect_prep_async(struct io_kiocb *req) -+{ -+ struct io_async_connect *io = req->async_data; -+ struct io_connect *conn = &req->connect; -+ -+ return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address); -+} -+ -+static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ struct io_connect *conn = &req->connect; -+ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags || -+ sqe->splice_fd_in) -+ return -EINVAL; -+ -+ conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); -+ conn->addr_len = READ_ONCE(sqe->addr2); -+ return 0; -+} -+ -+static int io_connect(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_async_connect __io, *io; -+ unsigned file_flags; -+ int ret; -+ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; -+ -+ if (req->async_data) { -+ io = req->async_data; -+ } else { -+ ret = move_addr_to_kernel(req->connect.addr, -+ req->connect.addr_len, -+ &__io.address); -+ if (ret) -+ goto out; -+ io = &__io; -+ } -+ -+ file_flags = force_nonblock ? O_NONBLOCK : 0; -+ -+ ret = __sys_connect_file(req->file, &io->address, -+ req->connect.addr_len, file_flags); -+ if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) { -+ if (req->async_data) -+ return -EAGAIN; -+ if (io_alloc_async_data(req)) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ memcpy(req->async_data, &__io, sizeof(__io)); -+ return -EAGAIN; -+ } -+ if (ret == -ERESTARTSYS) -+ ret = -EINTR; -+out: -+ if (ret < 0) -+ req_set_fail(req); -+ __io_req_complete(req, issue_flags, ret, 0); -+ return 0; -+} -+#else /* !CONFIG_NET */ -+#define IO_NETOP_FN(op) \ -+static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \ -+{ \ -+ return -EOPNOTSUPP; \ -+} -+ -+#define IO_NETOP_PREP(op) \ -+IO_NETOP_FN(op) \ -+static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \ -+{ \ -+ return -EOPNOTSUPP; \ -+} \ -+ -+#define IO_NETOP_PREP_ASYNC(op) \ -+IO_NETOP_PREP(op) \ -+static int io_##op##_prep_async(struct io_kiocb *req) \ -+{ \ -+ return -EOPNOTSUPP; \ -+} -+ -+IO_NETOP_PREP_ASYNC(sendmsg); -+IO_NETOP_PREP_ASYNC(recvmsg); -+IO_NETOP_PREP_ASYNC(connect); -+IO_NETOP_PREP(accept); -+IO_NETOP_FN(send); -+IO_NETOP_FN(recv); -+#endif /* CONFIG_NET */ -+ -+struct io_poll_table { -+ struct poll_table_struct pt; -+ struct io_kiocb *req; -+ int nr_entries; -+ int error; -+}; -+ -+#define IO_POLL_CANCEL_FLAG BIT(31) -+#define IO_POLL_RETRY_FLAG BIT(30) -+#define IO_POLL_REF_MASK GENMASK(29, 0) -+ -+/* -+ * We usually have 1-2 refs taken, 128 is more than enough and we want to -+ * maximise the margin between this amount and the moment when it overflows. -+ */ -+#define IO_POLL_REF_BIAS 128 -+ -+static bool io_poll_get_ownership_slowpath(struct io_kiocb *req) -+{ -+ int v; -+ -+ /* -+ * poll_refs are already elevated and we don't have much hope for -+ * grabbing the ownership. Instead of incrementing set a retry flag -+ * to notify the loop that there might have been some change. -+ */ -+ v = atomic_fetch_or(IO_POLL_RETRY_FLAG, &req->poll_refs); -+ if (v & IO_POLL_REF_MASK) -+ return false; -+ return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); -+} -+ -+/* -+ * If refs part of ->poll_refs (see IO_POLL_REF_MASK) is 0, it's free. We can -+ * bump it and acquire ownership. It's disallowed to modify requests while not -+ * owning it, that prevents from races for enqueueing task_work's and b/w -+ * arming poll and wakeups. -+ */ -+static inline bool io_poll_get_ownership(struct io_kiocb *req) -+{ -+ if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS)) -+ return io_poll_get_ownership_slowpath(req); -+ return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); -+} -+ -+static void io_poll_mark_cancelled(struct io_kiocb *req) -+{ -+ atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs); -+} -+ -+static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req) -+{ -+ /* pure poll stashes this in ->async_data, poll driven retry elsewhere */ -+ if (req->opcode == IORING_OP_POLL_ADD) -+ return req->async_data; -+ return req->apoll->double_poll; -+} -+ -+static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req) -+{ -+ if (req->opcode == IORING_OP_POLL_ADD) -+ return &req->poll; -+ return &req->apoll->poll; -+} -+ -+static void io_poll_req_insert(struct io_kiocb *req) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ struct hlist_head *list; -+ -+ list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)]; -+ hlist_add_head(&req->hash_node, list); -+} -+ -+static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events, -+ wait_queue_func_t wake_func) -+{ -+ poll->head = NULL; -+#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP) -+ /* mask in events that we always want/need */ -+ poll->events = events | IO_POLL_UNMASK; -+ INIT_LIST_HEAD(&poll->wait.entry); -+ init_waitqueue_func_entry(&poll->wait, wake_func); -+} -+ -+static inline void io_poll_remove_entry(struct io_poll_iocb *poll) -+{ -+ struct wait_queue_head *head = smp_load_acquire(&poll->head); -+ -+ if (head) { -+ spin_lock_irq(&head->lock); -+ list_del_init(&poll->wait.entry); -+ poll->head = NULL; -+ spin_unlock_irq(&head->lock); -+ } -+} -+ -+static void io_poll_remove_entries(struct io_kiocb *req) -+{ -+ struct io_poll_iocb *poll = io_poll_get_single(req); -+ struct io_poll_iocb *poll_double = io_poll_get_double(req); -+ -+ /* -+ * While we hold the waitqueue lock and the waitqueue is nonempty, -+ * wake_up_pollfree() will wait for us. However, taking the waitqueue -+ * lock in the first place can race with the waitqueue being freed. -+ * -+ * We solve this as eventpoll does: by taking advantage of the fact that -+ * all users of wake_up_pollfree() will RCU-delay the actual free. If -+ * we enter rcu_read_lock() and see that the pointer to the queue is -+ * non-NULL, we can then lock it without the memory being freed out from -+ * under us. -+ * -+ * Keep holding rcu_read_lock() as long as we hold the queue lock, in -+ * case the caller deletes the entry from the queue, leaving it empty. -+ * In that case, only RCU prevents the queue memory from being freed. -+ */ -+ rcu_read_lock(); -+ io_poll_remove_entry(poll); -+ if (poll_double) -+ io_poll_remove_entry(poll_double); -+ rcu_read_unlock(); -+} -+ -+/* -+ * All poll tw should go through this. Checks for poll events, manages -+ * references, does rewait, etc. -+ * -+ * Returns a negative error on failure. >0 when no action require, which is -+ * either spurious wakeup or multishot CQE is served. 0 when it's done with -+ * the request, then the mask is stored in req->result. -+ */ -+static int io_poll_check_events(struct io_kiocb *req) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ struct io_poll_iocb *poll = io_poll_get_single(req); -+ int v; -+ -+ /* req->task == current here, checking PF_EXITING is safe */ -+ if (unlikely(req->task->flags & PF_EXITING)) -+ io_poll_mark_cancelled(req); -+ -+ do { -+ v = atomic_read(&req->poll_refs); -+ -+ /* tw handler should be the owner, and so have some references */ -+ if (WARN_ON_ONCE(!(v & IO_POLL_REF_MASK))) -+ return 0; -+ if (v & IO_POLL_CANCEL_FLAG) -+ return -ECANCELED; -+ /* -+ * cqe.res contains only events of the first wake up -+ * and all others are be lost. Redo vfs_poll() to get -+ * up to date state. -+ */ -+ if ((v & IO_POLL_REF_MASK) != 1) -+ req->result = 0; -+ if (v & IO_POLL_RETRY_FLAG) { -+ req->result = 0; -+ /* -+ * We won't find new events that came in between -+ * vfs_poll and the ref put unless we clear the -+ * flag in advance. -+ */ -+ atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs); -+ v &= ~IO_POLL_RETRY_FLAG; -+ } -+ -+ if (!req->result) { -+ struct poll_table_struct pt = { ._key = poll->events }; -+ -+ req->result = vfs_poll(req->file, &pt) & poll->events; -+ } -+ -+ /* multishot, just fill an CQE and proceed */ -+ if (req->result && !(poll->events & EPOLLONESHOT)) { -+ __poll_t mask = mangle_poll(req->result & poll->events); -+ bool filled; -+ -+ spin_lock(&ctx->completion_lock); -+ filled = io_fill_cqe_aux(ctx, req->user_data, mask, -+ IORING_CQE_F_MORE); -+ io_commit_cqring(ctx); -+ spin_unlock(&ctx->completion_lock); -+ if (unlikely(!filled)) -+ return -ECANCELED; -+ io_cqring_ev_posted(ctx); -+ } else if (req->result) { -+ return 0; -+ } -+ -+ /* force the next iteration to vfs_poll() */ -+ req->result = 0; -+ -+ /* -+ * Release all references, retry if someone tried to restart -+ * task_work while we were executing it. -+ */ -+ } while (atomic_sub_return(v & IO_POLL_REF_MASK, &req->poll_refs) & -+ IO_POLL_REF_MASK); -+ -+ return 1; -+} -+ -+static void io_poll_task_func(struct io_kiocb *req, bool *locked) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ int ret; -+ -+ ret = io_poll_check_events(req); -+ if (ret > 0) -+ return; -+ -+ if (!ret) { -+ req->result = mangle_poll(req->result & req->poll.events); -+ } else { -+ req->result = ret; -+ req_set_fail(req); -+ } -+ -+ io_poll_remove_entries(req); -+ spin_lock(&ctx->completion_lock); -+ hash_del(&req->hash_node); -+ spin_unlock(&ctx->completion_lock); -+ io_req_complete_post(req, req->result, 0); -+} -+ -+static void io_apoll_task_func(struct io_kiocb *req, bool *locked) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ int ret; -+ -+ ret = io_poll_check_events(req); -+ if (ret > 0) -+ return; -+ -+ io_poll_remove_entries(req); -+ spin_lock(&ctx->completion_lock); -+ hash_del(&req->hash_node); -+ spin_unlock(&ctx->completion_lock); -+ -+ if (!ret) -+ io_req_task_submit(req, locked); -+ else -+ io_req_complete_failed(req, ret); -+} -+ -+static void __io_poll_execute(struct io_kiocb *req, int mask) -+{ -+ req->result = mask; -+ if (req->opcode == IORING_OP_POLL_ADD) -+ req->io_task_work.func = io_poll_task_func; -+ else -+ req->io_task_work.func = io_apoll_task_func; -+ -+ trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask); -+ io_req_task_work_add(req); -+} -+ -+static inline void io_poll_execute(struct io_kiocb *req, int res) -+{ -+ if (io_poll_get_ownership(req)) -+ __io_poll_execute(req, res); -+} -+ -+static void io_poll_cancel_req(struct io_kiocb *req) -+{ -+ io_poll_mark_cancelled(req); -+ /* kick tw, which should complete the request */ -+ io_poll_execute(req, 0); -+} -+ -+static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, -+ void *key) -+{ -+ struct io_kiocb *req = wait->private; -+ struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb, -+ wait); -+ __poll_t mask = key_to_poll(key); -+ -+ if (unlikely(mask & POLLFREE)) { -+ io_poll_mark_cancelled(req); -+ /* we have to kick tw in case it's not already */ -+ io_poll_execute(req, 0); -+ -+ /* -+ * If the waitqueue is being freed early but someone is already -+ * holds ownership over it, we have to tear down the request as -+ * best we can. That means immediately removing the request from -+ * its waitqueue and preventing all further accesses to the -+ * waitqueue via the request. -+ */ -+ list_del_init(&poll->wait.entry); -+ -+ /* -+ * Careful: this *must* be the last step, since as soon -+ * as req->head is NULL'ed out, the request can be -+ * completed and freed, since aio_poll_complete_work() -+ * will no longer need to take the waitqueue lock. -+ */ -+ smp_store_release(&poll->head, NULL); -+ return 1; -+ } -+ -+ /* for instances that support it check for an event match first */ -+ if (mask && !(mask & poll->events)) -+ return 0; -+ -+ if (io_poll_get_ownership(req)) -+ __io_poll_execute(req, mask); -+ return 1; -+} -+ -+static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, -+ struct wait_queue_head *head, -+ struct io_poll_iocb **poll_ptr) -+{ -+ struct io_kiocb *req = pt->req; -+ -+ /* -+ * The file being polled uses multiple waitqueues for poll handling -+ * (e.g. one for read, one for write). Setup a separate io_poll_iocb -+ * if this happens. -+ */ -+ if (unlikely(pt->nr_entries)) { -+ struct io_poll_iocb *first = poll; -+ -+ /* double add on the same waitqueue head, ignore */ -+ if (first->head == head) -+ return; -+ /* already have a 2nd entry, fail a third attempt */ -+ if (*poll_ptr) { -+ if ((*poll_ptr)->head == head) -+ return; -+ pt->error = -EINVAL; -+ return; -+ } -+ -+ poll = kmalloc(sizeof(*poll), GFP_ATOMIC); -+ if (!poll) { -+ pt->error = -ENOMEM; -+ return; -+ } -+ io_init_poll_iocb(poll, first->events, first->wait.func); -+ *poll_ptr = poll; -+ } -+ -+ pt->nr_entries++; -+ poll->head = head; -+ poll->wait.private = req; -+ -+ if (poll->events & EPOLLEXCLUSIVE) -+ add_wait_queue_exclusive(head, &poll->wait); -+ else -+ add_wait_queue(head, &poll->wait); -+} -+ -+static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head, -+ struct poll_table_struct *p) -+{ -+ struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); -+ -+ __io_queue_proc(&pt->req->poll, pt, head, -+ (struct io_poll_iocb **) &pt->req->async_data); -+} -+ -+static int __io_arm_poll_handler(struct io_kiocb *req, -+ struct io_poll_iocb *poll, -+ struct io_poll_table *ipt, __poll_t mask) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ INIT_HLIST_NODE(&req->hash_node); -+ io_init_poll_iocb(poll, mask, io_poll_wake); -+ poll->file = req->file; -+ poll->wait.private = req; -+ -+ ipt->pt._key = mask; -+ ipt->req = req; -+ ipt->error = 0; -+ ipt->nr_entries = 0; -+ -+ /* -+ * Take the ownership to delay any tw execution up until we're done -+ * with poll arming. see io_poll_get_ownership(). -+ */ -+ atomic_set(&req->poll_refs, 1); -+ mask = vfs_poll(req->file, &ipt->pt) & poll->events; -+ -+ if (mask && (poll->events & EPOLLONESHOT)) { -+ io_poll_remove_entries(req); -+ /* no one else has access to the req, forget about the ref */ -+ return mask; -+ } -+ if (!mask && unlikely(ipt->error || !ipt->nr_entries)) { -+ io_poll_remove_entries(req); -+ if (!ipt->error) -+ ipt->error = -EINVAL; -+ return 0; -+ } -+ -+ spin_lock(&ctx->completion_lock); -+ io_poll_req_insert(req); -+ spin_unlock(&ctx->completion_lock); -+ -+ if (mask) { -+ /* can't multishot if failed, just queue the event we've got */ -+ if (unlikely(ipt->error || !ipt->nr_entries)) { -+ poll->events |= EPOLLONESHOT; -+ ipt->error = 0; -+ } -+ __io_poll_execute(req, mask); -+ return 0; -+ } -+ -+ /* -+ * Try to release ownership. If we see a change of state, e.g. -+ * poll was waken up, queue up a tw, it'll deal with it. -+ */ -+ if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1) -+ __io_poll_execute(req, 0); -+ return 0; -+} -+ -+static void io_async_queue_proc(struct file *file, struct wait_queue_head *head, -+ struct poll_table_struct *p) -+{ -+ struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); -+ struct async_poll *apoll = pt->req->apoll; -+ -+ __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll); -+} -+ -+enum { -+ IO_APOLL_OK, -+ IO_APOLL_ABORTED, -+ IO_APOLL_READY -+}; -+ -+static int io_arm_poll_handler(struct io_kiocb *req) -+{ -+ const struct io_op_def *def = &io_op_defs[req->opcode]; -+ struct io_ring_ctx *ctx = req->ctx; -+ struct async_poll *apoll; -+ struct io_poll_table ipt; -+ __poll_t mask = EPOLLONESHOT | POLLERR | POLLPRI; -+ int ret; -+ -+ if (!req->file || !file_can_poll(req->file)) -+ return IO_APOLL_ABORTED; -+ if (req->flags & REQ_F_POLLED) -+ return IO_APOLL_ABORTED; -+ if (!def->pollin && !def->pollout) -+ return IO_APOLL_ABORTED; -+ -+ if (def->pollin) { -+ mask |= POLLIN | POLLRDNORM; -+ -+ /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */ -+ if ((req->opcode == IORING_OP_RECVMSG) && -+ (req->sr_msg.msg_flags & MSG_ERRQUEUE)) -+ mask &= ~POLLIN; -+ } else { -+ mask |= POLLOUT | POLLWRNORM; -+ } -+ -+ apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC); -+ if (unlikely(!apoll)) -+ return IO_APOLL_ABORTED; -+ apoll->double_poll = NULL; -+ req->apoll = apoll; -+ req->flags |= REQ_F_POLLED; -+ ipt.pt._qproc = io_async_queue_proc; -+ -+ ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask); -+ if (ret || ipt.error) -+ return ret ? IO_APOLL_READY : IO_APOLL_ABORTED; -+ -+ trace_io_uring_poll_arm(ctx, req, req->opcode, req->user_data, -+ mask, apoll->poll.events); -+ return IO_APOLL_OK; -+} -+ -+/* -+ * Returns true if we found and killed one or more poll requests -+ */ -+static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, -+ bool cancel_all) -+{ -+ struct hlist_node *tmp; -+ struct io_kiocb *req; -+ bool found = false; -+ int i; -+ -+ spin_lock(&ctx->completion_lock); -+ for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { -+ struct hlist_head *list; -+ -+ list = &ctx->cancel_hash[i]; -+ hlist_for_each_entry_safe(req, tmp, list, hash_node) { -+ if (io_match_task_safe(req, tsk, cancel_all)) { -+ hlist_del_init(&req->hash_node); -+ io_poll_cancel_req(req); -+ found = true; -+ } -+ } -+ } -+ spin_unlock(&ctx->completion_lock); -+ return found; -+} -+ -+static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr, -+ bool poll_only) -+ __must_hold(&ctx->completion_lock) -+{ -+ struct hlist_head *list; -+ struct io_kiocb *req; -+ -+ list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)]; -+ hlist_for_each_entry(req, list, hash_node) { -+ if (sqe_addr != req->user_data) -+ continue; -+ if (poll_only && req->opcode != IORING_OP_POLL_ADD) -+ continue; -+ return req; -+ } -+ return NULL; -+} -+ -+static bool io_poll_disarm(struct io_kiocb *req) -+ __must_hold(&ctx->completion_lock) -+{ -+ if (!io_poll_get_ownership(req)) -+ return false; -+ io_poll_remove_entries(req); -+ hash_del(&req->hash_node); -+ return true; -+} -+ -+static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr, -+ bool poll_only) -+ __must_hold(&ctx->completion_lock) -+{ -+ struct io_kiocb *req = io_poll_find(ctx, sqe_addr, poll_only); -+ -+ if (!req) -+ return -ENOENT; -+ io_poll_cancel_req(req); -+ return 0; -+} -+ -+static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe, -+ unsigned int flags) -+{ -+ u32 events; -+ -+ events = READ_ONCE(sqe->poll32_events); -+#ifdef __BIG_ENDIAN -+ events = swahw32(events); -+#endif -+ if (!(flags & IORING_POLL_ADD_MULTI)) -+ events |= EPOLLONESHOT; -+ return demangle_poll(events) | (events & (EPOLLEXCLUSIVE|EPOLLONESHOT)); -+} -+ -+static int io_poll_update_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+ struct io_poll_update *upd = &req->poll_update; -+ u32 flags; -+ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) -+ return -EINVAL; -+ flags = READ_ONCE(sqe->len); -+ if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA | -+ IORING_POLL_ADD_MULTI)) -+ return -EINVAL; -+ /* meaningless without update */ -+ if (flags == IORING_POLL_ADD_MULTI) -+ return -EINVAL; -+ -+ upd->old_user_data = READ_ONCE(sqe->addr); -+ upd->update_events = flags & IORING_POLL_UPDATE_EVENTS; -+ upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA; -+ -+ upd->new_user_data = READ_ONCE(sqe->off); -+ if (!upd->update_user_data && upd->new_user_data) -+ return -EINVAL; -+ if (upd->update_events) -+ upd->events = io_poll_parse_events(sqe, flags); -+ else if (sqe->poll32_events) -+ return -EINVAL; -+ -+ return 0; -+} -+ -+static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ struct io_poll_iocb *poll = &req->poll; -+ u32 flags; -+ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->addr) -+ return -EINVAL; -+ flags = READ_ONCE(sqe->len); -+ if (flags & ~IORING_POLL_ADD_MULTI) -+ return -EINVAL; -+ -+ io_req_set_refcount(req); -+ poll->events = io_poll_parse_events(sqe, flags); -+ return 0; -+} -+ -+static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_poll_iocb *poll = &req->poll; -+ struct io_poll_table ipt; -+ int ret; -+ -+ ipt.pt._qproc = io_poll_queue_proc; -+ -+ ret = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events); -+ if (!ret && ipt.error) -+ req_set_fail(req); -+ ret = ret ?: ipt.error; -+ if (ret) -+ __io_req_complete(req, issue_flags, ret, 0); -+ return 0; -+} -+ -+static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ struct io_kiocb *preq; -+ int ret2, ret = 0; -+ -+ spin_lock(&ctx->completion_lock); -+ preq = io_poll_find(ctx, req->poll_update.old_user_data, true); -+ if (!preq || !io_poll_disarm(preq)) { -+ spin_unlock(&ctx->completion_lock); -+ ret = preq ? -EALREADY : -ENOENT; -+ goto out; -+ } -+ spin_unlock(&ctx->completion_lock); -+ -+ if (req->poll_update.update_events || req->poll_update.update_user_data) { -+ /* only mask one event flags, keep behavior flags */ -+ if (req->poll_update.update_events) { -+ preq->poll.events &= ~0xffff; -+ preq->poll.events |= req->poll_update.events & 0xffff; -+ preq->poll.events |= IO_POLL_UNMASK; -+ } -+ if (req->poll_update.update_user_data) -+ preq->user_data = req->poll_update.new_user_data; -+ -+ ret2 = io_poll_add(preq, issue_flags); -+ /* successfully updated, don't complete poll request */ -+ if (!ret2) -+ goto out; -+ } -+ req_set_fail(preq); -+ io_req_complete(preq, -ECANCELED); -+out: -+ if (ret < 0) -+ req_set_fail(req); -+ /* complete update request, we're done with it */ -+ io_req_complete(req, ret); -+ return 0; -+} -+ -+static void io_req_task_timeout(struct io_kiocb *req, bool *locked) -+{ -+ req_set_fail(req); -+ io_req_complete_post(req, -ETIME, 0); -+} -+ -+static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) -+{ -+ struct io_timeout_data *data = container_of(timer, -+ struct io_timeout_data, timer); -+ struct io_kiocb *req = data->req; -+ struct io_ring_ctx *ctx = req->ctx; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&ctx->timeout_lock, flags); -+ list_del_init(&req->timeout.list); -+ atomic_set(&req->ctx->cq_timeouts, -+ atomic_read(&req->ctx->cq_timeouts) + 1); -+ spin_unlock_irqrestore(&ctx->timeout_lock, flags); -+ -+ req->io_task_work.func = io_req_task_timeout; -+ io_req_task_work_add(req); -+ return HRTIMER_NORESTART; -+} -+ -+static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, -+ __u64 user_data) -+ __must_hold(&ctx->timeout_lock) -+{ -+ struct io_timeout_data *io; -+ struct io_kiocb *req; -+ bool found = false; -+ -+ list_for_each_entry(req, &ctx->timeout_list, timeout.list) { -+ found = user_data == req->user_data; -+ if (found) -+ break; -+ } -+ if (!found) -+ return ERR_PTR(-ENOENT); -+ -+ io = req->async_data; -+ if (hrtimer_try_to_cancel(&io->timer) == -1) -+ return ERR_PTR(-EALREADY); -+ list_del_init(&req->timeout.list); -+ return req; -+} -+ -+static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data) -+ __must_hold(&ctx->completion_lock) -+ __must_hold(&ctx->timeout_lock) -+{ -+ struct io_kiocb *req = io_timeout_extract(ctx, user_data); -+ -+ if (IS_ERR(req)) -+ return PTR_ERR(req); -+ -+ req_set_fail(req); -+ io_fill_cqe_req(req, -ECANCELED, 0); -+ io_put_req_deferred(req); -+ return 0; -+} -+ -+static clockid_t io_timeout_get_clock(struct io_timeout_data *data) -+{ -+ switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) { -+ case IORING_TIMEOUT_BOOTTIME: -+ return CLOCK_BOOTTIME; -+ case IORING_TIMEOUT_REALTIME: -+ return CLOCK_REALTIME; -+ default: -+ /* can't happen, vetted at prep time */ -+ WARN_ON_ONCE(1); -+ fallthrough; -+ case 0: -+ return CLOCK_MONOTONIC; -+ } -+} -+ -+static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, -+ struct timespec64 *ts, enum hrtimer_mode mode) -+ __must_hold(&ctx->timeout_lock) -+{ -+ struct io_timeout_data *io; -+ struct io_kiocb *req; -+ bool found = false; -+ -+ list_for_each_entry(req, &ctx->ltimeout_list, timeout.list) { -+ found = user_data == req->user_data; -+ if (found) -+ break; -+ } -+ if (!found) -+ return -ENOENT; -+ -+ io = req->async_data; -+ if (hrtimer_try_to_cancel(&io->timer) == -1) -+ return -EALREADY; -+ hrtimer_init(&io->timer, io_timeout_get_clock(io), mode); -+ io->timer.function = io_link_timeout_fn; -+ hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode); -+ return 0; -+} -+ -+static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, -+ struct timespec64 *ts, enum hrtimer_mode mode) -+ __must_hold(&ctx->timeout_lock) -+{ -+ struct io_kiocb *req = io_timeout_extract(ctx, user_data); -+ struct io_timeout_data *data; -+ -+ if (IS_ERR(req)) -+ return PTR_ERR(req); -+ -+ req->timeout.off = 0; /* noseq */ -+ data = req->async_data; -+ list_add_tail(&req->timeout.list, &ctx->timeout_list); -+ hrtimer_init(&data->timer, io_timeout_get_clock(data), mode); -+ data->timer.function = io_timeout_fn; -+ hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode); -+ return 0; -+} -+ -+static int io_timeout_remove_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+ struct io_timeout_rem *tr = &req->timeout_rem; -+ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) -+ return -EINVAL; -+ if (sqe->ioprio || sqe->buf_index || sqe->len || sqe->splice_fd_in) -+ return -EINVAL; -+ -+ tr->ltimeout = false; -+ tr->addr = READ_ONCE(sqe->addr); -+ tr->flags = READ_ONCE(sqe->timeout_flags); -+ if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) { -+ if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1) -+ return -EINVAL; -+ if (tr->flags & IORING_LINK_TIMEOUT_UPDATE) -+ tr->ltimeout = true; -+ if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS)) -+ return -EINVAL; -+ if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2))) -+ return -EFAULT; -+ } else if (tr->flags) { -+ /* timeout removal doesn't support flags */ -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+ -+static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) -+{ -+ return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS -+ : HRTIMER_MODE_REL; -+} -+ -+/* -+ * Remove or update an existing timeout command -+ */ -+static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_timeout_rem *tr = &req->timeout_rem; -+ struct io_ring_ctx *ctx = req->ctx; -+ int ret; -+ -+ if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE)) { -+ spin_lock(&ctx->completion_lock); -+ spin_lock_irq(&ctx->timeout_lock); -+ ret = io_timeout_cancel(ctx, tr->addr); -+ spin_unlock_irq(&ctx->timeout_lock); -+ spin_unlock(&ctx->completion_lock); -+ } else { -+ enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags); -+ -+ spin_lock_irq(&ctx->timeout_lock); -+ if (tr->ltimeout) -+ ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode); -+ else -+ ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode); -+ spin_unlock_irq(&ctx->timeout_lock); -+ } -+ -+ if (ret < 0) -+ req_set_fail(req); -+ io_req_complete_post(req, ret, 0); -+ return 0; -+} -+ -+static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, -+ bool is_timeout_link) -+{ -+ struct io_timeout_data *data; -+ unsigned flags; -+ u32 off = READ_ONCE(sqe->off); -+ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (sqe->ioprio || sqe->buf_index || sqe->len != 1 || -+ sqe->splice_fd_in) -+ return -EINVAL; -+ if (off && is_timeout_link) -+ return -EINVAL; -+ flags = READ_ONCE(sqe->timeout_flags); -+ if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK)) -+ return -EINVAL; -+ /* more than one clock specified is invalid, obviously */ -+ if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1) -+ return -EINVAL; -+ -+ INIT_LIST_HEAD(&req->timeout.list); -+ req->timeout.off = off; -+ if (unlikely(off && !req->ctx->off_timeout_used)) -+ req->ctx->off_timeout_used = true; -+ -+ if (!req->async_data && io_alloc_async_data(req)) -+ return -ENOMEM; -+ -+ data = req->async_data; -+ data->req = req; -+ data->flags = flags; -+ -+ if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr))) -+ return -EFAULT; -+ -+ INIT_LIST_HEAD(&req->timeout.list); -+ data->mode = io_translate_timeout_mode(flags); -+ hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode); -+ -+ if (is_timeout_link) { -+ struct io_submit_link *link = &req->ctx->submit_state.link; -+ -+ if (!link->head) -+ return -EINVAL; -+ if (link->last->opcode == IORING_OP_LINK_TIMEOUT) -+ return -EINVAL; -+ req->timeout.head = link->last; -+ link->last->flags |= REQ_F_ARM_LTIMEOUT; -+ } -+ return 0; -+} -+ -+static int io_timeout(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ struct io_timeout_data *data = req->async_data; -+ struct list_head *entry; -+ u32 tail, off = req->timeout.off; -+ -+ spin_lock_irq(&ctx->timeout_lock); -+ -+ /* -+ * sqe->off holds how many events that need to occur for this -+ * timeout event to be satisfied. If it isn't set, then this is -+ * a pure timeout request, sequence isn't used. -+ */ -+ if (io_is_timeout_noseq(req)) { -+ entry = ctx->timeout_list.prev; -+ goto add; -+ } -+ -+ tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); -+ req->timeout.target_seq = tail + off; -+ -+ /* Update the last seq here in case io_flush_timeouts() hasn't. -+ * This is safe because ->completion_lock is held, and submissions -+ * and completions are never mixed in the same ->completion_lock section. -+ */ -+ ctx->cq_last_tm_flush = tail; -+ -+ /* -+ * Insertion sort, ensuring the first entry in the list is always -+ * the one we need first. -+ */ -+ list_for_each_prev(entry, &ctx->timeout_list) { -+ struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, -+ timeout.list); -+ -+ if (io_is_timeout_noseq(nxt)) -+ continue; -+ /* nxt.seq is behind @tail, otherwise would've been completed */ -+ if (off >= nxt->timeout.target_seq - tail) -+ break; -+ } -+add: -+ list_add(&req->timeout.list, entry); -+ data->timer.function = io_timeout_fn; -+ hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); -+ spin_unlock_irq(&ctx->timeout_lock); -+ return 0; -+} -+ -+struct io_cancel_data { -+ struct io_ring_ctx *ctx; -+ u64 user_data; -+}; -+ -+static bool io_cancel_cb(struct io_wq_work *work, void *data) -+{ -+ struct io_kiocb *req = container_of(work, struct io_kiocb, work); -+ struct io_cancel_data *cd = data; -+ -+ return req->ctx == cd->ctx && req->user_data == cd->user_data; -+} -+ -+static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data, -+ struct io_ring_ctx *ctx) -+{ -+ struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, }; -+ enum io_wq_cancel cancel_ret; -+ int ret = 0; -+ -+ if (!tctx || !tctx->io_wq) -+ return -ENOENT; -+ -+ cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false); -+ switch (cancel_ret) { -+ case IO_WQ_CANCEL_OK: -+ ret = 0; -+ break; -+ case IO_WQ_CANCEL_RUNNING: -+ ret = -EALREADY; -+ break; -+ case IO_WQ_CANCEL_NOTFOUND: -+ ret = -ENOENT; -+ break; -+ } -+ -+ return ret; -+} -+ -+static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ int ret; -+ -+ WARN_ON_ONCE(!io_wq_current_is_worker() && req->task != current); -+ -+ ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx); -+ if (ret != -ENOENT) -+ return ret; -+ -+ spin_lock(&ctx->completion_lock); -+ spin_lock_irq(&ctx->timeout_lock); -+ ret = io_timeout_cancel(ctx, sqe_addr); -+ spin_unlock_irq(&ctx->timeout_lock); -+ if (ret != -ENOENT) -+ goto out; -+ ret = io_poll_cancel(ctx, sqe_addr, false); -+out: -+ spin_unlock(&ctx->completion_lock); -+ return ret; -+} -+ -+static int io_async_cancel_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ return -EINVAL; -+ if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) -+ return -EINVAL; -+ if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags || -+ sqe->splice_fd_in) -+ return -EINVAL; -+ -+ req->cancel.addr = READ_ONCE(sqe->addr); -+ return 0; -+} -+ -+static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ u64 sqe_addr = req->cancel.addr; -+ struct io_tctx_node *node; -+ int ret; -+ -+ ret = io_try_cancel_userdata(req, sqe_addr); -+ if (ret != -ENOENT) -+ goto done; -+ -+ /* slow path, try all io-wq's */ -+ io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); -+ ret = -ENOENT; -+ list_for_each_entry(node, &ctx->tctx_list, ctx_node) { -+ struct io_uring_task *tctx = node->task->io_uring; -+ -+ ret = io_async_cancel_one(tctx, req->cancel.addr, ctx); -+ if (ret != -ENOENT) -+ break; -+ } -+ io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); -+done: -+ if (ret < 0) -+ req_set_fail(req); -+ io_req_complete_post(req, ret, 0); -+ return 0; -+} -+ -+static int io_rsrc_update_prep(struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+{ -+ if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) -+ return -EINVAL; -+ if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in) -+ return -EINVAL; -+ -+ req->rsrc_update.offset = READ_ONCE(sqe->off); -+ req->rsrc_update.nr_args = READ_ONCE(sqe->len); -+ if (!req->rsrc_update.nr_args) -+ return -EINVAL; -+ req->rsrc_update.arg = READ_ONCE(sqe->addr); -+ return 0; -+} -+ -+static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ struct io_uring_rsrc_update2 up; -+ int ret; -+ -+ up.offset = req->rsrc_update.offset; -+ up.data = req->rsrc_update.arg; -+ up.nr = 0; -+ up.tags = 0; -+ up.resv = 0; -+ up.resv2 = 0; -+ -+ io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); -+ ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE, -+ &up, req->rsrc_update.nr_args); -+ io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); -+ -+ if (ret < 0) -+ req_set_fail(req); -+ __io_req_complete(req, issue_flags, ret, 0); -+ return 0; -+} -+ -+static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -+{ -+ switch (req->opcode) { -+ case IORING_OP_NOP: -+ return 0; -+ case IORING_OP_READV: -+ case IORING_OP_READ_FIXED: -+ case IORING_OP_READ: -+ return io_read_prep(req, sqe); -+ case IORING_OP_WRITEV: -+ case IORING_OP_WRITE_FIXED: -+ case IORING_OP_WRITE: -+ return io_write_prep(req, sqe); -+ case IORING_OP_POLL_ADD: -+ return io_poll_add_prep(req, sqe); -+ case IORING_OP_POLL_REMOVE: -+ return io_poll_update_prep(req, sqe); -+ case IORING_OP_FSYNC: -+ return io_fsync_prep(req, sqe); -+ case IORING_OP_SYNC_FILE_RANGE: -+ return io_sfr_prep(req, sqe); -+ case IORING_OP_SENDMSG: -+ case IORING_OP_SEND: -+ return io_sendmsg_prep(req, sqe); -+ case IORING_OP_RECVMSG: -+ case IORING_OP_RECV: -+ return io_recvmsg_prep(req, sqe); -+ case IORING_OP_CONNECT: -+ return io_connect_prep(req, sqe); -+ case IORING_OP_TIMEOUT: -+ return io_timeout_prep(req, sqe, false); -+ case IORING_OP_TIMEOUT_REMOVE: -+ return io_timeout_remove_prep(req, sqe); -+ case IORING_OP_ASYNC_CANCEL: -+ return io_async_cancel_prep(req, sqe); -+ case IORING_OP_LINK_TIMEOUT: -+ return io_timeout_prep(req, sqe, true); -+ case IORING_OP_ACCEPT: -+ return io_accept_prep(req, sqe); -+ case IORING_OP_FALLOCATE: -+ return io_fallocate_prep(req, sqe); -+ case IORING_OP_OPENAT: -+ return io_openat_prep(req, sqe); -+ case IORING_OP_CLOSE: -+ return io_close_prep(req, sqe); -+ case IORING_OP_FILES_UPDATE: -+ return io_rsrc_update_prep(req, sqe); -+ case IORING_OP_STATX: -+ return io_statx_prep(req, sqe); -+ case IORING_OP_FADVISE: -+ return io_fadvise_prep(req, sqe); -+ case IORING_OP_MADVISE: -+ return io_madvise_prep(req, sqe); -+ case IORING_OP_OPENAT2: -+ return io_openat2_prep(req, sqe); -+ case IORING_OP_EPOLL_CTL: -+ return io_epoll_ctl_prep(req, sqe); -+ case IORING_OP_SPLICE: -+ return io_splice_prep(req, sqe); -+ case IORING_OP_PROVIDE_BUFFERS: -+ return io_provide_buffers_prep(req, sqe); -+ case IORING_OP_REMOVE_BUFFERS: -+ return io_remove_buffers_prep(req, sqe); -+ case IORING_OP_TEE: -+ return io_tee_prep(req, sqe); -+ case IORING_OP_SHUTDOWN: -+ return io_shutdown_prep(req, sqe); -+ case IORING_OP_RENAMEAT: -+ return io_renameat_prep(req, sqe); -+ case IORING_OP_UNLINKAT: -+ return io_unlinkat_prep(req, sqe); -+ } -+ -+ printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n", -+ req->opcode); -+ return -EINVAL; -+} -+ -+static int io_req_prep_async(struct io_kiocb *req) -+{ -+ if (!io_op_defs[req->opcode].needs_async_setup) -+ return 0; -+ if (WARN_ON_ONCE(req->async_data)) -+ return -EFAULT; -+ if (io_alloc_async_data(req)) -+ return -EAGAIN; -+ -+ switch (req->opcode) { -+ case IORING_OP_READV: -+ return io_rw_prep_async(req, READ); -+ case IORING_OP_WRITEV: -+ return io_rw_prep_async(req, WRITE); -+ case IORING_OP_SENDMSG: -+ return io_sendmsg_prep_async(req); -+ case IORING_OP_RECVMSG: -+ return io_recvmsg_prep_async(req); -+ case IORING_OP_CONNECT: -+ return io_connect_prep_async(req); -+ } -+ printk_once(KERN_WARNING "io_uring: prep_async() bad opcode %d\n", -+ req->opcode); -+ return -EFAULT; -+} -+ -+static u32 io_get_sequence(struct io_kiocb *req) -+{ -+ u32 seq = req->ctx->cached_sq_head; -+ -+ /* need original cached_sq_head, but it was increased for each req */ -+ io_for_each_link(req, req) -+ seq--; -+ return seq; -+} -+ -+static bool io_drain_req(struct io_kiocb *req) -+{ -+ struct io_kiocb *pos; -+ struct io_ring_ctx *ctx = req->ctx; -+ struct io_defer_entry *de; -+ int ret; -+ u32 seq; -+ -+ if (req->flags & REQ_F_FAIL) { -+ io_req_complete_fail_submit(req); -+ return true; -+ } -+ -+ /* -+ * If we need to drain a request in the middle of a link, drain the -+ * head request and the next request/link after the current link. -+ * Considering sequential execution of links, IOSQE_IO_DRAIN will be -+ * maintained for every request of our link. -+ */ -+ if (ctx->drain_next) { -+ req->flags |= REQ_F_IO_DRAIN; -+ ctx->drain_next = false; -+ } -+ /* not interested in head, start from the first linked */ -+ io_for_each_link(pos, req->link) { -+ if (pos->flags & REQ_F_IO_DRAIN) { -+ ctx->drain_next = true; -+ req->flags |= REQ_F_IO_DRAIN; -+ break; -+ } -+ } -+ -+ /* Still need defer if there is pending req in defer list. */ -+ spin_lock(&ctx->completion_lock); -+ if (likely(list_empty_careful(&ctx->defer_list) && -+ !(req->flags & REQ_F_IO_DRAIN))) { -+ spin_unlock(&ctx->completion_lock); -+ ctx->drain_active = false; -+ return false; -+ } -+ spin_unlock(&ctx->completion_lock); -+ -+ seq = io_get_sequence(req); -+ /* Still a chance to pass the sequence check */ -+ if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) -+ return false; -+ -+ ret = io_req_prep_async(req); -+ if (ret) -+ goto fail; -+ io_prep_async_link(req); -+ de = kmalloc(sizeof(*de), GFP_KERNEL); -+ if (!de) { -+ ret = -ENOMEM; -+fail: -+ io_req_complete_failed(req, ret); -+ return true; -+ } -+ -+ spin_lock(&ctx->completion_lock); -+ if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) { -+ spin_unlock(&ctx->completion_lock); -+ kfree(de); -+ io_queue_async_work(req, NULL); -+ return true; -+ } -+ -+ trace_io_uring_defer(ctx, req, req->user_data); -+ de->req = req; -+ de->seq = seq; -+ list_add_tail(&de->list, &ctx->defer_list); -+ spin_unlock(&ctx->completion_lock); -+ return true; -+} -+ -+static void io_clean_op(struct io_kiocb *req) -+{ -+ if (req->flags & REQ_F_BUFFER_SELECTED) { -+ switch (req->opcode) { -+ case IORING_OP_READV: -+ case IORING_OP_READ_FIXED: -+ case IORING_OP_READ: -+ kfree((void *)(unsigned long)req->rw.addr); -+ break; -+ case IORING_OP_RECVMSG: -+ case IORING_OP_RECV: -+ kfree(req->sr_msg.kbuf); -+ break; -+ } -+ } -+ -+ if (req->flags & REQ_F_NEED_CLEANUP) { -+ switch (req->opcode) { -+ case IORING_OP_READV: -+ case IORING_OP_READ_FIXED: -+ case IORING_OP_READ: -+ case IORING_OP_WRITEV: -+ case IORING_OP_WRITE_FIXED: -+ case IORING_OP_WRITE: { -+ struct io_async_rw *io = req->async_data; -+ -+ kfree(io->free_iovec); -+ break; -+ } -+ case IORING_OP_RECVMSG: -+ case IORING_OP_SENDMSG: { -+ struct io_async_msghdr *io = req->async_data; -+ -+ kfree(io->free_iov); -+ break; -+ } -+ case IORING_OP_OPENAT: -+ case IORING_OP_OPENAT2: -+ if (req->open.filename) -+ putname(req->open.filename); -+ break; -+ case IORING_OP_RENAMEAT: -+ putname(req->rename.oldpath); -+ putname(req->rename.newpath); -+ break; -+ case IORING_OP_UNLINKAT: -+ putname(req->unlink.filename); -+ break; -+ } -+ } -+ if ((req->flags & REQ_F_POLLED) && req->apoll) { -+ kfree(req->apoll->double_poll); -+ kfree(req->apoll); -+ req->apoll = NULL; -+ } -+ if (req->flags & REQ_F_INFLIGHT) { -+ struct io_uring_task *tctx = req->task->io_uring; -+ -+ atomic_dec(&tctx->inflight_tracked); -+ } -+ if (req->flags & REQ_F_CREDS) -+ put_cred(req->creds); -+ -+ req->flags &= ~IO_REQ_CLEAN_FLAGS; -+} -+ -+static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ const struct cred *creds = NULL; -+ int ret; -+ -+ if ((req->flags & REQ_F_CREDS) && req->creds != current_cred()) -+ creds = override_creds(req->creds); -+ -+ switch (req->opcode) { -+ case IORING_OP_NOP: -+ ret = io_nop(req, issue_flags); -+ break; -+ case IORING_OP_READV: -+ case IORING_OP_READ_FIXED: -+ case IORING_OP_READ: -+ ret = io_read(req, issue_flags); -+ break; -+ case IORING_OP_WRITEV: -+ case IORING_OP_WRITE_FIXED: -+ case IORING_OP_WRITE: -+ ret = io_write(req, issue_flags); -+ break; -+ case IORING_OP_FSYNC: -+ ret = io_fsync(req, issue_flags); -+ break; -+ case IORING_OP_POLL_ADD: -+ ret = io_poll_add(req, issue_flags); -+ break; -+ case IORING_OP_POLL_REMOVE: -+ ret = io_poll_update(req, issue_flags); -+ break; -+ case IORING_OP_SYNC_FILE_RANGE: -+ ret = io_sync_file_range(req, issue_flags); -+ break; -+ case IORING_OP_SENDMSG: -+ ret = io_sendmsg(req, issue_flags); -+ break; -+ case IORING_OP_SEND: -+ ret = io_send(req, issue_flags); -+ break; -+ case IORING_OP_RECVMSG: -+ ret = io_recvmsg(req, issue_flags); -+ break; -+ case IORING_OP_RECV: -+ ret = io_recv(req, issue_flags); -+ break; -+ case IORING_OP_TIMEOUT: -+ ret = io_timeout(req, issue_flags); -+ break; -+ case IORING_OP_TIMEOUT_REMOVE: -+ ret = io_timeout_remove(req, issue_flags); -+ break; -+ case IORING_OP_ACCEPT: -+ ret = io_accept(req, issue_flags); -+ break; -+ case IORING_OP_CONNECT: -+ ret = io_connect(req, issue_flags); -+ break; -+ case IORING_OP_ASYNC_CANCEL: -+ ret = io_async_cancel(req, issue_flags); -+ break; -+ case IORING_OP_FALLOCATE: -+ ret = io_fallocate(req, issue_flags); -+ break; -+ case IORING_OP_OPENAT: -+ ret = io_openat(req, issue_flags); -+ break; -+ case IORING_OP_CLOSE: -+ ret = io_close(req, issue_flags); -+ break; -+ case IORING_OP_FILES_UPDATE: -+ ret = io_files_update(req, issue_flags); -+ break; -+ case IORING_OP_STATX: -+ ret = io_statx(req, issue_flags); -+ break; -+ case IORING_OP_FADVISE: -+ ret = io_fadvise(req, issue_flags); -+ break; -+ case IORING_OP_MADVISE: -+ ret = io_madvise(req, issue_flags); -+ break; -+ case IORING_OP_OPENAT2: -+ ret = io_openat2(req, issue_flags); -+ break; -+ case IORING_OP_EPOLL_CTL: -+ ret = io_epoll_ctl(req, issue_flags); -+ break; -+ case IORING_OP_SPLICE: -+ ret = io_splice(req, issue_flags); -+ break; -+ case IORING_OP_PROVIDE_BUFFERS: -+ ret = io_provide_buffers(req, issue_flags); -+ break; -+ case IORING_OP_REMOVE_BUFFERS: -+ ret = io_remove_buffers(req, issue_flags); -+ break; -+ case IORING_OP_TEE: -+ ret = io_tee(req, issue_flags); -+ break; -+ case IORING_OP_SHUTDOWN: -+ ret = io_shutdown(req, issue_flags); -+ break; -+ case IORING_OP_RENAMEAT: -+ ret = io_renameat(req, issue_flags); -+ break; -+ case IORING_OP_UNLINKAT: -+ ret = io_unlinkat(req, issue_flags); -+ break; -+ default: -+ ret = -EINVAL; -+ break; -+ } -+ -+ if (creds) -+ revert_creds(creds); -+ if (ret) -+ return ret; -+ /* If the op doesn't have a file, we're not polling for it */ -+ if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) -+ io_iopoll_req_issued(req); -+ -+ return 0; -+} -+ -+static struct io_wq_work *io_wq_free_work(struct io_wq_work *work) -+{ -+ struct io_kiocb *req = container_of(work, struct io_kiocb, work); -+ -+ req = io_put_req_find_next(req); -+ return req ? &req->work : NULL; -+} -+ -+static void io_wq_submit_work(struct io_wq_work *work) -+{ -+ struct io_kiocb *req = container_of(work, struct io_kiocb, work); -+ struct io_kiocb *timeout; -+ int ret = 0; -+ -+ /* one will be dropped by ->io_free_work() after returning to io-wq */ -+ if (!(req->flags & REQ_F_REFCOUNT)) -+ __io_req_set_refcount(req, 2); -+ else -+ req_ref_get(req); -+ -+ timeout = io_prep_linked_timeout(req); -+ if (timeout) -+ io_queue_linked_timeout(timeout); -+ -+ /* either cancelled or io-wq is dying, so don't touch tctx->iowq */ -+ if (work->flags & IO_WQ_WORK_CANCEL) -+ ret = -ECANCELED; -+ -+ if (!ret) { -+ do { -+ ret = io_issue_sqe(req, 0); -+ /* -+ * We can get EAGAIN for polled IO even though we're -+ * forcing a sync submission from here, since we can't -+ * wait for request slots on the block side. -+ */ -+ if (ret != -EAGAIN || !(req->ctx->flags & IORING_SETUP_IOPOLL)) -+ break; -+ cond_resched(); -+ } while (1); -+ } -+ -+ /* avoid locking problems by failing it from a clean context */ -+ if (ret) -+ io_req_task_queue_fail(req, ret); -+} -+ -+static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table, -+ unsigned i) -+{ -+ return &table->files[i]; -+} -+ -+static inline struct file *io_file_from_index(struct io_ring_ctx *ctx, -+ int index) -+{ -+ struct io_fixed_file *slot = io_fixed_file_slot(&ctx->file_table, index); -+ -+ return (struct file *) (slot->file_ptr & FFS_MASK); -+} -+ -+static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file) -+{ -+ unsigned long file_ptr = (unsigned long) file; -+ -+ if (__io_file_supports_nowait(file, READ)) -+ file_ptr |= FFS_ASYNC_READ; -+ if (__io_file_supports_nowait(file, WRITE)) -+ file_ptr |= FFS_ASYNC_WRITE; -+ if (S_ISREG(file_inode(file)->i_mode)) -+ file_ptr |= FFS_ISREG; -+ file_slot->file_ptr = file_ptr; -+} -+ -+static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx, -+ struct io_kiocb *req, int fd) -+{ -+ struct file *file; -+ unsigned long file_ptr; -+ -+ if (unlikely((unsigned int)fd >= ctx->nr_user_files)) -+ return NULL; -+ fd = array_index_nospec(fd, ctx->nr_user_files); -+ file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr; -+ file = (struct file *) (file_ptr & FFS_MASK); -+ file_ptr &= ~FFS_MASK; -+ /* mask in overlapping REQ_F and FFS bits */ -+ req->flags |= (file_ptr << REQ_F_NOWAIT_READ_BIT); -+ io_req_set_rsrc_node(req); -+ return file; -+} -+ -+static struct file *io_file_get_normal(struct io_ring_ctx *ctx, -+ struct io_kiocb *req, int fd) -+{ -+ struct file *file = fget(fd); -+ -+ trace_io_uring_file_get(ctx, fd); -+ -+ /* we don't allow fixed io_uring files */ -+ if (file && unlikely(file->f_op == &io_uring_fops)) -+ io_req_track_inflight(req); -+ return file; -+} -+ -+static inline struct file *io_file_get(struct io_ring_ctx *ctx, -+ struct io_kiocb *req, int fd, bool fixed) -+{ -+ if (fixed) -+ return io_file_get_fixed(ctx, req, fd); -+ else -+ return io_file_get_normal(ctx, req, fd); -+} -+ -+static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked) -+{ -+ struct io_kiocb *prev = req->timeout.prev; -+ int ret = -ENOENT; -+ -+ if (prev) { -+ if (!(req->task->flags & PF_EXITING)) -+ ret = io_try_cancel_userdata(req, prev->user_data); -+ io_req_complete_post(req, ret ?: -ETIME, 0); -+ io_put_req(prev); -+ } else { -+ io_req_complete_post(req, -ETIME, 0); -+ } -+} -+ -+static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) -+{ -+ struct io_timeout_data *data = container_of(timer, -+ struct io_timeout_data, timer); -+ struct io_kiocb *prev, *req = data->req; -+ struct io_ring_ctx *ctx = req->ctx; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&ctx->timeout_lock, flags); -+ prev = req->timeout.head; -+ req->timeout.head = NULL; -+ -+ /* -+ * We don't expect the list to be empty, that will only happen if we -+ * race with the completion of the linked work. -+ */ -+ if (prev) { -+ io_remove_next_linked(prev); -+ if (!req_ref_inc_not_zero(prev)) -+ prev = NULL; -+ } -+ list_del(&req->timeout.list); -+ req->timeout.prev = prev; -+ spin_unlock_irqrestore(&ctx->timeout_lock, flags); -+ -+ req->io_task_work.func = io_req_task_link_timeout; -+ io_req_task_work_add(req); -+ return HRTIMER_NORESTART; -+} -+ -+static void io_queue_linked_timeout(struct io_kiocb *req) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ -+ spin_lock_irq(&ctx->timeout_lock); -+ /* -+ * If the back reference is NULL, then our linked request finished -+ * before we got a chance to setup the timer -+ */ -+ if (req->timeout.head) { -+ struct io_timeout_data *data = req->async_data; -+ -+ data->timer.function = io_link_timeout_fn; -+ hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), -+ data->mode); -+ list_add_tail(&req->timeout.list, &ctx->ltimeout_list); -+ } -+ spin_unlock_irq(&ctx->timeout_lock); -+ /* drop submission reference */ -+ io_put_req(req); -+} -+ -+static void __io_queue_sqe(struct io_kiocb *req) -+ __must_hold(&req->ctx->uring_lock) -+{ -+ struct io_kiocb *linked_timeout; -+ int ret; -+ -+issue_sqe: -+ ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); -+ -+ /* -+ * We async punt it if the file wasn't marked NOWAIT, or if the file -+ * doesn't support non-blocking read/write attempts -+ */ -+ if (likely(!ret)) { -+ if (req->flags & REQ_F_COMPLETE_INLINE) { -+ struct io_ring_ctx *ctx = req->ctx; -+ struct io_submit_state *state = &ctx->submit_state; -+ -+ state->compl_reqs[state->compl_nr++] = req; -+ if (state->compl_nr == ARRAY_SIZE(state->compl_reqs)) -+ io_submit_flush_completions(ctx); -+ return; -+ } -+ -+ linked_timeout = io_prep_linked_timeout(req); -+ if (linked_timeout) -+ io_queue_linked_timeout(linked_timeout); -+ } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) { -+ linked_timeout = io_prep_linked_timeout(req); -+ -+ switch (io_arm_poll_handler(req)) { -+ case IO_APOLL_READY: -+ if (linked_timeout) -+ io_queue_linked_timeout(linked_timeout); -+ goto issue_sqe; -+ case IO_APOLL_ABORTED: -+ /* -+ * Queued up for async execution, worker will release -+ * submit reference when the iocb is actually submitted. -+ */ -+ io_queue_async_work(req, NULL); -+ break; -+ } -+ -+ if (linked_timeout) -+ io_queue_linked_timeout(linked_timeout); -+ } else { -+ io_req_complete_failed(req, ret); -+ } -+} -+ -+static inline void io_queue_sqe(struct io_kiocb *req) -+ __must_hold(&req->ctx->uring_lock) -+{ -+ if (unlikely(req->ctx->drain_active) && io_drain_req(req)) -+ return; -+ -+ if (likely(!(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)))) { -+ __io_queue_sqe(req); -+ } else if (req->flags & REQ_F_FAIL) { -+ io_req_complete_fail_submit(req); -+ } else { -+ int ret = io_req_prep_async(req); -+ -+ if (unlikely(ret)) -+ io_req_complete_failed(req, ret); -+ else -+ io_queue_async_work(req, NULL); -+ } -+} -+ -+/* -+ * Check SQE restrictions (opcode and flags). -+ * -+ * Returns 'true' if SQE is allowed, 'false' otherwise. -+ */ -+static inline bool io_check_restriction(struct io_ring_ctx *ctx, -+ struct io_kiocb *req, -+ unsigned int sqe_flags) -+{ -+ if (likely(!ctx->restricted)) -+ return true; -+ -+ if (!test_bit(req->opcode, ctx->restrictions.sqe_op)) -+ return false; -+ -+ if ((sqe_flags & ctx->restrictions.sqe_flags_required) != -+ ctx->restrictions.sqe_flags_required) -+ return false; -+ -+ if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed | -+ ctx->restrictions.sqe_flags_required)) -+ return false; -+ -+ return true; -+} -+ -+static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+ __must_hold(&ctx->uring_lock) -+{ -+ struct io_submit_state *state; -+ unsigned int sqe_flags; -+ int personality, ret = 0; -+ -+ /* req is partially pre-initialised, see io_preinit_req() */ -+ req->opcode = READ_ONCE(sqe->opcode); -+ /* same numerical values with corresponding REQ_F_*, safe to copy */ -+ req->flags = sqe_flags = READ_ONCE(sqe->flags); -+ req->user_data = READ_ONCE(sqe->user_data); -+ req->file = NULL; -+ req->fixed_rsrc_refs = NULL; -+ req->task = current; -+ -+ /* enforce forwards compatibility on users */ -+ if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) -+ return -EINVAL; -+ if (unlikely(req->opcode >= IORING_OP_LAST)) -+ return -EINVAL; -+ if (!io_check_restriction(ctx, req, sqe_flags)) -+ return -EACCES; -+ -+ if ((sqe_flags & IOSQE_BUFFER_SELECT) && -+ !io_op_defs[req->opcode].buffer_select) -+ return -EOPNOTSUPP; -+ if (unlikely(sqe_flags & IOSQE_IO_DRAIN)) -+ ctx->drain_active = true; -+ -+ personality = READ_ONCE(sqe->personality); -+ if (personality) { -+ req->creds = xa_load(&ctx->personalities, personality); -+ if (!req->creds) -+ return -EINVAL; -+ get_cred(req->creds); -+ req->flags |= REQ_F_CREDS; -+ } -+ state = &ctx->submit_state; -+ -+ /* -+ * Plug now if we have more than 1 IO left after this, and the target -+ * is potentially a read/write to block based storage. -+ */ -+ if (!state->plug_started && state->ios_left > 1 && -+ io_op_defs[req->opcode].plug) { -+ blk_start_plug(&state->plug); -+ state->plug_started = true; -+ } -+ -+ if (io_op_defs[req->opcode].needs_file) { -+ req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd), -+ (sqe_flags & IOSQE_FIXED_FILE)); -+ if (unlikely(!req->file)) -+ ret = -EBADF; -+ } -+ -+ state->ios_left--; -+ return ret; -+} -+ -+static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, -+ const struct io_uring_sqe *sqe) -+ __must_hold(&ctx->uring_lock) -+{ -+ struct io_submit_link *link = &ctx->submit_state.link; -+ int ret; -+ -+ ret = io_init_req(ctx, req, sqe); -+ if (unlikely(ret)) { -+fail_req: -+ /* fail even hard links since we don't submit */ -+ if (link->head) { -+ /* -+ * we can judge a link req is failed or cancelled by if -+ * REQ_F_FAIL is set, but the head is an exception since -+ * it may be set REQ_F_FAIL because of other req's failure -+ * so let's leverage req->result to distinguish if a head -+ * is set REQ_F_FAIL because of its failure or other req's -+ * failure so that we can set the correct ret code for it. -+ * init result here to avoid affecting the normal path. -+ */ -+ if (!(link->head->flags & REQ_F_FAIL)) -+ req_fail_link_node(link->head, -ECANCELED); -+ } else if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) { -+ /* -+ * the current req is a normal req, we should return -+ * error and thus break the submittion loop. -+ */ -+ io_req_complete_failed(req, ret); -+ return ret; -+ } -+ req_fail_link_node(req, ret); -+ } else { -+ ret = io_req_prep(req, sqe); -+ if (unlikely(ret)) -+ goto fail_req; -+ } -+ -+ /* don't need @sqe from now on */ -+ trace_io_uring_submit_sqe(ctx, req, req->opcode, req->user_data, -+ req->flags, true, -+ ctx->flags & IORING_SETUP_SQPOLL); -+ -+ /* -+ * If we already have a head request, queue this one for async -+ * submittal once the head completes. If we don't have a head but -+ * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be -+ * submitted sync once the chain is complete. If none of those -+ * conditions are true (normal request), then just queue it. -+ */ -+ if (link->head) { -+ struct io_kiocb *head = link->head; -+ -+ if (!(req->flags & REQ_F_FAIL)) { -+ ret = io_req_prep_async(req); -+ if (unlikely(ret)) { -+ req_fail_link_node(req, ret); -+ if (!(head->flags & REQ_F_FAIL)) -+ req_fail_link_node(head, -ECANCELED); -+ } -+ } -+ trace_io_uring_link(ctx, req, head); -+ link->last->link = req; -+ link->last = req; -+ -+ /* last request of a link, enqueue the link */ -+ if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) { -+ link->head = NULL; -+ io_queue_sqe(head); -+ } -+ } else { -+ if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { -+ link->head = req; -+ link->last = req; -+ } else { -+ io_queue_sqe(req); -+ } -+ } -+ -+ return 0; -+} -+ -+/* -+ * Batched submission is done, ensure local IO is flushed out. -+ */ -+static void io_submit_state_end(struct io_submit_state *state, -+ struct io_ring_ctx *ctx) -+{ -+ if (state->link.head) -+ io_queue_sqe(state->link.head); -+ if (state->compl_nr) -+ io_submit_flush_completions(ctx); -+ if (state->plug_started) -+ blk_finish_plug(&state->plug); -+} -+ -+/* -+ * Start submission side cache. -+ */ -+static void io_submit_state_start(struct io_submit_state *state, -+ unsigned int max_ios) -+{ -+ state->plug_started = false; -+ state->ios_left = max_ios; -+ /* set only head, no need to init link_last in advance */ -+ state->link.head = NULL; -+} -+ -+static void io_commit_sqring(struct io_ring_ctx *ctx) -+{ -+ struct io_rings *rings = ctx->rings; -+ -+ /* -+ * Ensure any loads from the SQEs are done at this point, -+ * since once we write the new head, the application could -+ * write new data to them. -+ */ -+ smp_store_release(&rings->sq.head, ctx->cached_sq_head); -+} -+ -+/* -+ * Fetch an sqe, if one is available. Note this returns a pointer to memory -+ * that is mapped by userspace. This means that care needs to be taken to -+ * ensure that reads are stable, as we cannot rely on userspace always -+ * being a good citizen. If members of the sqe are validated and then later -+ * used, it's important that those reads are done through READ_ONCE() to -+ * prevent a re-load down the line. -+ */ -+static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) -+{ -+ unsigned head, mask = ctx->sq_entries - 1; -+ unsigned sq_idx = ctx->cached_sq_head++ & mask; -+ -+ /* -+ * The cached sq head (or cq tail) serves two purposes: -+ * -+ * 1) allows us to batch the cost of updating the user visible -+ * head updates. -+ * 2) allows the kernel side to track the head on its own, even -+ * though the application is the one updating it. -+ */ -+ head = READ_ONCE(ctx->sq_array[sq_idx]); -+ if (likely(head < ctx->sq_entries)) -+ return &ctx->sq_sqes[head]; -+ -+ /* drop invalid entries */ -+ ctx->cq_extra--; -+ WRITE_ONCE(ctx->rings->sq_dropped, -+ READ_ONCE(ctx->rings->sq_dropped) + 1); -+ return NULL; -+} -+ -+static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) -+ __must_hold(&ctx->uring_lock) -+{ -+ int submitted = 0; -+ -+ /* make sure SQ entry isn't read before tail */ -+ nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx)); -+ if (!percpu_ref_tryget_many(&ctx->refs, nr)) -+ return -EAGAIN; -+ io_get_task_refs(nr); -+ -+ io_submit_state_start(&ctx->submit_state, nr); -+ while (submitted < nr) { -+ const struct io_uring_sqe *sqe; -+ struct io_kiocb *req; -+ -+ req = io_alloc_req(ctx); -+ if (unlikely(!req)) { -+ if (!submitted) -+ submitted = -EAGAIN; -+ break; -+ } -+ sqe = io_get_sqe(ctx); -+ if (unlikely(!sqe)) { -+ list_add(&req->inflight_entry, &ctx->submit_state.free_list); -+ break; -+ } -+ /* will complete beyond this point, count as submitted */ -+ submitted++; -+ if (io_submit_sqe(ctx, req, sqe)) -+ break; -+ } -+ -+ if (unlikely(submitted != nr)) { -+ int ref_used = (submitted == -EAGAIN) ? 0 : submitted; -+ int unused = nr - ref_used; -+ -+ current->io_uring->cached_refs += unused; -+ percpu_ref_put_many(&ctx->refs, unused); -+ } -+ -+ io_submit_state_end(&ctx->submit_state, ctx); -+ /* Commit SQ ring head once we've consumed and submitted all SQEs */ -+ io_commit_sqring(ctx); -+ -+ return submitted; -+} -+ -+static inline bool io_sqd_events_pending(struct io_sq_data *sqd) -+{ -+ return READ_ONCE(sqd->state); -+} -+ -+static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx) -+{ -+ /* Tell userspace we may need a wakeup call */ -+ spin_lock(&ctx->completion_lock); -+ WRITE_ONCE(ctx->rings->sq_flags, -+ ctx->rings->sq_flags | IORING_SQ_NEED_WAKEUP); -+ spin_unlock(&ctx->completion_lock); -+} -+ -+static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx) -+{ -+ spin_lock(&ctx->completion_lock); -+ WRITE_ONCE(ctx->rings->sq_flags, -+ ctx->rings->sq_flags & ~IORING_SQ_NEED_WAKEUP); -+ spin_unlock(&ctx->completion_lock); -+} -+ -+static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) -+{ -+ unsigned int to_submit; -+ int ret = 0; -+ -+ to_submit = io_sqring_entries(ctx); -+ /* if we're handling multiple rings, cap submit size for fairness */ -+ if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE) -+ to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE; -+ -+ if (!list_empty(&ctx->iopoll_list) || to_submit) { -+ unsigned nr_events = 0; -+ const struct cred *creds = NULL; -+ -+ if (ctx->sq_creds != current_cred()) -+ creds = override_creds(ctx->sq_creds); -+ -+ mutex_lock(&ctx->uring_lock); -+ if (!list_empty(&ctx->iopoll_list)) -+ io_do_iopoll(ctx, &nr_events, 0); -+ -+ /* -+ * Don't submit if refs are dying, good for io_uring_register(), -+ * but also it is relied upon by io_ring_exit_work() -+ */ -+ if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) && -+ !(ctx->flags & IORING_SETUP_R_DISABLED)) -+ ret = io_submit_sqes(ctx, to_submit); -+ mutex_unlock(&ctx->uring_lock); -+ -+ if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait)) -+ wake_up(&ctx->sqo_sq_wait); -+ if (creds) -+ revert_creds(creds); -+ } -+ -+ return ret; -+} -+ -+static void io_sqd_update_thread_idle(struct io_sq_data *sqd) -+{ -+ struct io_ring_ctx *ctx; -+ unsigned sq_thread_idle = 0; -+ -+ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) -+ sq_thread_idle = max(sq_thread_idle, ctx->sq_thread_idle); -+ sqd->sq_thread_idle = sq_thread_idle; -+} -+ -+static bool io_sqd_handle_event(struct io_sq_data *sqd) -+{ -+ bool did_sig = false; -+ struct ksignal ksig; -+ -+ if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) || -+ signal_pending(current)) { -+ mutex_unlock(&sqd->lock); -+ if (signal_pending(current)) -+ did_sig = get_signal(&ksig); -+ cond_resched(); -+ mutex_lock(&sqd->lock); -+ } -+ return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); -+} -+ -+static int io_sq_thread(void *data) -+{ -+ struct io_sq_data *sqd = data; -+ struct io_ring_ctx *ctx; -+ unsigned long timeout = 0; -+ char buf[TASK_COMM_LEN]; -+ DEFINE_WAIT(wait); -+ -+ snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid); -+ set_task_comm(current, buf); -+ -+ if (sqd->sq_cpu != -1) -+ set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu)); -+ else -+ set_cpus_allowed_ptr(current, cpu_online_mask); -+ current->flags |= PF_NO_SETAFFINITY; -+ -+ mutex_lock(&sqd->lock); -+ while (1) { -+ bool cap_entries, sqt_spin = false; -+ -+ if (io_sqd_events_pending(sqd) || signal_pending(current)) { -+ if (io_sqd_handle_event(sqd)) -+ break; -+ timeout = jiffies + sqd->sq_thread_idle; -+ } -+ -+ cap_entries = !list_is_singular(&sqd->ctx_list); -+ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { -+ int ret = __io_sq_thread(ctx, cap_entries); -+ -+ if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list))) -+ sqt_spin = true; -+ } -+ if (io_run_task_work()) -+ sqt_spin = true; -+ -+ if (sqt_spin || !time_after(jiffies, timeout)) { -+ cond_resched(); -+ if (sqt_spin) -+ timeout = jiffies + sqd->sq_thread_idle; -+ continue; -+ } -+ -+ prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); -+ if (!io_sqd_events_pending(sqd) && !current->task_works) { -+ bool needs_sched = true; -+ -+ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { -+ io_ring_set_wakeup_flag(ctx); -+ -+ if ((ctx->flags & IORING_SETUP_IOPOLL) && -+ !list_empty_careful(&ctx->iopoll_list)) { -+ needs_sched = false; -+ break; -+ } -+ if (io_sqring_entries(ctx)) { -+ needs_sched = false; -+ break; -+ } -+ } -+ -+ if (needs_sched) { -+ mutex_unlock(&sqd->lock); -+ schedule(); -+ mutex_lock(&sqd->lock); -+ } -+ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) -+ io_ring_clear_wakeup_flag(ctx); -+ } -+ -+ finish_wait(&sqd->wait, &wait); -+ timeout = jiffies + sqd->sq_thread_idle; -+ } -+ -+ io_uring_cancel_generic(true, sqd); -+ sqd->thread = NULL; -+ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) -+ io_ring_set_wakeup_flag(ctx); -+ io_run_task_work(); -+ mutex_unlock(&sqd->lock); -+ -+ complete(&sqd->exited); -+ do_exit(0); -+} -+ -+struct io_wait_queue { -+ struct wait_queue_entry wq; -+ struct io_ring_ctx *ctx; -+ unsigned cq_tail; -+ unsigned nr_timeouts; -+}; -+ -+static inline bool io_should_wake(struct io_wait_queue *iowq) -+{ -+ struct io_ring_ctx *ctx = iowq->ctx; -+ int dist = ctx->cached_cq_tail - (int) iowq->cq_tail; -+ -+ /* -+ * Wake up if we have enough events, or if a timeout occurred since we -+ * started waiting. For timeouts, we always want to return to userspace, -+ * regardless of event count. -+ */ -+ return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; -+} -+ -+static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, -+ int wake_flags, void *key) -+{ -+ struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue, -+ wq); -+ -+ /* -+ * Cannot safely flush overflowed CQEs from here, ensure we wake up -+ * the task, and the next invocation will do it. -+ */ -+ if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->check_cq_overflow)) -+ return autoremove_wake_function(curr, mode, wake_flags, key); -+ return -1; -+} -+ -+static int io_run_task_work_sig(void) -+{ -+ if (io_run_task_work()) -+ return 1; -+ if (!signal_pending(current)) -+ return 0; -+ if (test_thread_flag(TIF_NOTIFY_SIGNAL)) -+ return -ERESTARTSYS; -+ return -EINTR; -+} -+ -+/* when returns >0, the caller should retry */ -+static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, -+ struct io_wait_queue *iowq, -+ ktime_t timeout) -+{ -+ int ret; -+ -+ /* make sure we run task_work before checking for signals */ -+ ret = io_run_task_work_sig(); -+ if (ret || io_should_wake(iowq)) -+ return ret; -+ /* let the caller flush overflows, retry */ -+ if (test_bit(0, &ctx->check_cq_overflow)) -+ return 1; -+ -+ if (!schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS)) -+ return -ETIME; -+ return 1; -+} -+ -+/* -+ * Wait until events become available, if we don't already have some. The -+ * application must reap them itself, as they reside on the shared cq ring. -+ */ -+static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, -+ const sigset_t __user *sig, size_t sigsz, -+ struct __kernel_timespec __user *uts) -+{ -+ struct io_wait_queue iowq; -+ struct io_rings *rings = ctx->rings; -+ ktime_t timeout = KTIME_MAX; -+ int ret; -+ -+ do { -+ io_cqring_overflow_flush(ctx); -+ if (io_cqring_events(ctx) >= min_events) -+ return 0; -+ if (!io_run_task_work()) -+ break; -+ } while (1); -+ -+ if (uts) { -+ struct timespec64 ts; -+ -+ if (get_timespec64(&ts, uts)) -+ return -EFAULT; -+ timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns()); -+ } -+ -+ if (sig) { -+#ifdef CONFIG_COMPAT -+ if (in_compat_syscall()) -+ ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig, -+ sigsz); -+ else -+#endif -+ ret = set_user_sigmask(sig, sigsz); -+ -+ if (ret) -+ return ret; -+ } -+ -+ init_waitqueue_func_entry(&iowq.wq, io_wake_function); -+ iowq.wq.private = current; -+ INIT_LIST_HEAD(&iowq.wq.entry); -+ iowq.ctx = ctx; -+ iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts); -+ iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events; -+ -+ trace_io_uring_cqring_wait(ctx, min_events); -+ do { -+ /* if we can't even flush overflow, don't wait for more */ -+ if (!io_cqring_overflow_flush(ctx)) { -+ ret = -EBUSY; -+ break; -+ } -+ prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, -+ TASK_INTERRUPTIBLE); -+ ret = io_cqring_wait_schedule(ctx, &iowq, timeout); -+ finish_wait(&ctx->cq_wait, &iowq.wq); -+ cond_resched(); -+ } while (ret > 0); -+ -+ restore_saved_sigmask_unless(ret == -EINTR); -+ -+ return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0; -+} -+ -+static void io_free_page_table(void **table, size_t size) -+{ -+ unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE); -+ -+ for (i = 0; i < nr_tables; i++) -+ kfree(table[i]); -+ kfree(table); -+} -+ -+static void **io_alloc_page_table(size_t size) -+{ -+ unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE); -+ size_t init_size = size; -+ void **table; -+ -+ table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL_ACCOUNT); -+ if (!table) -+ return NULL; -+ -+ for (i = 0; i < nr_tables; i++) { -+ unsigned int this_size = min_t(size_t, size, PAGE_SIZE); -+ -+ table[i] = kzalloc(this_size, GFP_KERNEL_ACCOUNT); -+ if (!table[i]) { -+ io_free_page_table(table, init_size); -+ return NULL; -+ } -+ size -= this_size; -+ } -+ return table; -+} -+ -+static void io_rsrc_node_destroy(struct io_rsrc_node *ref_node) -+{ -+ percpu_ref_exit(&ref_node->refs); -+ kfree(ref_node); -+} -+ -+static void io_rsrc_node_ref_zero(struct percpu_ref *ref) -+{ -+ struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs); -+ struct io_ring_ctx *ctx = node->rsrc_data->ctx; -+ unsigned long flags; -+ bool first_add = false; -+ unsigned long delay = HZ; -+ -+ spin_lock_irqsave(&ctx->rsrc_ref_lock, flags); -+ node->done = true; -+ -+ /* if we are mid-quiesce then do not delay */ -+ if (node->rsrc_data->quiesce) -+ delay = 0; -+ -+ while (!list_empty(&ctx->rsrc_ref_list)) { -+ node = list_first_entry(&ctx->rsrc_ref_list, -+ struct io_rsrc_node, node); -+ /* recycle ref nodes in order */ -+ if (!node->done) -+ break; -+ list_del(&node->node); -+ first_add |= llist_add(&node->llist, &ctx->rsrc_put_llist); -+ } -+ spin_unlock_irqrestore(&ctx->rsrc_ref_lock, flags); -+ -+ if (first_add) -+ mod_delayed_work(system_wq, &ctx->rsrc_put_work, delay); -+} -+ -+static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx) -+{ -+ struct io_rsrc_node *ref_node; -+ -+ ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL); -+ if (!ref_node) -+ return NULL; -+ -+ if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero, -+ 0, GFP_KERNEL)) { -+ kfree(ref_node); -+ return NULL; -+ } -+ INIT_LIST_HEAD(&ref_node->node); -+ INIT_LIST_HEAD(&ref_node->rsrc_list); -+ ref_node->done = false; -+ return ref_node; -+} -+ -+static void io_rsrc_node_switch(struct io_ring_ctx *ctx, -+ struct io_rsrc_data *data_to_kill) -+{ -+ WARN_ON_ONCE(!ctx->rsrc_backup_node); -+ WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node); -+ -+ if (data_to_kill) { -+ struct io_rsrc_node *rsrc_node = ctx->rsrc_node; -+ -+ rsrc_node->rsrc_data = data_to_kill; -+ spin_lock_irq(&ctx->rsrc_ref_lock); -+ list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list); -+ spin_unlock_irq(&ctx->rsrc_ref_lock); -+ -+ atomic_inc(&data_to_kill->refs); -+ percpu_ref_kill(&rsrc_node->refs); -+ ctx->rsrc_node = NULL; -+ } -+ -+ if (!ctx->rsrc_node) { -+ ctx->rsrc_node = ctx->rsrc_backup_node; -+ ctx->rsrc_backup_node = NULL; -+ } -+} -+ -+static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx) -+{ -+ if (ctx->rsrc_backup_node) -+ return 0; -+ ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx); -+ return ctx->rsrc_backup_node ? 0 : -ENOMEM; -+} -+ -+static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx) -+{ -+ int ret; -+ -+ /* As we may drop ->uring_lock, other task may have started quiesce */ -+ if (data->quiesce) -+ return -ENXIO; -+ -+ data->quiesce = true; -+ do { -+ ret = io_rsrc_node_switch_start(ctx); -+ if (ret) -+ break; -+ io_rsrc_node_switch(ctx, data); -+ -+ /* kill initial ref, already quiesced if zero */ -+ if (atomic_dec_and_test(&data->refs)) -+ break; -+ mutex_unlock(&ctx->uring_lock); -+ flush_delayed_work(&ctx->rsrc_put_work); -+ ret = wait_for_completion_interruptible(&data->done); -+ if (!ret) { -+ mutex_lock(&ctx->uring_lock); -+ if (atomic_read(&data->refs) > 0) { -+ /* -+ * it has been revived by another thread while -+ * we were unlocked -+ */ -+ mutex_unlock(&ctx->uring_lock); -+ } else { -+ break; -+ } -+ } -+ -+ atomic_inc(&data->refs); -+ /* wait for all works potentially completing data->done */ -+ flush_delayed_work(&ctx->rsrc_put_work); -+ reinit_completion(&data->done); -+ -+ ret = io_run_task_work_sig(); -+ mutex_lock(&ctx->uring_lock); -+ } while (ret >= 0); -+ data->quiesce = false; -+ -+ return ret; -+} -+ -+static u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx) -+{ -+ unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK; -+ unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT; -+ -+ return &data->tags[table_idx][off]; -+} -+ -+static void io_rsrc_data_free(struct io_rsrc_data *data) -+{ -+ size_t size = data->nr * sizeof(data->tags[0][0]); -+ -+ if (data->tags) -+ io_free_page_table((void **)data->tags, size); -+ kfree(data); -+} -+ -+static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put, -+ u64 __user *utags, unsigned nr, -+ struct io_rsrc_data **pdata) -+{ -+ struct io_rsrc_data *data; -+ int ret = -ENOMEM; -+ unsigned i; -+ -+ data = kzalloc(sizeof(*data), GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ data->tags = (u64 **)io_alloc_page_table(nr * sizeof(data->tags[0][0])); -+ if (!data->tags) { -+ kfree(data); -+ return -ENOMEM; -+ } -+ -+ data->nr = nr; -+ data->ctx = ctx; -+ data->do_put = do_put; -+ if (utags) { -+ ret = -EFAULT; -+ for (i = 0; i < nr; i++) { -+ u64 *tag_slot = io_get_tag_slot(data, i); -+ -+ if (copy_from_user(tag_slot, &utags[i], -+ sizeof(*tag_slot))) -+ goto fail; -+ } -+ } -+ -+ atomic_set(&data->refs, 1); -+ init_completion(&data->done); -+ *pdata = data; -+ return 0; -+fail: -+ io_rsrc_data_free(data); -+ return ret; -+} -+ -+static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) -+{ -+ table->files = kvcalloc(nr_files, sizeof(table->files[0]), -+ GFP_KERNEL_ACCOUNT); -+ return !!table->files; -+} -+ -+static void io_free_file_tables(struct io_file_table *table) -+{ -+ kvfree(table->files); -+ table->files = NULL; -+} -+ -+static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) -+{ -+#if defined(CONFIG_UNIX) -+ if (ctx->ring_sock) { -+ struct sock *sock = ctx->ring_sock->sk; -+ struct sk_buff *skb; -+ -+ while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL) -+ kfree_skb(skb); -+ } -+#else -+ int i; -+ -+ for (i = 0; i < ctx->nr_user_files; i++) { -+ struct file *file; -+ -+ file = io_file_from_index(ctx, i); -+ if (file) -+ fput(file); -+ } -+#endif -+ io_free_file_tables(&ctx->file_table); -+ io_rsrc_data_free(ctx->file_data); -+ ctx->file_data = NULL; -+ ctx->nr_user_files = 0; -+} -+ -+static int io_sqe_files_unregister(struct io_ring_ctx *ctx) -+{ -+ unsigned nr = ctx->nr_user_files; -+ int ret; -+ -+ if (!ctx->file_data) -+ return -ENXIO; -+ -+ /* -+ * Quiesce may unlock ->uring_lock, and while it's not held -+ * prevent new requests using the table. -+ */ -+ ctx->nr_user_files = 0; -+ ret = io_rsrc_ref_quiesce(ctx->file_data, ctx); -+ ctx->nr_user_files = nr; -+ if (!ret) -+ __io_sqe_files_unregister(ctx); -+ return ret; -+} -+ -+static void io_sq_thread_unpark(struct io_sq_data *sqd) -+ __releases(&sqd->lock) -+{ -+ WARN_ON_ONCE(sqd->thread == current); -+ -+ /* -+ * Do the dance but not conditional clear_bit() because it'd race with -+ * other threads incrementing park_pending and setting the bit. -+ */ -+ clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); -+ if (atomic_dec_return(&sqd->park_pending)) -+ set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); -+ mutex_unlock(&sqd->lock); -+} -+ -+static void io_sq_thread_park(struct io_sq_data *sqd) -+ __acquires(&sqd->lock) -+{ -+ WARN_ON_ONCE(sqd->thread == current); -+ -+ atomic_inc(&sqd->park_pending); -+ set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); -+ mutex_lock(&sqd->lock); -+ if (sqd->thread) -+ wake_up_process(sqd->thread); -+} -+ -+static void io_sq_thread_stop(struct io_sq_data *sqd) -+{ -+ WARN_ON_ONCE(sqd->thread == current); -+ WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)); -+ -+ set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); -+ mutex_lock(&sqd->lock); -+ if (sqd->thread) -+ wake_up_process(sqd->thread); -+ mutex_unlock(&sqd->lock); -+ wait_for_completion(&sqd->exited); -+} -+ -+static void io_put_sq_data(struct io_sq_data *sqd) -+{ -+ if (refcount_dec_and_test(&sqd->refs)) { -+ WARN_ON_ONCE(atomic_read(&sqd->park_pending)); -+ -+ io_sq_thread_stop(sqd); -+ kfree(sqd); -+ } -+} -+ -+static void io_sq_thread_finish(struct io_ring_ctx *ctx) -+{ -+ struct io_sq_data *sqd = ctx->sq_data; -+ -+ if (sqd) { -+ io_sq_thread_park(sqd); -+ list_del_init(&ctx->sqd_list); -+ io_sqd_update_thread_idle(sqd); -+ io_sq_thread_unpark(sqd); -+ -+ io_put_sq_data(sqd); -+ ctx->sq_data = NULL; -+ } -+} -+ -+static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p) -+{ -+ struct io_ring_ctx *ctx_attach; -+ struct io_sq_data *sqd; -+ struct fd f; -+ -+ f = fdget(p->wq_fd); -+ if (!f.file) -+ return ERR_PTR(-ENXIO); -+ if (f.file->f_op != &io_uring_fops) { -+ fdput(f); -+ return ERR_PTR(-EINVAL); -+ } -+ -+ ctx_attach = f.file->private_data; -+ sqd = ctx_attach->sq_data; -+ if (!sqd) { -+ fdput(f); -+ return ERR_PTR(-EINVAL); -+ } -+ if (sqd->task_tgid != current->tgid) { -+ fdput(f); -+ return ERR_PTR(-EPERM); -+ } -+ -+ refcount_inc(&sqd->refs); -+ fdput(f); -+ return sqd; -+} -+ -+static struct io_sq_data *io_get_sq_data(struct io_uring_params *p, -+ bool *attached) -+{ -+ struct io_sq_data *sqd; -+ -+ *attached = false; -+ if (p->flags & IORING_SETUP_ATTACH_WQ) { -+ sqd = io_attach_sq_data(p); -+ if (!IS_ERR(sqd)) { -+ *attached = true; -+ return sqd; -+ } -+ /* fall through for EPERM case, setup new sqd/task */ -+ if (PTR_ERR(sqd) != -EPERM) -+ return sqd; -+ } -+ -+ sqd = kzalloc(sizeof(*sqd), GFP_KERNEL); -+ if (!sqd) -+ return ERR_PTR(-ENOMEM); -+ -+ atomic_set(&sqd->park_pending, 0); -+ refcount_set(&sqd->refs, 1); -+ INIT_LIST_HEAD(&sqd->ctx_list); -+ mutex_init(&sqd->lock); -+ init_waitqueue_head(&sqd->wait); -+ init_completion(&sqd->exited); -+ return sqd; -+} -+ -+#if defined(CONFIG_UNIX) -+/* -+ * Ensure the UNIX gc is aware of our file set, so we are certain that -+ * the io_uring can be safely unregistered on process exit, even if we have -+ * loops in the file referencing. -+ */ -+static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset) -+{ -+ struct sock *sk = ctx->ring_sock->sk; -+ struct scm_fp_list *fpl; -+ struct sk_buff *skb; -+ int i, nr_files; -+ -+ fpl = kzalloc(sizeof(*fpl), GFP_KERNEL); -+ if (!fpl) -+ return -ENOMEM; -+ -+ skb = alloc_skb(0, GFP_KERNEL); -+ if (!skb) { -+ kfree(fpl); -+ return -ENOMEM; -+ } -+ -+ skb->sk = sk; -+ skb->scm_io_uring = 1; -+ -+ nr_files = 0; -+ fpl->user = get_uid(current_user()); -+ for (i = 0; i < nr; i++) { -+ struct file *file = io_file_from_index(ctx, i + offset); -+ -+ if (!file) -+ continue; -+ fpl->fp[nr_files] = get_file(file); -+ unix_inflight(fpl->user, fpl->fp[nr_files]); -+ nr_files++; -+ } -+ -+ if (nr_files) { -+ fpl->max = SCM_MAX_FD; -+ fpl->count = nr_files; -+ UNIXCB(skb).fp = fpl; -+ skb->destructor = unix_destruct_scm; -+ refcount_add(skb->truesize, &sk->sk_wmem_alloc); -+ skb_queue_head(&sk->sk_receive_queue, skb); -+ -+ for (i = 0; i < nr; i++) { -+ struct file *file = io_file_from_index(ctx, i + offset); -+ -+ if (file) -+ fput(file); -+ } -+ } else { -+ kfree_skb(skb); -+ free_uid(fpl->user); -+ kfree(fpl); -+ } -+ -+ return 0; -+} -+ -+/* -+ * If UNIX sockets are enabled, fd passing can cause a reference cycle which -+ * causes regular reference counting to break down. We rely on the UNIX -+ * garbage collection to take care of this problem for us. -+ */ -+static int io_sqe_files_scm(struct io_ring_ctx *ctx) -+{ -+ unsigned left, total; -+ int ret = 0; -+ -+ total = 0; -+ left = ctx->nr_user_files; -+ while (left) { -+ unsigned this_files = min_t(unsigned, left, SCM_MAX_FD); -+ -+ ret = __io_sqe_files_scm(ctx, this_files, total); -+ if (ret) -+ break; -+ left -= this_files; -+ total += this_files; -+ } -+ -+ if (!ret) -+ return 0; -+ -+ while (total < ctx->nr_user_files) { -+ struct file *file = io_file_from_index(ctx, total); -+ -+ if (file) -+ fput(file); -+ total++; -+ } -+ -+ return ret; -+} -+#else -+static int io_sqe_files_scm(struct io_ring_ctx *ctx) -+{ -+ return 0; -+} -+#endif -+ -+static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc) -+{ -+ struct file *file = prsrc->file; -+#if defined(CONFIG_UNIX) -+ struct sock *sock = ctx->ring_sock->sk; -+ struct sk_buff_head list, *head = &sock->sk_receive_queue; -+ struct sk_buff *skb; -+ int i; -+ -+ __skb_queue_head_init(&list); -+ -+ /* -+ * Find the skb that holds this file in its SCM_RIGHTS. When found, -+ * remove this entry and rearrange the file array. -+ */ -+ skb = skb_dequeue(head); -+ while (skb) { -+ struct scm_fp_list *fp; -+ -+ fp = UNIXCB(skb).fp; -+ for (i = 0; i < fp->count; i++) { -+ int left; -+ -+ if (fp->fp[i] != file) -+ continue; -+ -+ unix_notinflight(fp->user, fp->fp[i]); -+ left = fp->count - 1 - i; -+ if (left) { -+ memmove(&fp->fp[i], &fp->fp[i + 1], -+ left * sizeof(struct file *)); -+ } -+ fp->count--; -+ if (!fp->count) { -+ kfree_skb(skb); -+ skb = NULL; -+ } else { -+ __skb_queue_tail(&list, skb); -+ } -+ fput(file); -+ file = NULL; -+ break; -+ } -+ -+ if (!file) -+ break; -+ -+ __skb_queue_tail(&list, skb); -+ -+ skb = skb_dequeue(head); -+ } -+ -+ if (skb_peek(&list)) { -+ spin_lock_irq(&head->lock); -+ while ((skb = __skb_dequeue(&list)) != NULL) -+ __skb_queue_tail(head, skb); -+ spin_unlock_irq(&head->lock); -+ } -+#else -+ fput(file); -+#endif -+} -+ -+static void __io_rsrc_put_work(struct io_rsrc_node *ref_node) -+{ -+ struct io_rsrc_data *rsrc_data = ref_node->rsrc_data; -+ struct io_ring_ctx *ctx = rsrc_data->ctx; -+ struct io_rsrc_put *prsrc, *tmp; -+ -+ list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) { -+ list_del(&prsrc->list); -+ -+ if (prsrc->tag) { -+ bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL; -+ -+ io_ring_submit_lock(ctx, lock_ring); -+ spin_lock(&ctx->completion_lock); -+ io_fill_cqe_aux(ctx, prsrc->tag, 0, 0); -+ io_commit_cqring(ctx); -+ spin_unlock(&ctx->completion_lock); -+ io_cqring_ev_posted(ctx); -+ io_ring_submit_unlock(ctx, lock_ring); -+ } -+ -+ rsrc_data->do_put(ctx, prsrc); -+ kfree(prsrc); -+ } -+ -+ io_rsrc_node_destroy(ref_node); -+ if (atomic_dec_and_test(&rsrc_data->refs)) -+ complete(&rsrc_data->done); -+} -+ -+static void io_rsrc_put_work(struct work_struct *work) -+{ -+ struct io_ring_ctx *ctx; -+ struct llist_node *node; -+ -+ ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work); -+ node = llist_del_all(&ctx->rsrc_put_llist); -+ -+ while (node) { -+ struct io_rsrc_node *ref_node; -+ struct llist_node *next = node->next; -+ -+ ref_node = llist_entry(node, struct io_rsrc_node, llist); -+ __io_rsrc_put_work(ref_node); -+ node = next; -+ } -+} -+ -+static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, -+ unsigned nr_args, u64 __user *tags) -+{ -+ __s32 __user *fds = (__s32 __user *) arg; -+ struct file *file; -+ int fd, ret; -+ unsigned i; -+ -+ if (ctx->file_data) -+ return -EBUSY; -+ if (!nr_args) -+ return -EINVAL; -+ if (nr_args > IORING_MAX_FIXED_FILES) -+ return -EMFILE; -+ if (nr_args > rlimit(RLIMIT_NOFILE)) -+ return -EMFILE; -+ ret = io_rsrc_node_switch_start(ctx); -+ if (ret) -+ return ret; -+ ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args, -+ &ctx->file_data); -+ if (ret) -+ return ret; -+ -+ ret = -ENOMEM; -+ if (!io_alloc_file_tables(&ctx->file_table, nr_args)) -+ goto out_free; -+ -+ for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { -+ if (copy_from_user(&fd, &fds[i], sizeof(fd))) { -+ ret = -EFAULT; -+ goto out_fput; -+ } -+ /* allow sparse sets */ -+ if (fd == -1) { -+ ret = -EINVAL; -+ if (unlikely(*io_get_tag_slot(ctx->file_data, i))) -+ goto out_fput; -+ continue; -+ } -+ -+ file = fget(fd); -+ ret = -EBADF; -+ if (unlikely(!file)) -+ goto out_fput; -+ -+ /* -+ * Don't allow io_uring instances to be registered. If UNIX -+ * isn't enabled, then this causes a reference cycle and this -+ * instance can never get freed. If UNIX is enabled we'll -+ * handle it just fine, but there's still no point in allowing -+ * a ring fd as it doesn't support regular read/write anyway. -+ */ -+ if (file->f_op == &io_uring_fops) { -+ fput(file); -+ goto out_fput; -+ } -+ io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file); -+ } -+ -+ ret = io_sqe_files_scm(ctx); -+ if (ret) { -+ __io_sqe_files_unregister(ctx); -+ return ret; -+ } -+ -+ io_rsrc_node_switch(ctx, NULL); -+ return ret; -+out_fput: -+ for (i = 0; i < ctx->nr_user_files; i++) { -+ file = io_file_from_index(ctx, i); -+ if (file) -+ fput(file); -+ } -+ io_free_file_tables(&ctx->file_table); -+ ctx->nr_user_files = 0; -+out_free: -+ io_rsrc_data_free(ctx->file_data); -+ ctx->file_data = NULL; -+ return ret; -+} -+ -+static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file, -+ int index) -+{ -+#if defined(CONFIG_UNIX) -+ struct sock *sock = ctx->ring_sock->sk; -+ struct sk_buff_head *head = &sock->sk_receive_queue; -+ struct sk_buff *skb; -+ -+ /* -+ * See if we can merge this file into an existing skb SCM_RIGHTS -+ * file set. If there's no room, fall back to allocating a new skb -+ * and filling it in. -+ */ -+ spin_lock_irq(&head->lock); -+ skb = skb_peek(head); -+ if (skb) { -+ struct scm_fp_list *fpl = UNIXCB(skb).fp; -+ -+ if (fpl->count < SCM_MAX_FD) { -+ __skb_unlink(skb, head); -+ spin_unlock_irq(&head->lock); -+ fpl->fp[fpl->count] = get_file(file); -+ unix_inflight(fpl->user, fpl->fp[fpl->count]); -+ fpl->count++; -+ spin_lock_irq(&head->lock); -+ __skb_queue_head(head, skb); -+ } else { -+ skb = NULL; -+ } -+ } -+ spin_unlock_irq(&head->lock); -+ -+ if (skb) { -+ fput(file); -+ return 0; -+ } -+ -+ return __io_sqe_files_scm(ctx, 1, index); -+#else -+ return 0; -+#endif -+} -+ -+static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, -+ struct io_rsrc_node *node, void *rsrc) -+{ -+ u64 *tag_slot = io_get_tag_slot(data, idx); -+ struct io_rsrc_put *prsrc; -+ -+ prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL); -+ if (!prsrc) -+ return -ENOMEM; -+ -+ prsrc->tag = *tag_slot; -+ *tag_slot = 0; -+ prsrc->rsrc = rsrc; -+ list_add(&prsrc->list, &node->rsrc_list); -+ return 0; -+} -+ -+static int io_install_fixed_file(struct io_kiocb *req, struct file *file, -+ unsigned int issue_flags, u32 slot_index) -+{ -+ struct io_ring_ctx *ctx = req->ctx; -+ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; -+ bool needs_switch = false; -+ struct io_fixed_file *file_slot; -+ int ret = -EBADF; -+ -+ io_ring_submit_lock(ctx, !force_nonblock); -+ if (file->f_op == &io_uring_fops) -+ goto err; -+ ret = -ENXIO; -+ if (!ctx->file_data) -+ goto err; -+ ret = -EINVAL; -+ if (slot_index >= ctx->nr_user_files) -+ goto err; -+ -+ slot_index = array_index_nospec(slot_index, ctx->nr_user_files); -+ file_slot = io_fixed_file_slot(&ctx->file_table, slot_index); -+ -+ if (file_slot->file_ptr) { -+ struct file *old_file; -+ -+ ret = io_rsrc_node_switch_start(ctx); -+ if (ret) -+ goto err; -+ -+ old_file = (struct file *)(file_slot->file_ptr & FFS_MASK); -+ ret = io_queue_rsrc_removal(ctx->file_data, slot_index, -+ ctx->rsrc_node, old_file); -+ if (ret) -+ goto err; -+ file_slot->file_ptr = 0; -+ needs_switch = true; -+ } -+ -+ *io_get_tag_slot(ctx->file_data, slot_index) = 0; -+ io_fixed_file_set(file_slot, file); -+ ret = io_sqe_file_register(ctx, file, slot_index); -+ if (ret) { -+ file_slot->file_ptr = 0; -+ goto err; -+ } -+ -+ ret = 0; -+err: -+ if (needs_switch) -+ io_rsrc_node_switch(ctx, ctx->file_data); -+ io_ring_submit_unlock(ctx, !force_nonblock); -+ if (ret) -+ fput(file); -+ return ret; -+} -+ -+static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) -+{ -+ unsigned int offset = req->close.file_slot - 1; -+ struct io_ring_ctx *ctx = req->ctx; -+ struct io_fixed_file *file_slot; -+ struct file *file; -+ int ret; -+ -+ io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); -+ ret = -ENXIO; -+ if (unlikely(!ctx->file_data)) -+ goto out; -+ ret = -EINVAL; -+ if (offset >= ctx->nr_user_files) -+ goto out; -+ ret = io_rsrc_node_switch_start(ctx); -+ if (ret) -+ goto out; -+ -+ offset = array_index_nospec(offset, ctx->nr_user_files); -+ file_slot = io_fixed_file_slot(&ctx->file_table, offset); -+ ret = -EBADF; -+ if (!file_slot->file_ptr) -+ goto out; -+ -+ file = (struct file *)(file_slot->file_ptr & FFS_MASK); -+ ret = io_queue_rsrc_removal(ctx->file_data, offset, ctx->rsrc_node, file); -+ if (ret) -+ goto out; -+ -+ file_slot->file_ptr = 0; -+ io_rsrc_node_switch(ctx, ctx->file_data); -+ ret = 0; -+out: -+ io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); -+ return ret; -+} -+ -+static int __io_sqe_files_update(struct io_ring_ctx *ctx, -+ struct io_uring_rsrc_update2 *up, -+ unsigned nr_args) -+{ -+ u64 __user *tags = u64_to_user_ptr(up->tags); -+ __s32 __user *fds = u64_to_user_ptr(up->data); -+ struct io_rsrc_data *data = ctx->file_data; -+ struct io_fixed_file *file_slot; -+ struct file *file; -+ int fd, i, err = 0; -+ unsigned int done; -+ bool needs_switch = false; -+ -+ if (!ctx->file_data) -+ return -ENXIO; -+ if (up->offset + nr_args > ctx->nr_user_files) -+ return -EINVAL; -+ -+ for (done = 0; done < nr_args; done++) { -+ u64 tag = 0; -+ -+ if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) || -+ copy_from_user(&fd, &fds[done], sizeof(fd))) { -+ err = -EFAULT; -+ break; -+ } -+ if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) { -+ err = -EINVAL; -+ break; -+ } -+ if (fd == IORING_REGISTER_FILES_SKIP) -+ continue; -+ -+ i = array_index_nospec(up->offset + done, ctx->nr_user_files); -+ file_slot = io_fixed_file_slot(&ctx->file_table, i); -+ -+ if (file_slot->file_ptr) { -+ file = (struct file *)(file_slot->file_ptr & FFS_MASK); -+ err = io_queue_rsrc_removal(data, i, ctx->rsrc_node, file); -+ if (err) -+ break; -+ file_slot->file_ptr = 0; -+ needs_switch = true; -+ } -+ if (fd != -1) { -+ file = fget(fd); -+ if (!file) { -+ err = -EBADF; -+ break; -+ } -+ /* -+ * Don't allow io_uring instances to be registered. If -+ * UNIX isn't enabled, then this causes a reference -+ * cycle and this instance can never get freed. If UNIX -+ * is enabled we'll handle it just fine, but there's -+ * still no point in allowing a ring fd as it doesn't -+ * support regular read/write anyway. -+ */ -+ if (file->f_op == &io_uring_fops) { -+ fput(file); -+ err = -EBADF; -+ break; -+ } -+ *io_get_tag_slot(data, i) = tag; -+ io_fixed_file_set(file_slot, file); -+ err = io_sqe_file_register(ctx, file, i); -+ if (err) { -+ file_slot->file_ptr = 0; -+ fput(file); -+ break; -+ } -+ } -+ } -+ -+ if (needs_switch) -+ io_rsrc_node_switch(ctx, data); -+ return done ? done : err; -+} -+ -+static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx, -+ struct task_struct *task) -+{ -+ struct io_wq_hash *hash; -+ struct io_wq_data data; -+ unsigned int concurrency; -+ -+ mutex_lock(&ctx->uring_lock); -+ hash = ctx->hash_map; -+ if (!hash) { -+ hash = kzalloc(sizeof(*hash), GFP_KERNEL); -+ if (!hash) { -+ mutex_unlock(&ctx->uring_lock); -+ return ERR_PTR(-ENOMEM); -+ } -+ refcount_set(&hash->refs, 1); -+ init_waitqueue_head(&hash->wait); -+ ctx->hash_map = hash; -+ } -+ mutex_unlock(&ctx->uring_lock); -+ -+ data.hash = hash; -+ data.task = task; -+ data.free_work = io_wq_free_work; -+ data.do_work = io_wq_submit_work; -+ -+ /* Do QD, or 4 * CPUS, whatever is smallest */ -+ concurrency = min(ctx->sq_entries, 4 * num_online_cpus()); -+ -+ return io_wq_create(concurrency, &data); -+} -+ -+static int io_uring_alloc_task_context(struct task_struct *task, -+ struct io_ring_ctx *ctx) -+{ -+ struct io_uring_task *tctx; -+ int ret; -+ -+ tctx = kzalloc(sizeof(*tctx), GFP_KERNEL); -+ if (unlikely(!tctx)) -+ return -ENOMEM; -+ -+ ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL); -+ if (unlikely(ret)) { -+ kfree(tctx); -+ return ret; -+ } -+ -+ tctx->io_wq = io_init_wq_offload(ctx, task); -+ if (IS_ERR(tctx->io_wq)) { -+ ret = PTR_ERR(tctx->io_wq); -+ percpu_counter_destroy(&tctx->inflight); -+ kfree(tctx); -+ return ret; -+ } -+ -+ xa_init(&tctx->xa); -+ init_waitqueue_head(&tctx->wait); -+ atomic_set(&tctx->in_idle, 0); -+ atomic_set(&tctx->inflight_tracked, 0); -+ task->io_uring = tctx; -+ spin_lock_init(&tctx->task_lock); -+ INIT_WQ_LIST(&tctx->task_list); -+ init_task_work(&tctx->task_work, tctx_task_work); -+ return 0; -+} -+ -+void __io_uring_free(struct task_struct *tsk) -+{ -+ struct io_uring_task *tctx = tsk->io_uring; -+ -+ WARN_ON_ONCE(!xa_empty(&tctx->xa)); -+ WARN_ON_ONCE(tctx->io_wq); -+ WARN_ON_ONCE(tctx->cached_refs); -+ -+ percpu_counter_destroy(&tctx->inflight); -+ kfree(tctx); -+ tsk->io_uring = NULL; -+} -+ -+static int io_sq_offload_create(struct io_ring_ctx *ctx, -+ struct io_uring_params *p) -+{ -+ int ret; -+ -+ /* Retain compatibility with failing for an invalid attach attempt */ -+ if ((ctx->flags & (IORING_SETUP_ATTACH_WQ | IORING_SETUP_SQPOLL)) == -+ IORING_SETUP_ATTACH_WQ) { -+ struct fd f; -+ -+ f = fdget(p->wq_fd); -+ if (!f.file) -+ return -ENXIO; -+ if (f.file->f_op != &io_uring_fops) { -+ fdput(f); -+ return -EINVAL; -+ } -+ fdput(f); -+ } -+ if (ctx->flags & IORING_SETUP_SQPOLL) { -+ struct task_struct *tsk; -+ struct io_sq_data *sqd; -+ bool attached; -+ -+ sqd = io_get_sq_data(p, &attached); -+ if (IS_ERR(sqd)) { -+ ret = PTR_ERR(sqd); -+ goto err; -+ } -+ -+ ctx->sq_creds = get_current_cred(); -+ ctx->sq_data = sqd; -+ ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle); -+ if (!ctx->sq_thread_idle) -+ ctx->sq_thread_idle = HZ; -+ -+ io_sq_thread_park(sqd); -+ list_add(&ctx->sqd_list, &sqd->ctx_list); -+ io_sqd_update_thread_idle(sqd); -+ /* don't attach to a dying SQPOLL thread, would be racy */ -+ ret = (attached && !sqd->thread) ? -ENXIO : 0; -+ io_sq_thread_unpark(sqd); -+ -+ if (ret < 0) -+ goto err; -+ if (attached) -+ return 0; -+ -+ if (p->flags & IORING_SETUP_SQ_AFF) { -+ int cpu = p->sq_thread_cpu; -+ -+ ret = -EINVAL; -+ if (cpu >= nr_cpu_ids || !cpu_online(cpu)) -+ goto err_sqpoll; -+ sqd->sq_cpu = cpu; -+ } else { -+ sqd->sq_cpu = -1; -+ } -+ -+ sqd->task_pid = current->pid; -+ sqd->task_tgid = current->tgid; -+ tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE); -+ if (IS_ERR(tsk)) { -+ ret = PTR_ERR(tsk); -+ goto err_sqpoll; -+ } -+ -+ sqd->thread = tsk; -+ ret = io_uring_alloc_task_context(tsk, ctx); -+ wake_up_new_task(tsk); -+ if (ret) -+ goto err; -+ } else if (p->flags & IORING_SETUP_SQ_AFF) { -+ /* Can't have SQ_AFF without SQPOLL */ -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ return 0; -+err_sqpoll: -+ complete(&ctx->sq_data->exited); -+err: -+ io_sq_thread_finish(ctx); -+ return ret; -+} -+ -+static inline void __io_unaccount_mem(struct user_struct *user, -+ unsigned long nr_pages) -+{ -+ atomic_long_sub(nr_pages, &user->locked_vm); -+} -+ -+static inline int __io_account_mem(struct user_struct *user, -+ unsigned long nr_pages) -+{ -+ unsigned long page_limit, cur_pages, new_pages; -+ -+ /* Don't allow more pages than we can safely lock */ -+ page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; -+ -+ do { -+ cur_pages = atomic_long_read(&user->locked_vm); -+ new_pages = cur_pages + nr_pages; -+ if (new_pages > page_limit) -+ return -ENOMEM; -+ } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages, -+ new_pages) != cur_pages); -+ -+ return 0; -+} -+ -+static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages) -+{ -+ if (ctx->user) -+ __io_unaccount_mem(ctx->user, nr_pages); -+ -+ if (ctx->mm_account) -+ atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm); -+} -+ -+static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages) -+{ -+ int ret; -+ -+ if (ctx->user) { -+ ret = __io_account_mem(ctx->user, nr_pages); -+ if (ret) -+ return ret; -+ } -+ -+ if (ctx->mm_account) -+ atomic64_add(nr_pages, &ctx->mm_account->pinned_vm); -+ -+ return 0; -+} -+ -+static void io_mem_free(void *ptr) -+{ -+ struct page *page; -+ -+ if (!ptr) -+ return; -+ -+ page = virt_to_head_page(ptr); -+ if (put_page_testzero(page)) -+ free_compound_page(page); -+} -+ -+static void *io_mem_alloc(size_t size) -+{ -+ gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP; -+ -+ return (void *) __get_free_pages(gfp, get_order(size)); -+} -+ -+static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries, -+ size_t *sq_offset) -+{ -+ struct io_rings *rings; -+ size_t off, sq_array_size; -+ -+ off = struct_size(rings, cqes, cq_entries); -+ if (off == SIZE_MAX) -+ return SIZE_MAX; -+ -+#ifdef CONFIG_SMP -+ off = ALIGN(off, SMP_CACHE_BYTES); -+ if (off == 0) -+ return SIZE_MAX; -+#endif -+ -+ if (sq_offset) -+ *sq_offset = off; -+ -+ sq_array_size = array_size(sizeof(u32), sq_entries); -+ if (sq_array_size == SIZE_MAX) -+ return SIZE_MAX; -+ -+ if (check_add_overflow(off, sq_array_size, &off)) -+ return SIZE_MAX; -+ -+ return off; -+} -+ -+static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot) -+{ -+ struct io_mapped_ubuf *imu = *slot; -+ unsigned int i; -+ -+ if (imu != ctx->dummy_ubuf) { -+ for (i = 0; i < imu->nr_bvecs; i++) -+ unpin_user_page(imu->bvec[i].bv_page); -+ if (imu->acct_pages) -+ io_unaccount_mem(ctx, imu->acct_pages); -+ kvfree(imu); -+ } -+ *slot = NULL; -+} -+ -+static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc) -+{ -+ io_buffer_unmap(ctx, &prsrc->buf); -+ prsrc->buf = NULL; -+} -+ -+static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx) -+{ -+ unsigned int i; -+ -+ for (i = 0; i < ctx->nr_user_bufs; i++) -+ io_buffer_unmap(ctx, &ctx->user_bufs[i]); -+ kfree(ctx->user_bufs); -+ io_rsrc_data_free(ctx->buf_data); -+ ctx->user_bufs = NULL; -+ ctx->buf_data = NULL; -+ ctx->nr_user_bufs = 0; -+} -+ -+static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) -+{ -+ unsigned nr = ctx->nr_user_bufs; -+ int ret; -+ -+ if (!ctx->buf_data) -+ return -ENXIO; -+ -+ /* -+ * Quiesce may unlock ->uring_lock, and while it's not held -+ * prevent new requests using the table. -+ */ -+ ctx->nr_user_bufs = 0; -+ ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx); -+ ctx->nr_user_bufs = nr; -+ if (!ret) -+ __io_sqe_buffers_unregister(ctx); -+ return ret; -+} -+ -+static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst, -+ void __user *arg, unsigned index) -+{ -+ struct iovec __user *src; -+ -+#ifdef CONFIG_COMPAT -+ if (ctx->compat) { -+ struct compat_iovec __user *ciovs; -+ struct compat_iovec ciov; -+ -+ ciovs = (struct compat_iovec __user *) arg; -+ if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov))) -+ return -EFAULT; -+ -+ dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base); -+ dst->iov_len = ciov.iov_len; -+ return 0; -+ } -+#endif -+ src = (struct iovec __user *) arg; -+ if (copy_from_user(dst, &src[index], sizeof(*dst))) -+ return -EFAULT; -+ return 0; -+} -+ -+/* -+ * Not super efficient, but this is just a registration time. And we do cache -+ * the last compound head, so generally we'll only do a full search if we don't -+ * match that one. -+ * -+ * We check if the given compound head page has already been accounted, to -+ * avoid double accounting it. This allows us to account the full size of the -+ * page, not just the constituent pages of a huge page. -+ */ -+static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages, -+ int nr_pages, struct page *hpage) -+{ -+ int i, j; -+ -+ /* check current page array */ -+ for (i = 0; i < nr_pages; i++) { -+ if (!PageCompound(pages[i])) -+ continue; -+ if (compound_head(pages[i]) == hpage) -+ return true; -+ } -+ -+ /* check previously registered pages */ -+ for (i = 0; i < ctx->nr_user_bufs; i++) { -+ struct io_mapped_ubuf *imu = ctx->user_bufs[i]; -+ -+ for (j = 0; j < imu->nr_bvecs; j++) { -+ if (!PageCompound(imu->bvec[j].bv_page)) -+ continue; -+ if (compound_head(imu->bvec[j].bv_page) == hpage) -+ return true; -+ } -+ } -+ -+ return false; -+} -+ -+static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages, -+ int nr_pages, struct io_mapped_ubuf *imu, -+ struct page **last_hpage) -+{ -+ int i, ret; -+ -+ imu->acct_pages = 0; -+ for (i = 0; i < nr_pages; i++) { -+ if (!PageCompound(pages[i])) { -+ imu->acct_pages++; -+ } else { -+ struct page *hpage; -+ -+ hpage = compound_head(pages[i]); -+ if (hpage == *last_hpage) -+ continue; -+ *last_hpage = hpage; -+ if (headpage_already_acct(ctx, pages, i, hpage)) -+ continue; -+ imu->acct_pages += page_size(hpage) >> PAGE_SHIFT; -+ } -+ } -+ -+ if (!imu->acct_pages) -+ return 0; -+ -+ ret = io_account_mem(ctx, imu->acct_pages); -+ if (ret) -+ imu->acct_pages = 0; -+ return ret; -+} -+ -+static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, -+ struct io_mapped_ubuf **pimu, -+ struct page **last_hpage) -+{ -+ struct io_mapped_ubuf *imu = NULL; -+ struct vm_area_struct **vmas = NULL; -+ struct page **pages = NULL; -+ unsigned long off, start, end, ubuf; -+ size_t size; -+ int ret, pret, nr_pages, i; -+ -+ if (!iov->iov_base) { -+ *pimu = ctx->dummy_ubuf; -+ return 0; -+ } -+ -+ ubuf = (unsigned long) iov->iov_base; -+ end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT; -+ start = ubuf >> PAGE_SHIFT; -+ nr_pages = end - start; -+ -+ *pimu = NULL; -+ ret = -ENOMEM; -+ -+ pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL); -+ if (!pages) -+ goto done; -+ -+ vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *), -+ GFP_KERNEL); -+ if (!vmas) -+ goto done; -+ -+ imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL); -+ if (!imu) -+ goto done; -+ -+ ret = 0; -+ mmap_read_lock(current->mm); -+ pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM, -+ pages, vmas); -+ if (pret == nr_pages) { -+ /* don't support file backed memory */ -+ for (i = 0; i < nr_pages; i++) { -+ struct vm_area_struct *vma = vmas[i]; -+ -+ if (vma_is_shmem(vma)) -+ continue; -+ if (vma->vm_file && -+ !is_file_hugepages(vma->vm_file)) { -+ ret = -EOPNOTSUPP; -+ break; -+ } -+ } -+ } else { -+ ret = pret < 0 ? pret : -EFAULT; -+ } -+ mmap_read_unlock(current->mm); -+ if (ret) { -+ /* -+ * if we did partial map, or found file backed vmas, -+ * release any pages we did get -+ */ -+ if (pret > 0) -+ unpin_user_pages(pages, pret); -+ goto done; -+ } -+ -+ ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage); -+ if (ret) { -+ unpin_user_pages(pages, pret); -+ goto done; -+ } -+ -+ off = ubuf & ~PAGE_MASK; -+ size = iov->iov_len; -+ for (i = 0; i < nr_pages; i++) { -+ size_t vec_len; -+ -+ vec_len = min_t(size_t, size, PAGE_SIZE - off); -+ imu->bvec[i].bv_page = pages[i]; -+ imu->bvec[i].bv_len = vec_len; -+ imu->bvec[i].bv_offset = off; -+ off = 0; -+ size -= vec_len; -+ } -+ /* store original address for later verification */ -+ imu->ubuf = ubuf; -+ imu->ubuf_end = ubuf + iov->iov_len; -+ imu->nr_bvecs = nr_pages; -+ *pimu = imu; -+ ret = 0; -+done: -+ if (ret) -+ kvfree(imu); -+ kvfree(pages); -+ kvfree(vmas); -+ return ret; -+} -+ -+static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args) -+{ -+ ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL); -+ return ctx->user_bufs ? 0 : -ENOMEM; -+} -+ -+static int io_buffer_validate(struct iovec *iov) -+{ -+ unsigned long tmp, acct_len = iov->iov_len + (PAGE_SIZE - 1); -+ -+ /* -+ * Don't impose further limits on the size and buffer -+ * constraints here, we'll -EINVAL later when IO is -+ * submitted if they are wrong. -+ */ -+ if (!iov->iov_base) -+ return iov->iov_len ? -EFAULT : 0; -+ if (!iov->iov_len) -+ return -EFAULT; -+ -+ /* arbitrary limit, but we need something */ -+ if (iov->iov_len > SZ_1G) -+ return -EFAULT; -+ -+ if (check_add_overflow((unsigned long)iov->iov_base, acct_len, &tmp)) -+ return -EOVERFLOW; -+ -+ return 0; -+} -+ -+static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, -+ unsigned int nr_args, u64 __user *tags) -+{ -+ struct page *last_hpage = NULL; -+ struct io_rsrc_data *data; -+ int i, ret; -+ struct iovec iov; -+ -+ if (ctx->user_bufs) -+ return -EBUSY; -+ if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS) -+ return -EINVAL; -+ ret = io_rsrc_node_switch_start(ctx); -+ if (ret) -+ return ret; -+ ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data); -+ if (ret) -+ return ret; -+ ret = io_buffers_map_alloc(ctx, nr_args); -+ if (ret) { -+ io_rsrc_data_free(data); -+ return ret; -+ } -+ -+ for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) { -+ ret = io_copy_iov(ctx, &iov, arg, i); -+ if (ret) -+ break; -+ ret = io_buffer_validate(&iov); -+ if (ret) -+ break; -+ if (!iov.iov_base && *io_get_tag_slot(data, i)) { -+ ret = -EINVAL; -+ break; -+ } -+ -+ ret = io_sqe_buffer_register(ctx, &iov, &ctx->user_bufs[i], -+ &last_hpage); -+ if (ret) -+ break; -+ } -+ -+ WARN_ON_ONCE(ctx->buf_data); -+ -+ ctx->buf_data = data; -+ if (ret) -+ __io_sqe_buffers_unregister(ctx); -+ else -+ io_rsrc_node_switch(ctx, NULL); -+ return ret; -+} -+ -+static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, -+ struct io_uring_rsrc_update2 *up, -+ unsigned int nr_args) -+{ -+ u64 __user *tags = u64_to_user_ptr(up->tags); -+ struct iovec iov, __user *iovs = u64_to_user_ptr(up->data); -+ struct page *last_hpage = NULL; -+ bool needs_switch = false; -+ __u32 done; -+ int i, err; -+ -+ if (!ctx->buf_data) -+ return -ENXIO; -+ if (up->offset + nr_args > ctx->nr_user_bufs) -+ return -EINVAL; -+ -+ for (done = 0; done < nr_args; done++) { -+ struct io_mapped_ubuf *imu; -+ int offset = up->offset + done; -+ u64 tag = 0; -+ -+ err = io_copy_iov(ctx, &iov, iovs, done); -+ if (err) -+ break; -+ if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) { -+ err = -EFAULT; -+ break; -+ } -+ err = io_buffer_validate(&iov); -+ if (err) -+ break; -+ if (!iov.iov_base && tag) { -+ err = -EINVAL; -+ break; -+ } -+ err = io_sqe_buffer_register(ctx, &iov, &imu, &last_hpage); -+ if (err) -+ break; -+ -+ i = array_index_nospec(offset, ctx->nr_user_bufs); -+ if (ctx->user_bufs[i] != ctx->dummy_ubuf) { -+ err = io_queue_rsrc_removal(ctx->buf_data, i, -+ ctx->rsrc_node, ctx->user_bufs[i]); -+ if (unlikely(err)) { -+ io_buffer_unmap(ctx, &imu); -+ break; -+ } -+ ctx->user_bufs[i] = NULL; -+ needs_switch = true; -+ } -+ -+ ctx->user_bufs[i] = imu; -+ *io_get_tag_slot(ctx->buf_data, offset) = tag; -+ } -+ -+ if (needs_switch) -+ io_rsrc_node_switch(ctx, ctx->buf_data); -+ return done ? done : err; -+} -+ -+static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg) -+{ -+ __s32 __user *fds = arg; -+ int fd; -+ -+ if (ctx->cq_ev_fd) -+ return -EBUSY; -+ -+ if (copy_from_user(&fd, fds, sizeof(*fds))) -+ return -EFAULT; -+ -+ ctx->cq_ev_fd = eventfd_ctx_fdget(fd); -+ if (IS_ERR(ctx->cq_ev_fd)) { -+ int ret = PTR_ERR(ctx->cq_ev_fd); -+ -+ ctx->cq_ev_fd = NULL; -+ return ret; -+ } -+ -+ return 0; -+} -+ -+static int io_eventfd_unregister(struct io_ring_ctx *ctx) -+{ -+ if (ctx->cq_ev_fd) { -+ eventfd_ctx_put(ctx->cq_ev_fd); -+ ctx->cq_ev_fd = NULL; -+ return 0; -+ } -+ -+ return -ENXIO; -+} -+ -+static void io_destroy_buffers(struct io_ring_ctx *ctx) -+{ -+ struct io_buffer *buf; -+ unsigned long index; -+ -+ xa_for_each(&ctx->io_buffers, index, buf) -+ __io_remove_buffers(ctx, buf, index, -1U); -+} -+ -+static void io_req_cache_free(struct list_head *list) -+{ -+ struct io_kiocb *req, *nxt; -+ -+ list_for_each_entry_safe(req, nxt, list, inflight_entry) { -+ list_del(&req->inflight_entry); -+ kmem_cache_free(req_cachep, req); -+ } -+} -+ -+static void io_req_caches_free(struct io_ring_ctx *ctx) -+{ -+ struct io_submit_state *state = &ctx->submit_state; -+ -+ mutex_lock(&ctx->uring_lock); -+ -+ if (state->free_reqs) { -+ kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs); -+ state->free_reqs = 0; -+ } -+ -+ io_flush_cached_locked_reqs(ctx, state); -+ io_req_cache_free(&state->free_list); -+ mutex_unlock(&ctx->uring_lock); -+} -+ -+static void io_wait_rsrc_data(struct io_rsrc_data *data) -+{ -+ if (data && !atomic_dec_and_test(&data->refs)) -+ wait_for_completion(&data->done); -+} -+ -+static void io_ring_ctx_free(struct io_ring_ctx *ctx) -+{ -+ io_sq_thread_finish(ctx); -+ -+ /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */ -+ io_wait_rsrc_data(ctx->buf_data); -+ io_wait_rsrc_data(ctx->file_data); -+ -+ mutex_lock(&ctx->uring_lock); -+ if (ctx->buf_data) -+ __io_sqe_buffers_unregister(ctx); -+ if (ctx->file_data) -+ __io_sqe_files_unregister(ctx); -+ if (ctx->rings) -+ __io_cqring_overflow_flush(ctx, true); -+ mutex_unlock(&ctx->uring_lock); -+ io_eventfd_unregister(ctx); -+ io_destroy_buffers(ctx); -+ if (ctx->sq_creds) -+ put_cred(ctx->sq_creds); -+ -+ /* there are no registered resources left, nobody uses it */ -+ if (ctx->rsrc_node) -+ io_rsrc_node_destroy(ctx->rsrc_node); -+ if (ctx->rsrc_backup_node) -+ io_rsrc_node_destroy(ctx->rsrc_backup_node); -+ flush_delayed_work(&ctx->rsrc_put_work); -+ -+ WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list)); -+ WARN_ON_ONCE(!llist_empty(&ctx->rsrc_put_llist)); -+ -+#if defined(CONFIG_UNIX) -+ if (ctx->ring_sock) { -+ ctx->ring_sock->file = NULL; /* so that iput() is called */ -+ sock_release(ctx->ring_sock); -+ } -+#endif -+ WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list)); -+ -+ if (ctx->mm_account) { -+ mmdrop(ctx->mm_account); -+ ctx->mm_account = NULL; -+ } -+ -+ io_mem_free(ctx->rings); -+ io_mem_free(ctx->sq_sqes); -+ -+ percpu_ref_exit(&ctx->refs); -+ free_uid(ctx->user); -+ io_req_caches_free(ctx); -+ if (ctx->hash_map) -+ io_wq_put_hash(ctx->hash_map); -+ kfree(ctx->cancel_hash); -+ kfree(ctx->dummy_ubuf); -+ kfree(ctx); -+} -+ -+static __poll_t io_uring_poll(struct file *file, poll_table *wait) -+{ -+ struct io_ring_ctx *ctx = file->private_data; -+ __poll_t mask = 0; -+ -+ poll_wait(file, &ctx->poll_wait, wait); -+ /* -+ * synchronizes with barrier from wq_has_sleeper call in -+ * io_commit_cqring -+ */ -+ smp_rmb(); -+ if (!io_sqring_full(ctx)) -+ mask |= EPOLLOUT | EPOLLWRNORM; -+ -+ /* -+ * Don't flush cqring overflow list here, just do a simple check. -+ * Otherwise there could possible be ABBA deadlock: -+ * CPU0 CPU1 -+ * ---- ---- -+ * lock(&ctx->uring_lock); -+ * lock(&ep->mtx); -+ * lock(&ctx->uring_lock); -+ * lock(&ep->mtx); -+ * -+ * Users may get EPOLLIN meanwhile seeing nothing in cqring, this -+ * pushs them to do the flush. -+ */ -+ if (io_cqring_events(ctx) || test_bit(0, &ctx->check_cq_overflow)) -+ mask |= EPOLLIN | EPOLLRDNORM; -+ -+ return mask; -+} -+ -+static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) -+{ -+ const struct cred *creds; -+ -+ creds = xa_erase(&ctx->personalities, id); -+ if (creds) { -+ put_cred(creds); -+ return 0; -+ } -+ -+ return -EINVAL; -+} -+ -+struct io_tctx_exit { -+ struct callback_head task_work; -+ struct completion completion; -+ struct io_ring_ctx *ctx; -+}; -+ -+static void io_tctx_exit_cb(struct callback_head *cb) -+{ -+ struct io_uring_task *tctx = current->io_uring; -+ struct io_tctx_exit *work; -+ -+ work = container_of(cb, struct io_tctx_exit, task_work); -+ /* -+ * When @in_idle, we're in cancellation and it's racy to remove the -+ * node. It'll be removed by the end of cancellation, just ignore it. -+ * tctx can be NULL if the queueing of this task_work raced with -+ * work cancelation off the exec path. -+ */ -+ if (tctx && !atomic_read(&tctx->in_idle)) -+ io_uring_del_tctx_node((unsigned long)work->ctx); -+ complete(&work->completion); -+} -+ -+static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data) -+{ -+ struct io_kiocb *req = container_of(work, struct io_kiocb, work); -+ -+ return req->ctx == data; -+} -+ -+static void io_ring_exit_work(struct work_struct *work) -+{ -+ struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work); -+ unsigned long timeout = jiffies + HZ * 60 * 5; -+ unsigned long interval = HZ / 20; -+ struct io_tctx_exit exit; -+ struct io_tctx_node *node; -+ int ret; -+ -+ /* -+ * If we're doing polled IO and end up having requests being -+ * submitted async (out-of-line), then completions can come in while -+ * we're waiting for refs to drop. We need to reap these manually, -+ * as nobody else will be looking for them. -+ */ -+ do { -+ io_uring_try_cancel_requests(ctx, NULL, true); -+ if (ctx->sq_data) { -+ struct io_sq_data *sqd = ctx->sq_data; -+ struct task_struct *tsk; -+ -+ io_sq_thread_park(sqd); -+ tsk = sqd->thread; -+ if (tsk && tsk->io_uring && tsk->io_uring->io_wq) -+ io_wq_cancel_cb(tsk->io_uring->io_wq, -+ io_cancel_ctx_cb, ctx, true); -+ io_sq_thread_unpark(sqd); -+ } -+ -+ if (WARN_ON_ONCE(time_after(jiffies, timeout))) { -+ /* there is little hope left, don't run it too often */ -+ interval = HZ * 60; -+ } -+ } while (!wait_for_completion_timeout(&ctx->ref_comp, interval)); -+ -+ init_completion(&exit.completion); -+ init_task_work(&exit.task_work, io_tctx_exit_cb); -+ exit.ctx = ctx; -+ /* -+ * Some may use context even when all refs and requests have been put, -+ * and they are free to do so while still holding uring_lock or -+ * completion_lock, see io_req_task_submit(). Apart from other work, -+ * this lock/unlock section also waits them to finish. -+ */ -+ mutex_lock(&ctx->uring_lock); -+ while (!list_empty(&ctx->tctx_list)) { -+ WARN_ON_ONCE(time_after(jiffies, timeout)); -+ -+ node = list_first_entry(&ctx->tctx_list, struct io_tctx_node, -+ ctx_node); -+ /* don't spin on a single task if cancellation failed */ -+ list_rotate_left(&ctx->tctx_list); -+ ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL); -+ if (WARN_ON_ONCE(ret)) -+ continue; -+ wake_up_process(node->task); -+ -+ mutex_unlock(&ctx->uring_lock); -+ wait_for_completion(&exit.completion); -+ mutex_lock(&ctx->uring_lock); -+ } -+ mutex_unlock(&ctx->uring_lock); -+ spin_lock(&ctx->completion_lock); -+ spin_unlock(&ctx->completion_lock); -+ -+ io_ring_ctx_free(ctx); -+} -+ -+/* Returns true if we found and killed one or more timeouts */ -+static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, -+ bool cancel_all) -+{ -+ struct io_kiocb *req, *tmp; -+ int canceled = 0; -+ -+ spin_lock(&ctx->completion_lock); -+ spin_lock_irq(&ctx->timeout_lock); -+ list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { -+ if (io_match_task(req, tsk, cancel_all)) { -+ io_kill_timeout(req, -ECANCELED); -+ canceled++; -+ } -+ } -+ spin_unlock_irq(&ctx->timeout_lock); -+ if (canceled != 0) -+ io_commit_cqring(ctx); -+ spin_unlock(&ctx->completion_lock); -+ if (canceled != 0) -+ io_cqring_ev_posted(ctx); -+ return canceled != 0; -+} -+ -+static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) -+{ -+ unsigned long index; -+ struct creds *creds; -+ -+ mutex_lock(&ctx->uring_lock); -+ percpu_ref_kill(&ctx->refs); -+ if (ctx->rings) -+ __io_cqring_overflow_flush(ctx, true); -+ xa_for_each(&ctx->personalities, index, creds) -+ io_unregister_personality(ctx, index); -+ mutex_unlock(&ctx->uring_lock); -+ -+ io_kill_timeouts(ctx, NULL, true); -+ io_poll_remove_all(ctx, NULL, true); -+ -+ /* if we failed setting up the ctx, we might not have any rings */ -+ io_iopoll_try_reap_events(ctx); -+ -+ INIT_WORK(&ctx->exit_work, io_ring_exit_work); -+ /* -+ * Use system_unbound_wq to avoid spawning tons of event kworkers -+ * if we're exiting a ton of rings at the same time. It just adds -+ * noise and overhead, there's no discernable change in runtime -+ * over using system_wq. -+ */ -+ queue_work(system_unbound_wq, &ctx->exit_work); -+} -+ -+static int io_uring_release(struct inode *inode, struct file *file) -+{ -+ struct io_ring_ctx *ctx = file->private_data; -+ -+ file->private_data = NULL; -+ io_ring_ctx_wait_and_kill(ctx); -+ return 0; -+} -+ -+struct io_task_cancel { -+ struct task_struct *task; -+ bool all; -+}; -+ -+static bool io_cancel_task_cb(struct io_wq_work *work, void *data) -+{ -+ struct io_kiocb *req = container_of(work, struct io_kiocb, work); -+ struct io_task_cancel *cancel = data; -+ -+ return io_match_task_safe(req, cancel->task, cancel->all); -+} -+ -+static bool io_cancel_defer_files(struct io_ring_ctx *ctx, -+ struct task_struct *task, bool cancel_all) -+{ -+ struct io_defer_entry *de; -+ LIST_HEAD(list); -+ -+ spin_lock(&ctx->completion_lock); -+ list_for_each_entry_reverse(de, &ctx->defer_list, list) { -+ if (io_match_task_safe(de->req, task, cancel_all)) { -+ list_cut_position(&list, &ctx->defer_list, &de->list); -+ break; -+ } -+ } -+ spin_unlock(&ctx->completion_lock); -+ if (list_empty(&list)) -+ return false; -+ -+ while (!list_empty(&list)) { -+ de = list_first_entry(&list, struct io_defer_entry, list); -+ list_del_init(&de->list); -+ io_req_complete_failed(de->req, -ECANCELED); -+ kfree(de); -+ } -+ return true; -+} -+ -+static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx) -+{ -+ struct io_tctx_node *node; -+ enum io_wq_cancel cret; -+ bool ret = false; -+ -+ mutex_lock(&ctx->uring_lock); -+ list_for_each_entry(node, &ctx->tctx_list, ctx_node) { -+ struct io_uring_task *tctx = node->task->io_uring; -+ -+ /* -+ * io_wq will stay alive while we hold uring_lock, because it's -+ * killed after ctx nodes, which requires to take the lock. -+ */ -+ if (!tctx || !tctx->io_wq) -+ continue; -+ cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true); -+ ret |= (cret != IO_WQ_CANCEL_NOTFOUND); -+ } -+ mutex_unlock(&ctx->uring_lock); -+ -+ return ret; -+} -+ -+static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, -+ struct task_struct *task, -+ bool cancel_all) -+{ -+ struct io_task_cancel cancel = { .task = task, .all = cancel_all, }; -+ struct io_uring_task *tctx = task ? task->io_uring : NULL; -+ -+ while (1) { -+ enum io_wq_cancel cret; -+ bool ret = false; -+ -+ if (!task) { -+ ret |= io_uring_try_cancel_iowq(ctx); -+ } else if (tctx && tctx->io_wq) { -+ /* -+ * Cancels requests of all rings, not only @ctx, but -+ * it's fine as the task is in exit/exec. -+ */ -+ cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb, -+ &cancel, true); -+ ret |= (cret != IO_WQ_CANCEL_NOTFOUND); -+ } -+ -+ /* SQPOLL thread does its own polling */ -+ if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) || -+ (ctx->sq_data && ctx->sq_data->thread == current)) { -+ while (!list_empty_careful(&ctx->iopoll_list)) { -+ io_iopoll_try_reap_events(ctx); -+ ret = true; -+ } -+ } -+ -+ ret |= io_cancel_defer_files(ctx, task, cancel_all); -+ ret |= io_poll_remove_all(ctx, task, cancel_all); -+ ret |= io_kill_timeouts(ctx, task, cancel_all); -+ if (task) -+ ret |= io_run_task_work(); -+ if (!ret) -+ break; -+ cond_resched(); -+ } -+} -+ -+static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) -+{ -+ struct io_uring_task *tctx = current->io_uring; -+ struct io_tctx_node *node; -+ int ret; -+ -+ if (unlikely(!tctx)) { -+ ret = io_uring_alloc_task_context(current, ctx); -+ if (unlikely(ret)) -+ return ret; -+ -+ tctx = current->io_uring; -+ if (ctx->iowq_limits_set) { -+ unsigned int limits[2] = { ctx->iowq_limits[0], -+ ctx->iowq_limits[1], }; -+ -+ ret = io_wq_max_workers(tctx->io_wq, limits); -+ if (ret) -+ return ret; -+ } -+ } -+ if (!xa_load(&tctx->xa, (unsigned long)ctx)) { -+ node = kmalloc(sizeof(*node), GFP_KERNEL); -+ if (!node) -+ return -ENOMEM; -+ node->ctx = ctx; -+ node->task = current; -+ -+ ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx, -+ node, GFP_KERNEL)); -+ if (ret) { -+ kfree(node); -+ return ret; -+ } -+ -+ mutex_lock(&ctx->uring_lock); -+ list_add(&node->ctx_node, &ctx->tctx_list); -+ mutex_unlock(&ctx->uring_lock); -+ } -+ tctx->last = ctx; -+ return 0; -+} -+ -+/* -+ * Note that this task has used io_uring. We use it for cancelation purposes. -+ */ -+static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx) -+{ -+ struct io_uring_task *tctx = current->io_uring; -+ -+ if (likely(tctx && tctx->last == ctx)) -+ return 0; -+ return __io_uring_add_tctx_node(ctx); -+} -+ -+/* -+ * Remove this io_uring_file -> task mapping. -+ */ -+static void io_uring_del_tctx_node(unsigned long index) -+{ -+ struct io_uring_task *tctx = current->io_uring; -+ struct io_tctx_node *node; -+ -+ if (!tctx) -+ return; -+ node = xa_erase(&tctx->xa, index); -+ if (!node) -+ return; -+ -+ WARN_ON_ONCE(current != node->task); -+ WARN_ON_ONCE(list_empty(&node->ctx_node)); -+ -+ mutex_lock(&node->ctx->uring_lock); -+ list_del(&node->ctx_node); -+ mutex_unlock(&node->ctx->uring_lock); -+ -+ if (tctx->last == node->ctx) -+ tctx->last = NULL; -+ kfree(node); -+} -+ -+static void io_uring_clean_tctx(struct io_uring_task *tctx) -+{ -+ struct io_wq *wq = tctx->io_wq; -+ struct io_tctx_node *node; -+ unsigned long index; -+ -+ xa_for_each(&tctx->xa, index, node) { -+ io_uring_del_tctx_node(index); -+ cond_resched(); -+ } -+ if (wq) { -+ /* -+ * Must be after io_uring_del_task_file() (removes nodes under -+ * uring_lock) to avoid race with io_uring_try_cancel_iowq(). -+ */ -+ io_wq_put_and_exit(wq); -+ tctx->io_wq = NULL; -+ } -+} -+ -+static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked) -+{ -+ if (tracked) -+ return atomic_read(&tctx->inflight_tracked); -+ return percpu_counter_sum(&tctx->inflight); -+} -+ -+/* -+ * Find any io_uring ctx that this task has registered or done IO on, and cancel -+ * requests. @sqd should be not-null IFF it's an SQPOLL thread cancellation. -+ */ -+static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd) -+{ -+ struct io_uring_task *tctx = current->io_uring; -+ struct io_ring_ctx *ctx; -+ s64 inflight; -+ DEFINE_WAIT(wait); -+ -+ WARN_ON_ONCE(sqd && sqd->thread != current); -+ -+ if (!current->io_uring) -+ return; -+ if (tctx->io_wq) -+ io_wq_exit_start(tctx->io_wq); -+ -+ atomic_inc(&tctx->in_idle); -+ do { -+ io_uring_drop_tctx_refs(current); -+ /* read completions before cancelations */ -+ inflight = tctx_inflight(tctx, !cancel_all); -+ if (!inflight) -+ break; -+ -+ if (!sqd) { -+ struct io_tctx_node *node; -+ unsigned long index; -+ -+ xa_for_each(&tctx->xa, index, node) { -+ /* sqpoll task will cancel all its requests */ -+ if (node->ctx->sq_data) -+ continue; -+ io_uring_try_cancel_requests(node->ctx, current, -+ cancel_all); -+ } -+ } else { -+ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) -+ io_uring_try_cancel_requests(ctx, current, -+ cancel_all); -+ } -+ -+ prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE); -+ io_run_task_work(); -+ io_uring_drop_tctx_refs(current); -+ -+ /* -+ * If we've seen completions, retry without waiting. This -+ * avoids a race where a completion comes in before we did -+ * prepare_to_wait(). -+ */ -+ if (inflight == tctx_inflight(tctx, !cancel_all)) -+ schedule(); -+ finish_wait(&tctx->wait, &wait); -+ } while (1); -+ -+ io_uring_clean_tctx(tctx); -+ if (cancel_all) { -+ /* -+ * We shouldn't run task_works after cancel, so just leave -+ * ->in_idle set for normal exit. -+ */ -+ atomic_dec(&tctx->in_idle); -+ /* for exec all current's requests should be gone, kill tctx */ -+ __io_uring_free(current); -+ } -+} -+ -+void __io_uring_cancel(bool cancel_all) -+{ -+ io_uring_cancel_generic(cancel_all, NULL); -+} -+ -+static void *io_uring_validate_mmap_request(struct file *file, -+ loff_t pgoff, size_t sz) -+{ -+ struct io_ring_ctx *ctx = file->private_data; -+ loff_t offset = pgoff << PAGE_SHIFT; -+ struct page *page; -+ void *ptr; -+ -+ switch (offset) { -+ case IORING_OFF_SQ_RING: -+ case IORING_OFF_CQ_RING: -+ ptr = ctx->rings; -+ break; -+ case IORING_OFF_SQES: -+ ptr = ctx->sq_sqes; -+ break; -+ default: -+ return ERR_PTR(-EINVAL); -+ } -+ -+ page = virt_to_head_page(ptr); -+ if (sz > page_size(page)) -+ return ERR_PTR(-EINVAL); -+ -+ return ptr; -+} -+ -+#ifdef CONFIG_MMU -+ -+static int io_uring_mmap(struct file *file, struct vm_area_struct *vma) -+{ -+ size_t sz = vma->vm_end - vma->vm_start; -+ unsigned long pfn; -+ void *ptr; -+ -+ ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz); -+ if (IS_ERR(ptr)) -+ return PTR_ERR(ptr); -+ -+ pfn = virt_to_phys(ptr) >> PAGE_SHIFT; -+ return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot); -+} -+ -+#else /* !CONFIG_MMU */ -+ -+static int io_uring_mmap(struct file *file, struct vm_area_struct *vma) -+{ -+ return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL; -+} -+ -+static unsigned int io_uring_nommu_mmap_capabilities(struct file *file) -+{ -+ return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE; -+} -+ -+static unsigned long io_uring_nommu_get_unmapped_area(struct file *file, -+ unsigned long addr, unsigned long len, -+ unsigned long pgoff, unsigned long flags) -+{ -+ void *ptr; -+ -+ ptr = io_uring_validate_mmap_request(file, pgoff, len); -+ if (IS_ERR(ptr)) -+ return PTR_ERR(ptr); -+ -+ return (unsigned long) ptr; -+} -+ -+#endif /* !CONFIG_MMU */ -+ -+static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx) -+{ -+ DEFINE_WAIT(wait); -+ -+ do { -+ if (!io_sqring_full(ctx)) -+ break; -+ prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE); -+ -+ if (!io_sqring_full(ctx)) -+ break; -+ schedule(); -+ } while (!signal_pending(current)); -+ -+ finish_wait(&ctx->sqo_sq_wait, &wait); -+ return 0; -+} -+ -+static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz, -+ struct __kernel_timespec __user **ts, -+ const sigset_t __user **sig) -+{ -+ struct io_uring_getevents_arg arg; -+ -+ /* -+ * If EXT_ARG isn't set, then we have no timespec and the argp pointer -+ * is just a pointer to the sigset_t. -+ */ -+ if (!(flags & IORING_ENTER_EXT_ARG)) { -+ *sig = (const sigset_t __user *) argp; -+ *ts = NULL; -+ return 0; -+ } -+ -+ /* -+ * EXT_ARG is set - ensure we agree on the size of it and copy in our -+ * timespec and sigset_t pointers if good. -+ */ -+ if (*argsz != sizeof(arg)) -+ return -EINVAL; -+ if (copy_from_user(&arg, argp, sizeof(arg))) -+ return -EFAULT; -+ if (arg.pad) -+ return -EINVAL; -+ *sig = u64_to_user_ptr(arg.sigmask); -+ *argsz = arg.sigmask_sz; -+ *ts = u64_to_user_ptr(arg.ts); -+ return 0; -+} -+ -+SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, -+ u32, min_complete, u32, flags, const void __user *, argp, -+ size_t, argsz) -+{ -+ struct io_ring_ctx *ctx; -+ int submitted = 0; -+ struct fd f; -+ long ret; -+ -+ io_run_task_work(); -+ -+ if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP | -+ IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG))) -+ return -EINVAL; -+ -+ f = fdget(fd); -+ if (unlikely(!f.file)) -+ return -EBADF; -+ -+ ret = -EOPNOTSUPP; -+ if (unlikely(f.file->f_op != &io_uring_fops)) -+ goto out_fput; -+ -+ ret = -ENXIO; -+ ctx = f.file->private_data; -+ if (unlikely(!percpu_ref_tryget(&ctx->refs))) -+ goto out_fput; -+ -+ ret = -EBADFD; -+ if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED)) -+ goto out; -+ -+ /* -+ * For SQ polling, the thread will do all submissions and completions. -+ * Just return the requested submit count, and wake the thread if -+ * we were asked to. -+ */ -+ ret = 0; -+ if (ctx->flags & IORING_SETUP_SQPOLL) { -+ io_cqring_overflow_flush(ctx); -+ -+ if (unlikely(ctx->sq_data->thread == NULL)) { -+ ret = -EOWNERDEAD; -+ goto out; -+ } -+ if (flags & IORING_ENTER_SQ_WAKEUP) -+ wake_up(&ctx->sq_data->wait); -+ if (flags & IORING_ENTER_SQ_WAIT) { -+ ret = io_sqpoll_wait_sq(ctx); -+ if (ret) -+ goto out; -+ } -+ submitted = to_submit; -+ } else if (to_submit) { -+ ret = io_uring_add_tctx_node(ctx); -+ if (unlikely(ret)) -+ goto out; -+ mutex_lock(&ctx->uring_lock); -+ submitted = io_submit_sqes(ctx, to_submit); -+ mutex_unlock(&ctx->uring_lock); -+ -+ if (submitted != to_submit) -+ goto out; -+ } -+ if (flags & IORING_ENTER_GETEVENTS) { -+ const sigset_t __user *sig; -+ struct __kernel_timespec __user *ts; -+ -+ ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig); -+ if (unlikely(ret)) -+ goto out; -+ -+ min_complete = min(min_complete, ctx->cq_entries); -+ -+ /* -+ * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user -+ * space applications don't need to do io completion events -+ * polling again, they can rely on io_sq_thread to do polling -+ * work, which can reduce cpu usage and uring_lock contention. -+ */ -+ if (ctx->flags & IORING_SETUP_IOPOLL && -+ !(ctx->flags & IORING_SETUP_SQPOLL)) { -+ ret = io_iopoll_check(ctx, min_complete); -+ } else { -+ ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts); -+ } -+ } -+ -+out: -+ percpu_ref_put(&ctx->refs); -+out_fput: -+ fdput(f); -+ return submitted ? submitted : ret; -+} -+ -+#ifdef CONFIG_PROC_FS -+static int io_uring_show_cred(struct seq_file *m, unsigned int id, -+ const struct cred *cred) -+{ -+ struct user_namespace *uns = seq_user_ns(m); -+ struct group_info *gi; -+ kernel_cap_t cap; -+ unsigned __capi; -+ int g; -+ -+ seq_printf(m, "%5d\n", id); -+ seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid)); -+ seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid)); -+ seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid)); -+ seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid)); -+ seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid)); -+ seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid)); -+ seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid)); -+ seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid)); -+ seq_puts(m, "\n\tGroups:\t"); -+ gi = cred->group_info; -+ for (g = 0; g < gi->ngroups; g++) { -+ seq_put_decimal_ull(m, g ? " " : "", -+ from_kgid_munged(uns, gi->gid[g])); -+ } -+ seq_puts(m, "\n\tCapEff:\t"); -+ cap = cred->cap_effective; -+ CAP_FOR_EACH_U32(__capi) -+ seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8); -+ seq_putc(m, '\n'); -+ return 0; -+} -+ -+static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) -+{ -+ struct io_sq_data *sq = NULL; -+ bool has_lock; -+ int i; -+ -+ /* -+ * Avoid ABBA deadlock between the seq lock and the io_uring mutex, -+ * since fdinfo case grabs it in the opposite direction of normal use -+ * cases. If we fail to get the lock, we just don't iterate any -+ * structures that could be going away outside the io_uring mutex. -+ */ -+ has_lock = mutex_trylock(&ctx->uring_lock); -+ -+ if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) { -+ sq = ctx->sq_data; -+ if (!sq->thread) -+ sq = NULL; -+ } -+ -+ seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1); -+ seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1); -+ seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files); -+ for (i = 0; has_lock && i < ctx->nr_user_files; i++) { -+ struct file *f = io_file_from_index(ctx, i); -+ -+ if (f) -+ seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname); -+ else -+ seq_printf(m, "%5u: \n", i); -+ } -+ seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs); -+ for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) { -+ struct io_mapped_ubuf *buf = ctx->user_bufs[i]; -+ unsigned int len = buf->ubuf_end - buf->ubuf; -+ -+ seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, len); -+ } -+ if (has_lock && !xa_empty(&ctx->personalities)) { -+ unsigned long index; -+ const struct cred *cred; -+ -+ seq_printf(m, "Personalities:\n"); -+ xa_for_each(&ctx->personalities, index, cred) -+ io_uring_show_cred(m, index, cred); -+ } -+ seq_printf(m, "PollList:\n"); -+ spin_lock(&ctx->completion_lock); -+ for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { -+ struct hlist_head *list = &ctx->cancel_hash[i]; -+ struct io_kiocb *req; -+ -+ hlist_for_each_entry(req, list, hash_node) -+ seq_printf(m, " op=%d, task_works=%d\n", req->opcode, -+ req->task->task_works != NULL); -+ } -+ spin_unlock(&ctx->completion_lock); -+ if (has_lock) -+ mutex_unlock(&ctx->uring_lock); -+} -+ -+static void io_uring_show_fdinfo(struct seq_file *m, struct file *f) -+{ -+ struct io_ring_ctx *ctx = f->private_data; -+ -+ if (percpu_ref_tryget(&ctx->refs)) { -+ __io_uring_show_fdinfo(ctx, m); -+ percpu_ref_put(&ctx->refs); -+ } -+} -+#endif -+ -+static const struct file_operations io_uring_fops = { -+ .release = io_uring_release, -+ .mmap = io_uring_mmap, -+#ifndef CONFIG_MMU -+ .get_unmapped_area = io_uring_nommu_get_unmapped_area, -+ .mmap_capabilities = io_uring_nommu_mmap_capabilities, -+#endif -+ .poll = io_uring_poll, -+#ifdef CONFIG_PROC_FS -+ .show_fdinfo = io_uring_show_fdinfo, -+#endif -+}; -+ -+static int io_allocate_scq_urings(struct io_ring_ctx *ctx, -+ struct io_uring_params *p) -+{ -+ struct io_rings *rings; -+ size_t size, sq_array_offset; -+ -+ /* make sure these are sane, as we already accounted them */ -+ ctx->sq_entries = p->sq_entries; -+ ctx->cq_entries = p->cq_entries; -+ -+ size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset); -+ if (size == SIZE_MAX) -+ return -EOVERFLOW; -+ -+ rings = io_mem_alloc(size); -+ if (!rings) -+ return -ENOMEM; -+ -+ ctx->rings = rings; -+ ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); -+ rings->sq_ring_mask = p->sq_entries - 1; -+ rings->cq_ring_mask = p->cq_entries - 1; -+ rings->sq_ring_entries = p->sq_entries; -+ rings->cq_ring_entries = p->cq_entries; -+ -+ size = array_size(sizeof(struct io_uring_sqe), p->sq_entries); -+ if (size == SIZE_MAX) { -+ io_mem_free(ctx->rings); -+ ctx->rings = NULL; -+ return -EOVERFLOW; -+ } -+ -+ ctx->sq_sqes = io_mem_alloc(size); -+ if (!ctx->sq_sqes) { -+ io_mem_free(ctx->rings); -+ ctx->rings = NULL; -+ return -ENOMEM; -+ } -+ -+ return 0; -+} -+ -+static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file) -+{ -+ int ret, fd; -+ -+ fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); -+ if (fd < 0) -+ return fd; -+ -+ ret = io_uring_add_tctx_node(ctx); -+ if (ret) { -+ put_unused_fd(fd); -+ return ret; -+ } -+ fd_install(fd, file); -+ return fd; -+} -+ -+/* -+ * Allocate an anonymous fd, this is what constitutes the application -+ * visible backing of an io_uring instance. The application mmaps this -+ * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled, -+ * we have to tie this fd to a socket for file garbage collection purposes. -+ */ -+static struct file *io_uring_get_file(struct io_ring_ctx *ctx) -+{ -+ struct file *file; -+#if defined(CONFIG_UNIX) -+ int ret; -+ -+ ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP, -+ &ctx->ring_sock); -+ if (ret) -+ return ERR_PTR(ret); -+#endif -+ -+ file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx, -+ O_RDWR | O_CLOEXEC); -+#if defined(CONFIG_UNIX) -+ if (IS_ERR(file)) { -+ sock_release(ctx->ring_sock); -+ ctx->ring_sock = NULL; -+ } else { -+ ctx->ring_sock->file = file; -+ } -+#endif -+ return file; -+} -+ -+static int io_uring_create(unsigned entries, struct io_uring_params *p, -+ struct io_uring_params __user *params) -+{ -+ struct io_ring_ctx *ctx; -+ struct file *file; -+ int ret; -+ -+ if (!entries) -+ return -EINVAL; -+ if (entries > IORING_MAX_ENTRIES) { -+ if (!(p->flags & IORING_SETUP_CLAMP)) -+ return -EINVAL; -+ entries = IORING_MAX_ENTRIES; -+ } -+ -+ /* -+ * Use twice as many entries for the CQ ring. It's possible for the -+ * application to drive a higher depth than the size of the SQ ring, -+ * since the sqes are only used at submission time. This allows for -+ * some flexibility in overcommitting a bit. If the application has -+ * set IORING_SETUP_CQSIZE, it will have passed in the desired number -+ * of CQ ring entries manually. -+ */ -+ p->sq_entries = roundup_pow_of_two(entries); -+ if (p->flags & IORING_SETUP_CQSIZE) { -+ /* -+ * If IORING_SETUP_CQSIZE is set, we do the same roundup -+ * to a power-of-two, if it isn't already. We do NOT impose -+ * any cq vs sq ring sizing. -+ */ -+ if (!p->cq_entries) -+ return -EINVAL; -+ if (p->cq_entries > IORING_MAX_CQ_ENTRIES) { -+ if (!(p->flags & IORING_SETUP_CLAMP)) -+ return -EINVAL; -+ p->cq_entries = IORING_MAX_CQ_ENTRIES; -+ } -+ p->cq_entries = roundup_pow_of_two(p->cq_entries); -+ if (p->cq_entries < p->sq_entries) -+ return -EINVAL; -+ } else { -+ p->cq_entries = 2 * p->sq_entries; -+ } -+ -+ ctx = io_ring_ctx_alloc(p); -+ if (!ctx) -+ return -ENOMEM; -+ ctx->compat = in_compat_syscall(); -+ if (!capable(CAP_IPC_LOCK)) -+ ctx->user = get_uid(current_user()); -+ -+ /* -+ * This is just grabbed for accounting purposes. When a process exits, -+ * the mm is exited and dropped before the files, hence we need to hang -+ * on to this mm purely for the purposes of being able to unaccount -+ * memory (locked/pinned vm). It's not used for anything else. -+ */ -+ mmgrab(current->mm); -+ ctx->mm_account = current->mm; -+ -+ ret = io_allocate_scq_urings(ctx, p); -+ if (ret) -+ goto err; -+ -+ ret = io_sq_offload_create(ctx, p); -+ if (ret) -+ goto err; -+ /* always set a rsrc node */ -+ ret = io_rsrc_node_switch_start(ctx); -+ if (ret) -+ goto err; -+ io_rsrc_node_switch(ctx, NULL); -+ -+ memset(&p->sq_off, 0, sizeof(p->sq_off)); -+ p->sq_off.head = offsetof(struct io_rings, sq.head); -+ p->sq_off.tail = offsetof(struct io_rings, sq.tail); -+ p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask); -+ p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries); -+ p->sq_off.flags = offsetof(struct io_rings, sq_flags); -+ p->sq_off.dropped = offsetof(struct io_rings, sq_dropped); -+ p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; -+ -+ memset(&p->cq_off, 0, sizeof(p->cq_off)); -+ p->cq_off.head = offsetof(struct io_rings, cq.head); -+ p->cq_off.tail = offsetof(struct io_rings, cq.tail); -+ p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask); -+ p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries); -+ p->cq_off.overflow = offsetof(struct io_rings, cq_overflow); -+ p->cq_off.cqes = offsetof(struct io_rings, cqes); -+ p->cq_off.flags = offsetof(struct io_rings, cq_flags); -+ -+ p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP | -+ IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS | -+ IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL | -+ IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED | -+ IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS | -+ IORING_FEAT_RSRC_TAGS; -+ -+ if (copy_to_user(params, p, sizeof(*p))) { -+ ret = -EFAULT; -+ goto err; -+ } -+ -+ file = io_uring_get_file(ctx); -+ if (IS_ERR(file)) { -+ ret = PTR_ERR(file); -+ goto err; -+ } -+ -+ /* -+ * Install ring fd as the very last thing, so we don't risk someone -+ * having closed it before we finish setup -+ */ -+ ret = io_uring_install_fd(ctx, file); -+ if (ret < 0) { -+ /* fput will clean it up */ -+ fput(file); -+ return ret; -+ } -+ -+ trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags); -+ return ret; -+err: -+ io_ring_ctx_wait_and_kill(ctx); -+ return ret; -+} -+ -+/* -+ * Sets up an aio uring context, and returns the fd. Applications asks for a -+ * ring size, we return the actual sq/cq ring sizes (among other things) in the -+ * params structure passed in. -+ */ -+static long io_uring_setup(u32 entries, struct io_uring_params __user *params) -+{ -+ struct io_uring_params p; -+ int i; -+ -+ if (copy_from_user(&p, params, sizeof(p))) -+ return -EFAULT; -+ for (i = 0; i < ARRAY_SIZE(p.resv); i++) { -+ if (p.resv[i]) -+ return -EINVAL; -+ } -+ -+ if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL | -+ IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE | -+ IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ | -+ IORING_SETUP_R_DISABLED)) -+ return -EINVAL; -+ -+ return io_uring_create(entries, &p, params); -+} -+ -+SYSCALL_DEFINE2(io_uring_setup, u32, entries, -+ struct io_uring_params __user *, params) -+{ -+ return io_uring_setup(entries, params); -+} -+ -+static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) -+{ -+ struct io_uring_probe *p; -+ size_t size; -+ int i, ret; -+ -+ size = struct_size(p, ops, nr_args); -+ if (size == SIZE_MAX) -+ return -EOVERFLOW; -+ p = kzalloc(size, GFP_KERNEL); -+ if (!p) -+ return -ENOMEM; -+ -+ ret = -EFAULT; -+ if (copy_from_user(p, arg, size)) -+ goto out; -+ ret = -EINVAL; -+ if (memchr_inv(p, 0, size)) -+ goto out; -+ -+ p->last_op = IORING_OP_LAST - 1; -+ if (nr_args > IORING_OP_LAST) -+ nr_args = IORING_OP_LAST; -+ -+ for (i = 0; i < nr_args; i++) { -+ p->ops[i].op = i; -+ if (!io_op_defs[i].not_supported) -+ p->ops[i].flags = IO_URING_OP_SUPPORTED; -+ } -+ p->ops_len = i; -+ -+ ret = 0; -+ if (copy_to_user(arg, p, size)) -+ ret = -EFAULT; -+out: -+ kfree(p); -+ return ret; -+} -+ -+static int io_register_personality(struct io_ring_ctx *ctx) -+{ -+ const struct cred *creds; -+ u32 id; -+ int ret; -+ -+ creds = get_current_cred(); -+ -+ ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds, -+ XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL); -+ if (ret < 0) { -+ put_cred(creds); -+ return ret; -+ } -+ return id; -+} -+ -+static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg, -+ unsigned int nr_args) -+{ -+ struct io_uring_restriction *res; -+ size_t size; -+ int i, ret; -+ -+ /* Restrictions allowed only if rings started disabled */ -+ if (!(ctx->flags & IORING_SETUP_R_DISABLED)) -+ return -EBADFD; -+ -+ /* We allow only a single restrictions registration */ -+ if (ctx->restrictions.registered) -+ return -EBUSY; -+ -+ if (!arg || nr_args > IORING_MAX_RESTRICTIONS) -+ return -EINVAL; -+ -+ size = array_size(nr_args, sizeof(*res)); -+ if (size == SIZE_MAX) -+ return -EOVERFLOW; -+ -+ res = memdup_user(arg, size); -+ if (IS_ERR(res)) -+ return PTR_ERR(res); -+ -+ ret = 0; -+ -+ for (i = 0; i < nr_args; i++) { -+ switch (res[i].opcode) { -+ case IORING_RESTRICTION_REGISTER_OP: -+ if (res[i].register_op >= IORING_REGISTER_LAST) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ __set_bit(res[i].register_op, -+ ctx->restrictions.register_op); -+ break; -+ case IORING_RESTRICTION_SQE_OP: -+ if (res[i].sqe_op >= IORING_OP_LAST) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op); -+ break; -+ case IORING_RESTRICTION_SQE_FLAGS_ALLOWED: -+ ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags; -+ break; -+ case IORING_RESTRICTION_SQE_FLAGS_REQUIRED: -+ ctx->restrictions.sqe_flags_required = res[i].sqe_flags; -+ break; -+ default: -+ ret = -EINVAL; -+ goto out; -+ } -+ } -+ -+out: -+ /* Reset all restrictions if an error happened */ -+ if (ret != 0) -+ memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); -+ else -+ ctx->restrictions.registered = true; -+ -+ kfree(res); -+ return ret; -+} -+ -+static int io_register_enable_rings(struct io_ring_ctx *ctx) -+{ -+ if (!(ctx->flags & IORING_SETUP_R_DISABLED)) -+ return -EBADFD; -+ -+ if (ctx->restrictions.registered) -+ ctx->restricted = 1; -+ -+ ctx->flags &= ~IORING_SETUP_R_DISABLED; -+ if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) -+ wake_up(&ctx->sq_data->wait); -+ return 0; -+} -+ -+static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, -+ struct io_uring_rsrc_update2 *up, -+ unsigned nr_args) -+{ -+ __u32 tmp; -+ int err; -+ -+ if (check_add_overflow(up->offset, nr_args, &tmp)) -+ return -EOVERFLOW; -+ err = io_rsrc_node_switch_start(ctx); -+ if (err) -+ return err; -+ -+ switch (type) { -+ case IORING_RSRC_FILE: -+ return __io_sqe_files_update(ctx, up, nr_args); -+ case IORING_RSRC_BUFFER: -+ return __io_sqe_buffers_update(ctx, up, nr_args); -+ } -+ return -EINVAL; -+} -+ -+static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg, -+ unsigned nr_args) -+{ -+ struct io_uring_rsrc_update2 up; -+ -+ if (!nr_args) -+ return -EINVAL; -+ memset(&up, 0, sizeof(up)); -+ if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update))) -+ return -EFAULT; -+ if (up.resv || up.resv2) -+ return -EINVAL; -+ return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args); -+} -+ -+static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, -+ unsigned size, unsigned type) -+{ -+ struct io_uring_rsrc_update2 up; -+ -+ if (size != sizeof(up)) -+ return -EINVAL; -+ if (copy_from_user(&up, arg, sizeof(up))) -+ return -EFAULT; -+ if (!up.nr || up.resv || up.resv2) -+ return -EINVAL; -+ return __io_register_rsrc_update(ctx, type, &up, up.nr); -+} -+ -+static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, -+ unsigned int size, unsigned int type) -+{ -+ struct io_uring_rsrc_register rr; -+ -+ /* keep it extendible */ -+ if (size != sizeof(rr)) -+ return -EINVAL; -+ -+ memset(&rr, 0, sizeof(rr)); -+ if (copy_from_user(&rr, arg, size)) -+ return -EFAULT; -+ if (!rr.nr || rr.resv || rr.resv2) -+ return -EINVAL; -+ -+ switch (type) { -+ case IORING_RSRC_FILE: -+ return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data), -+ rr.nr, u64_to_user_ptr(rr.tags)); -+ case IORING_RSRC_BUFFER: -+ return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data), -+ rr.nr, u64_to_user_ptr(rr.tags)); -+ } -+ return -EINVAL; -+} -+ -+static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg, -+ unsigned len) -+{ -+ struct io_uring_task *tctx = current->io_uring; -+ cpumask_var_t new_mask; -+ int ret; -+ -+ if (!tctx || !tctx->io_wq) -+ return -EINVAL; -+ -+ if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) -+ return -ENOMEM; -+ -+ cpumask_clear(new_mask); -+ if (len > cpumask_size()) -+ len = cpumask_size(); -+ -+#ifdef CONFIG_COMPAT -+ if (in_compat_syscall()) { -+ ret = compat_get_bitmap(cpumask_bits(new_mask), -+ (const compat_ulong_t __user *)arg, -+ len * 8 /* CHAR_BIT */); -+ } else { -+ ret = copy_from_user(new_mask, arg, len); -+ } -+#else -+ ret = copy_from_user(new_mask, arg, len); -+#endif -+ -+ if (ret) { -+ free_cpumask_var(new_mask); -+ return -EFAULT; -+ } -+ -+ ret = io_wq_cpu_affinity(tctx->io_wq, new_mask); -+ free_cpumask_var(new_mask); -+ return ret; -+} -+ -+static int io_unregister_iowq_aff(struct io_ring_ctx *ctx) -+{ -+ struct io_uring_task *tctx = current->io_uring; -+ -+ if (!tctx || !tctx->io_wq) -+ return -EINVAL; -+ -+ return io_wq_cpu_affinity(tctx->io_wq, NULL); -+} -+ -+static int io_register_iowq_max_workers(struct io_ring_ctx *ctx, -+ void __user *arg) -+ __must_hold(&ctx->uring_lock) -+{ -+ struct io_tctx_node *node; -+ struct io_uring_task *tctx = NULL; -+ struct io_sq_data *sqd = NULL; -+ __u32 new_count[2]; -+ int i, ret; -+ -+ if (copy_from_user(new_count, arg, sizeof(new_count))) -+ return -EFAULT; -+ for (i = 0; i < ARRAY_SIZE(new_count); i++) -+ if (new_count[i] > INT_MAX) -+ return -EINVAL; -+ -+ if (ctx->flags & IORING_SETUP_SQPOLL) { -+ sqd = ctx->sq_data; -+ if (sqd) { -+ /* -+ * Observe the correct sqd->lock -> ctx->uring_lock -+ * ordering. Fine to drop uring_lock here, we hold -+ * a ref to the ctx. -+ */ -+ refcount_inc(&sqd->refs); -+ mutex_unlock(&ctx->uring_lock); -+ mutex_lock(&sqd->lock); -+ mutex_lock(&ctx->uring_lock); -+ if (sqd->thread) -+ tctx = sqd->thread->io_uring; -+ } -+ } else { -+ tctx = current->io_uring; -+ } -+ -+ BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits)); -+ -+ for (i = 0; i < ARRAY_SIZE(new_count); i++) -+ if (new_count[i]) -+ ctx->iowq_limits[i] = new_count[i]; -+ ctx->iowq_limits_set = true; -+ -+ ret = -EINVAL; -+ if (tctx && tctx->io_wq) { -+ ret = io_wq_max_workers(tctx->io_wq, new_count); -+ if (ret) -+ goto err; -+ } else { -+ memset(new_count, 0, sizeof(new_count)); -+ } -+ -+ if (sqd) { -+ mutex_unlock(&sqd->lock); -+ io_put_sq_data(sqd); -+ } -+ -+ if (copy_to_user(arg, new_count, sizeof(new_count))) -+ return -EFAULT; -+ -+ /* that's it for SQPOLL, only the SQPOLL task creates requests */ -+ if (sqd) -+ return 0; -+ -+ /* now propagate the restriction to all registered users */ -+ list_for_each_entry(node, &ctx->tctx_list, ctx_node) { -+ struct io_uring_task *tctx = node->task->io_uring; -+ -+ if (WARN_ON_ONCE(!tctx->io_wq)) -+ continue; -+ -+ for (i = 0; i < ARRAY_SIZE(new_count); i++) -+ new_count[i] = ctx->iowq_limits[i]; -+ /* ignore errors, it always returns zero anyway */ -+ (void)io_wq_max_workers(tctx->io_wq, new_count); -+ } -+ return 0; -+err: -+ if (sqd) { -+ mutex_unlock(&sqd->lock); -+ io_put_sq_data(sqd); -+ } -+ return ret; -+} -+ -+static bool io_register_op_must_quiesce(int op) -+{ -+ switch (op) { -+ case IORING_REGISTER_BUFFERS: -+ case IORING_UNREGISTER_BUFFERS: -+ case IORING_REGISTER_FILES: -+ case IORING_UNREGISTER_FILES: -+ case IORING_REGISTER_FILES_UPDATE: -+ case IORING_REGISTER_PROBE: -+ case IORING_REGISTER_PERSONALITY: -+ case IORING_UNREGISTER_PERSONALITY: -+ case IORING_REGISTER_FILES2: -+ case IORING_REGISTER_FILES_UPDATE2: -+ case IORING_REGISTER_BUFFERS2: -+ case IORING_REGISTER_BUFFERS_UPDATE: -+ case IORING_REGISTER_IOWQ_AFF: -+ case IORING_UNREGISTER_IOWQ_AFF: -+ case IORING_REGISTER_IOWQ_MAX_WORKERS: -+ return false; -+ default: -+ return true; -+ } -+} -+ -+static int io_ctx_quiesce(struct io_ring_ctx *ctx) -+{ -+ long ret; -+ -+ percpu_ref_kill(&ctx->refs); -+ -+ /* -+ * Drop uring mutex before waiting for references to exit. If another -+ * thread is currently inside io_uring_enter() it might need to grab the -+ * uring_lock to make progress. If we hold it here across the drain -+ * wait, then we can deadlock. It's safe to drop the mutex here, since -+ * no new references will come in after we've killed the percpu ref. -+ */ -+ mutex_unlock(&ctx->uring_lock); -+ do { -+ ret = wait_for_completion_interruptible(&ctx->ref_comp); -+ if (!ret) -+ break; -+ ret = io_run_task_work_sig(); -+ } while (ret >= 0); -+ mutex_lock(&ctx->uring_lock); -+ -+ if (ret) -+ io_refs_resurrect(&ctx->refs, &ctx->ref_comp); -+ return ret; -+} -+ -+static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, -+ void __user *arg, unsigned nr_args) -+ __releases(ctx->uring_lock) -+ __acquires(ctx->uring_lock) -+{ -+ int ret; -+ -+ /* -+ * We're inside the ring mutex, if the ref is already dying, then -+ * someone else killed the ctx or is already going through -+ * io_uring_register(). -+ */ -+ if (percpu_ref_is_dying(&ctx->refs)) -+ return -ENXIO; -+ -+ if (ctx->restricted) { -+ if (opcode >= IORING_REGISTER_LAST) -+ return -EINVAL; -+ opcode = array_index_nospec(opcode, IORING_REGISTER_LAST); -+ if (!test_bit(opcode, ctx->restrictions.register_op)) -+ return -EACCES; -+ } -+ -+ if (io_register_op_must_quiesce(opcode)) { -+ ret = io_ctx_quiesce(ctx); -+ if (ret) -+ return ret; -+ } -+ -+ switch (opcode) { -+ case IORING_REGISTER_BUFFERS: -+ ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL); -+ break; -+ case IORING_UNREGISTER_BUFFERS: -+ ret = -EINVAL; -+ if (arg || nr_args) -+ break; -+ ret = io_sqe_buffers_unregister(ctx); -+ break; -+ case IORING_REGISTER_FILES: -+ ret = io_sqe_files_register(ctx, arg, nr_args, NULL); -+ break; -+ case IORING_UNREGISTER_FILES: -+ ret = -EINVAL; -+ if (arg || nr_args) -+ break; -+ ret = io_sqe_files_unregister(ctx); -+ break; -+ case IORING_REGISTER_FILES_UPDATE: -+ ret = io_register_files_update(ctx, arg, nr_args); -+ break; -+ case IORING_REGISTER_EVENTFD: -+ case IORING_REGISTER_EVENTFD_ASYNC: -+ ret = -EINVAL; -+ if (nr_args != 1) -+ break; -+ ret = io_eventfd_register(ctx, arg); -+ if (ret) -+ break; -+ if (opcode == IORING_REGISTER_EVENTFD_ASYNC) -+ ctx->eventfd_async = 1; -+ else -+ ctx->eventfd_async = 0; -+ break; -+ case IORING_UNREGISTER_EVENTFD: -+ ret = -EINVAL; -+ if (arg || nr_args) -+ break; -+ ret = io_eventfd_unregister(ctx); -+ break; -+ case IORING_REGISTER_PROBE: -+ ret = -EINVAL; -+ if (!arg || nr_args > 256) -+ break; -+ ret = io_probe(ctx, arg, nr_args); -+ break; -+ case IORING_REGISTER_PERSONALITY: -+ ret = -EINVAL; -+ if (arg || nr_args) -+ break; -+ ret = io_register_personality(ctx); -+ break; -+ case IORING_UNREGISTER_PERSONALITY: -+ ret = -EINVAL; -+ if (arg) -+ break; -+ ret = io_unregister_personality(ctx, nr_args); -+ break; -+ case IORING_REGISTER_ENABLE_RINGS: -+ ret = -EINVAL; -+ if (arg || nr_args) -+ break; -+ ret = io_register_enable_rings(ctx); -+ break; -+ case IORING_REGISTER_RESTRICTIONS: -+ ret = io_register_restrictions(ctx, arg, nr_args); -+ break; -+ case IORING_REGISTER_FILES2: -+ ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE); -+ break; -+ case IORING_REGISTER_FILES_UPDATE2: -+ ret = io_register_rsrc_update(ctx, arg, nr_args, -+ IORING_RSRC_FILE); -+ break; -+ case IORING_REGISTER_BUFFERS2: -+ ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER); -+ break; -+ case IORING_REGISTER_BUFFERS_UPDATE: -+ ret = io_register_rsrc_update(ctx, arg, nr_args, -+ IORING_RSRC_BUFFER); -+ break; -+ case IORING_REGISTER_IOWQ_AFF: -+ ret = -EINVAL; -+ if (!arg || !nr_args) -+ break; -+ ret = io_register_iowq_aff(ctx, arg, nr_args); -+ break; -+ case IORING_UNREGISTER_IOWQ_AFF: -+ ret = -EINVAL; -+ if (arg || nr_args) -+ break; -+ ret = io_unregister_iowq_aff(ctx); -+ break; -+ case IORING_REGISTER_IOWQ_MAX_WORKERS: -+ ret = -EINVAL; -+ if (!arg || nr_args != 2) -+ break; -+ ret = io_register_iowq_max_workers(ctx, arg); -+ break; -+ default: -+ ret = -EINVAL; -+ break; -+ } -+ -+ if (io_register_op_must_quiesce(opcode)) { -+ /* bring the ctx back to life */ -+ percpu_ref_reinit(&ctx->refs); -+ reinit_completion(&ctx->ref_comp); -+ } -+ return ret; -+} -+ -+SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, -+ void __user *, arg, unsigned int, nr_args) -+{ -+ struct io_ring_ctx *ctx; -+ long ret = -EBADF; -+ struct fd f; -+ -+ f = fdget(fd); -+ if (!f.file) -+ return -EBADF; -+ -+ ret = -EOPNOTSUPP; -+ if (f.file->f_op != &io_uring_fops) -+ goto out_fput; -+ -+ ctx = f.file->private_data; -+ -+ io_run_task_work(); -+ -+ mutex_lock(&ctx->uring_lock); -+ ret = __io_uring_register(ctx, opcode, arg, nr_args); -+ mutex_unlock(&ctx->uring_lock); -+ trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, -+ ctx->cq_ev_fd != NULL, ret); -+out_fput: -+ fdput(f); -+ return ret; -+} -+ -+static int __init io_uring_init(void) -+{ -+#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \ -+ BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \ -+ BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \ -+} while (0) -+ -+#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \ -+ __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename) -+ BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64); -+ BUILD_BUG_SQE_ELEM(0, __u8, opcode); -+ BUILD_BUG_SQE_ELEM(1, __u8, flags); -+ BUILD_BUG_SQE_ELEM(2, __u16, ioprio); -+ BUILD_BUG_SQE_ELEM(4, __s32, fd); -+ BUILD_BUG_SQE_ELEM(8, __u64, off); -+ BUILD_BUG_SQE_ELEM(8, __u64, addr2); -+ BUILD_BUG_SQE_ELEM(16, __u64, addr); -+ BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in); -+ BUILD_BUG_SQE_ELEM(24, __u32, len); -+ BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags); -+ BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags); -+ BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags); -+ BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags); -+ BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events); -+ BUILD_BUG_SQE_ELEM(28, __u32, poll32_events); -+ BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags); -+ BUILD_BUG_SQE_ELEM(28, __u32, msg_flags); -+ BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags); -+ BUILD_BUG_SQE_ELEM(28, __u32, accept_flags); -+ BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags); -+ BUILD_BUG_SQE_ELEM(28, __u32, open_flags); -+ BUILD_BUG_SQE_ELEM(28, __u32, statx_flags); -+ BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice); -+ BUILD_BUG_SQE_ELEM(28, __u32, splice_flags); -+ BUILD_BUG_SQE_ELEM(32, __u64, user_data); -+ BUILD_BUG_SQE_ELEM(40, __u16, buf_index); -+ BUILD_BUG_SQE_ELEM(40, __u16, buf_group); -+ BUILD_BUG_SQE_ELEM(42, __u16, personality); -+ BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in); -+ BUILD_BUG_SQE_ELEM(44, __u32, file_index); -+ -+ BUILD_BUG_ON(sizeof(struct io_uring_files_update) != -+ sizeof(struct io_uring_rsrc_update)); -+ BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) > -+ sizeof(struct io_uring_rsrc_update2)); -+ -+ /* ->buf_index is u16 */ -+ BUILD_BUG_ON(IORING_MAX_REG_BUFFERS >= (1u << 16)); -+ -+ /* should fit into one byte */ -+ BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8)); -+ -+ BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); -+ BUILD_BUG_ON(__REQ_F_LAST_BIT > 8 * sizeof(int)); -+ -+ req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC | -+ SLAB_ACCOUNT); -+ return 0; -+}; -+__initcall(io_uring_init); ---- a/kernel/exit.c -+++ b/kernel/exit.c -@@ -763,7 +763,7 @@ void __noreturn do_exit(long code) - schedule(); - } - -- io_uring_files_cancel(tsk->files); -+ io_uring_files_cancel(); - exit_signals(tsk); /* sets PF_EXITING */ - - /* sync mm's RSS info before statistics gathering */ ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -926,6 +926,7 @@ static struct task_struct *dup_task_stru - tsk->splice_pipe = NULL; - tsk->task_frag.page = NULL; - tsk->wake_q.next = NULL; -+ tsk->pf_io_worker = NULL; - - account_kernel_stack(tsk, 1); - ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -21,7 +21,7 @@ - #include - - #include "../workqueue_internal.h" --#include "../../fs/io-wq.h" -+#include "../../io_uring/io-wq.h" - #include "../smpboot.h" - - #include "pelt.h" diff --git a/next-5.10/io_uring-pass-in-epoll_uring_wake-for-eventfd-signaling-and-wakeups.patch b/next-5.10/io_uring-pass-in-epoll_uring_wake-for-eventfd-signaling-and-wakeups.patch deleted file mode 100644 index f651bebcea7..00000000000 --- a/next-5.10/io_uring-pass-in-epoll_uring_wake-for-eventfd-signaling-and-wakeups.patch +++ /dev/null @@ -1,82 +0,0 @@ -From 4029c28b8d6792778bdadc4e431be30302ff4a3b Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 23 Dec 2022 07:04:49 -0700 -Subject: io_uring: pass in EPOLL_URING_WAKE for eventfd signaling and wakeups - -From: Jens Axboe - -[ Upstream commit 4464853277d0ccdb9914608dd1332f0fa2f9846f ] - -Pass in EPOLL_URING_WAKE when signaling eventfd or doing poll related -wakups, so that we can check for a circular event dependency between -eventfd and epoll. If this flag is set when our wakeup handlers are -called, then we know we have a dependency that needs to terminate -multishot requests. - -eventfd and epoll are the only such possible dependencies. - -Cc: stable@vger.kernel.org # 6.0 -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - io_uring/io_uring.c | 27 ++++++++++++++++++++------- - 1 file changed, 20 insertions(+), 7 deletions(-) - ---- a/io_uring/io_uring.c -+++ b/io_uring/io_uring.c -@@ -1626,13 +1626,15 @@ static void io_cqring_ev_posted(struct i - * wake as many waiters as we need to. - */ - if (wq_has_sleeper(&ctx->cq_wait)) -- wake_up_all(&ctx->cq_wait); -+ __wake_up(&ctx->cq_wait, TASK_NORMAL, 0, -+ poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); - if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait)) - wake_up(&ctx->sq_data->wait); - if (io_should_trigger_evfd(ctx)) -- eventfd_signal(ctx->cq_ev_fd, 1); -+ eventfd_signal_mask(ctx->cq_ev_fd, 1, EPOLL_URING_WAKE); - if (waitqueue_active(&ctx->poll_wait)) -- wake_up_interruptible(&ctx->poll_wait); -+ __wake_up(&ctx->poll_wait, TASK_INTERRUPTIBLE, 0, -+ poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); - } - - static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) -@@ -1642,12 +1644,14 @@ static void io_cqring_ev_posted_iopoll(s - - if (ctx->flags & IORING_SETUP_SQPOLL) { - if (waitqueue_active(&ctx->cq_wait)) -- wake_up_all(&ctx->cq_wait); -+ __wake_up(&ctx->cq_wait, TASK_NORMAL, 0, -+ poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); - } - if (io_should_trigger_evfd(ctx)) -- eventfd_signal(ctx->cq_ev_fd, 1); -+ eventfd_signal_mask(ctx->cq_ev_fd, 1, EPOLL_URING_WAKE); - if (waitqueue_active(&ctx->poll_wait)) -- wake_up_interruptible(&ctx->poll_wait); -+ __wake_up(&ctx->poll_wait, TASK_INTERRUPTIBLE, 0, -+ poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); - } - - /* Returns true if there are no backlogged entries after the flush */ -@@ -5477,8 +5481,17 @@ static int io_poll_wake(struct wait_queu - if (mask && !(mask & poll->events)) - return 0; - -- if (io_poll_get_ownership(req)) -+ if (io_poll_get_ownership(req)) { -+ /* -+ * If we trigger a multishot poll off our own wakeup path, -+ * disable multishot as there is a circular dependency between -+ * CQ posting and triggering the event. -+ */ -+ if (mask & EPOLL_URING_WAKE) -+ poll->events |= EPOLLONESHOT; -+ - __io_poll_execute(req, mask); -+ } - return 1; - } - diff --git a/next-5.10/iov_iter-add-helper-to-save-iov_iter-state.patch b/next-5.10/iov_iter-add-helper-to-save-iov_iter-state.patch deleted file mode 100644 index f5d5927a7dc..00000000000 --- a/next-5.10/iov_iter-add-helper-to-save-iov_iter-state.patch +++ /dev/null @@ -1,131 +0,0 @@ -From 4d95aa07b6f1059990366f88112c48f217fbbc5d Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 10 Sep 2021 11:18:36 -0600 -Subject: iov_iter: add helper to save iov_iter state - -From: Jens Axboe - -[ Upstream commit 8fb0f47a9d7acf620d0fd97831b69da9bc5e22ed ] - -In an ideal world, when someone is passed an iov_iter and returns X bytes, -then X bytes would have been consumed/advanced from the iov_iter. But we -have use cases that always consume the entire iterator, a few examples -of that are iomap and bdev O_DIRECT. This means we cannot rely on the -state of the iov_iter once we've called ->read_iter() or ->write_iter(). - -This would be easier if we didn't always have to deal with truncate of -the iov_iter, as rewinding would be trivial without that. We recently -added a commit to track the truncate state, but that grew the iov_iter -by 8 bytes and wasn't the best solution. - -Implement a helper to save enough of the iov_iter state to sanely restore -it after we've called the read/write iterator helpers. This currently -only works for IOVEC/BVEC/KVEC as that's all we need, support for other -iterator types are left as an exercise for the reader. - -Link: https://lore.kernel.org/linux-fsdevel/CAHk-=wiacKV4Gh-MYjteU0LwNBSGpWrK-Ov25HdqB1ewinrFPg@mail.gmail.com/ -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - include/linux/uio.h | 15 +++++++++++++++ - lib/iov_iter.c | 52 +++++++++++++++++++++++++++++++++------------------- - 2 files changed, 48 insertions(+), 19 deletions(-) - ---- a/include/linux/uio.h -+++ b/include/linux/uio.h -@@ -26,6 +26,12 @@ enum iter_type { - ITER_DISCARD = 64, - }; - -+struct iov_iter_state { -+ size_t iov_offset; -+ size_t count; -+ unsigned long nr_segs; -+}; -+ - struct iov_iter { - /* - * Bit 0 is the read/write bit, set if we're writing. -@@ -55,6 +61,14 @@ static inline enum iter_type iov_iter_ty - return i->type & ~(READ | WRITE); - } - -+static inline void iov_iter_save_state(struct iov_iter *iter, -+ struct iov_iter_state *state) -+{ -+ state->iov_offset = iter->iov_offset; -+ state->count = iter->count; -+ state->nr_segs = iter->nr_segs; -+} -+ - static inline bool iter_is_iovec(const struct iov_iter *i) - { - return iov_iter_type(i) == ITER_IOVEC; -@@ -226,6 +240,7 @@ ssize_t iov_iter_get_pages(struct iov_it - ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, struct page ***pages, - size_t maxsize, size_t *start); - int iov_iter_npages(const struct iov_iter *i, int maxpages); -+void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state); - - const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags); - ---- a/lib/iov_iter.c -+++ b/lib/iov_iter.c -@@ -1836,24 +1836,38 @@ int import_single_range(int rw, void __u - } - EXPORT_SYMBOL(import_single_range); - --int iov_iter_for_each_range(struct iov_iter *i, size_t bytes, -- int (*f)(struct kvec *vec, void *context), -- void *context) -+/** -+ * iov_iter_restore() - Restore a &struct iov_iter to the same state as when -+ * iov_iter_save_state() was called. -+ * -+ * @i: &struct iov_iter to restore -+ * @state: state to restore from -+ * -+ * Used after iov_iter_save_state() to bring restore @i, if operations may -+ * have advanced it. -+ * -+ * Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC -+ */ -+void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) - { -- struct kvec w; -- int err = -EINVAL; -- if (!bytes) -- return 0; -- -- iterate_all_kinds(i, bytes, v, -EINVAL, ({ -- w.iov_base = kmap(v.bv_page) + v.bv_offset; -- w.iov_len = v.bv_len; -- err = f(&w, context); -- kunmap(v.bv_page); -- err;}), ({ -- w = v; -- err = f(&w, context);}) -- ) -- return err; -+ if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) && -+ !iov_iter_is_kvec(i)) -+ return; -+ i->iov_offset = state->iov_offset; -+ i->count = state->count; -+ /* -+ * For the *vec iters, nr_segs + iov is constant - if we increment -+ * the vec, then we also decrement the nr_segs count. Hence we don't -+ * need to track both of these, just one is enough and we can deduct -+ * the other from that. ITER_KVEC and ITER_IOVEC are the same struct -+ * size, so we can just increment the iov pointer as they are unionzed. -+ * ITER_BVEC _may_ be the same size on some archs, but on others it is -+ * not. Be safe and handle it separately. -+ */ -+ BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec)); -+ if (iov_iter_is_bvec(i)) -+ i->bvec -= state->nr_segs - i->nr_segs; -+ else -+ i->iov -= state->nr_segs - i->nr_segs; -+ i->nr_segs = state->nr_segs; - } --EXPORT_SYMBOL(iov_iter_for_each_range); diff --git a/next-5.10/kernel-allow-fork-with-tif_notify_signal-pending.patch b/next-5.10/kernel-allow-fork-with-tif_notify_signal-pending.patch deleted file mode 100644 index 894afb5d84f..00000000000 --- a/next-5.10/kernel-allow-fork-with-tif_notify_signal-pending.patch +++ /dev/null @@ -1,38 +0,0 @@ -From b6ea49a1293ab63ef65553c576b8d919bdd3d42e Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Mon, 22 Mar 2021 09:39:12 -0600 -Subject: kernel: allow fork with TIF_NOTIFY_SIGNAL pending - -From: Jens Axboe - -[ Upstream commit 66ae0d1e2d9fe6ec70e73fcfdcf4b390e271c1ac ] - -fork() fails if signal_pending() is true, but there are two conditions -that can lead to that: - -1) An actual signal is pending. We want fork to fail for that one, like - we always have. - -2) TIF_NOTIFY_SIGNAL is pending, because the task has pending task_work. - We don't need to make it fail for that case. - -Allow fork() to proceed if just task_work is pending, by changing the -signal_pending() check to task_sigpending(). - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - kernel/fork.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -1942,7 +1942,7 @@ static __latent_entropy struct task_stru - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - retval = -ERESTARTNOINTR; -- if (signal_pending(current)) -+ if (task_sigpending(current)) - goto fork_out; - - retval = -ENOMEM; diff --git a/next-5.10/kernel-don-t-call-do_exit-for-pf_io_worker-threads.patch b/next-5.10/kernel-don-t-call-do_exit-for-pf_io_worker-threads.patch deleted file mode 100644 index 56ab9362794..00000000000 --- a/next-5.10/kernel-don-t-call-do_exit-for-pf_io_worker-threads.patch +++ /dev/null @@ -1,46 +0,0 @@ -From e11f9f61752f252ee19e9c0dee22218f9078a75b Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 26 Mar 2021 08:57:10 -0600 -Subject: kernel: don't call do_exit() for PF_IO_WORKER threads - -From: Jens Axboe - -[ Upstream commit 10442994ba195efef6fdcc0c3699e4633cb5161b ] - -Right now we're never calling get_signal() from PF_IO_WORKER threads, but -in preparation for doing so, don't handle a fatal signal for them. The -workers have state they need to cleanup when exiting, so just return -instead of calling do_exit() on their behalf. The threads themselves will -detect a fatal signal and do proper shutdown. - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - kernel/signal.c | 10 +++++++++- - 1 file changed, 9 insertions(+), 1 deletion(-) - ---- a/kernel/signal.c -+++ b/kernel/signal.c -@@ -2755,13 +2755,21 @@ relock: - } - - /* -+ * PF_IO_WORKER threads will catch and exit on fatal signals -+ * themselves. They have cleanup that must be performed, so -+ * we cannot call do_exit() on their behalf. -+ */ -+ if (current->flags & PF_IO_WORKER) -+ goto out; -+ -+ /* - * Death signals, no core dump. - */ - do_group_exit(ksig->info.si_signo); - /* NOTREACHED */ - } - spin_unlock_irq(&sighand->siglock); -- -+out: - ksig->sig = signr; - return ksig->sig > 0; - } diff --git a/next-5.10/kernel-provide-create_io_thread-helper.patch b/next-5.10/kernel-provide-create_io_thread-helper.patch deleted file mode 100644 index bfb70a9300b..00000000000 --- a/next-5.10/kernel-provide-create_io_thread-helper.patch +++ /dev/null @@ -1,87 +0,0 @@ -From a3db03dc7a4f33de20188638d5b193c2946c5dfe Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Thu, 4 Mar 2021 12:21:05 -0700 -Subject: kernel: provide create_io_thread() helper - -From: Jens Axboe - -[ Upstream commit cc440e8738e5c875297ac0e90316745093be7e28 ] - -Provide a generic helper for setting up an io_uring worker. Returns a -task_struct so that the caller can do whatever setup is needed, then call -wake_up_new_task() to kick it into gear. - -Add a kernel_clone_args member, io_thread, which tells copy_process() to -mark the task with PF_IO_WORKER. - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - include/linux/sched/task.h | 2 ++ - kernel/fork.c | 30 ++++++++++++++++++++++++++++++ - 2 files changed, 32 insertions(+) - ---- a/include/linux/sched/task.h -+++ b/include/linux/sched/task.h -@@ -31,6 +31,7 @@ struct kernel_clone_args { - /* Number of elements in *set_tid */ - size_t set_tid_size; - int cgroup; -+ int io_thread; - struct cgroup *cgrp; - struct css_set *cset; - }; -@@ -85,6 +86,7 @@ extern void exit_files(struct task_struc - extern void exit_itimers(struct task_struct *); - - extern pid_t kernel_clone(struct kernel_clone_args *kargs); -+struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node); - struct task_struct *fork_idle(int); - struct mm_struct *copy_init_mm(void); - extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -1949,6 +1949,8 @@ static __latent_entropy struct task_stru - p = dup_task_struct(current, node); - if (!p) - goto fork_out; -+ if (args->io_thread) -+ p->flags |= PF_IO_WORKER; - - /* - * This _must_ happen before we call free_task(), i.e. before we jump -@@ -2416,6 +2418,34 @@ struct mm_struct *copy_init_mm(void) - } - - /* -+ * This is like kernel_clone(), but shaved down and tailored to just -+ * creating io_uring workers. It returns a created task, or an error pointer. -+ * The returned task is inactive, and the caller must fire it up through -+ * wake_up_new_task(p). All signals are blocked in the created task. -+ */ -+struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) -+{ -+ unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD| -+ CLONE_IO; -+ struct kernel_clone_args args = { -+ .flags = ((lower_32_bits(flags) | CLONE_VM | -+ CLONE_UNTRACED) & ~CSIGNAL), -+ .exit_signal = (lower_32_bits(flags) & CSIGNAL), -+ .stack = (unsigned long)fn, -+ .stack_size = (unsigned long)arg, -+ .io_thread = 1, -+ }; -+ struct task_struct *tsk; -+ -+ tsk = copy_process(NULL, 0, node, &args); -+ if (!IS_ERR(tsk)) { -+ sigfillset(&tsk->blocked); -+ sigdelsetmask(&tsk->blocked, sigmask(SIGKILL)); -+ } -+ return tsk; -+} -+ -+/* - * Ok, this is the main fork-routine. - * - * It copies the process, and if successful kick-starts diff --git a/next-5.10/kernel-remove-checking-for-tif_notify_signal.patch b/next-5.10/kernel-remove-checking-for-tif_notify_signal.patch deleted file mode 100644 index 0fe2ab88679..00000000000 --- a/next-5.10/kernel-remove-checking-for-tif_notify_signal.patch +++ /dev/null @@ -1,95 +0,0 @@ -From 16021acf95cec0ada90dae2b8409ac234c678f3b Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 16:04:39 -0600 -Subject: kernel: remove checking for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit e296dc4996b8094ccde45d19090d804c4103513e ] - -It's available everywhere now, no need to check or add dummy defines. - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - include/linux/entry-common.h | 4 ---- - include/linux/sched/signal.h | 2 -- - include/linux/tracehook.h | 4 ---- - kernel/signal.c | 2 -- - 4 files changed, 12 deletions(-) - ---- a/include/linux/entry-common.h -+++ b/include/linux/entry-common.h -@@ -37,10 +37,6 @@ - # define _TIF_UPROBE (0) - #endif - --#ifndef _TIF_NOTIFY_SIGNAL --# define _TIF_NOTIFY_SIGNAL (0) --#endif -- - /* - * TIF flags handled in syscall_enter_from_user_mode() - */ ---- a/include/linux/sched/signal.h -+++ b/include/linux/sched/signal.h -@@ -361,7 +361,6 @@ static inline int task_sigpending(struct - - static inline int signal_pending(struct task_struct *p) - { --#if defined(TIF_NOTIFY_SIGNAL) - /* - * TIF_NOTIFY_SIGNAL isn't really a signal, but it requires the same - * behavior in terms of ensuring that we break out of wait loops -@@ -369,7 +368,6 @@ static inline int signal_pending(struct - */ - if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL))) - return 1; --#endif - return task_sigpending(p); - } - ---- a/include/linux/tracehook.h -+++ b/include/linux/tracehook.h -@@ -205,12 +205,10 @@ static inline void tracehook_notify_resu - */ - static inline void tracehook_notify_signal(void) - { --#if defined(TIF_NOTIFY_SIGNAL) - clear_thread_flag(TIF_NOTIFY_SIGNAL); - smp_mb__after_atomic(); - if (current->task_works) - task_work_run(); --#endif - } - - /* -@@ -218,11 +216,9 @@ static inline void tracehook_notify_sign - */ - static inline void set_notify_signal(struct task_struct *task) - { --#if defined(TIF_NOTIFY_SIGNAL) - if (!test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) && - !wake_up_state(task, TASK_INTERRUPTIBLE)) - kick_process(task); --#endif - } - - #endif /* */ ---- a/kernel/signal.c -+++ b/kernel/signal.c -@@ -2525,14 +2525,12 @@ bool get_signal(struct ksignal *ksig) - * that the arch handlers don't all have to do it. If we get here - * without TIF_SIGPENDING, just exit after running signal work. - */ --#ifdef TIF_NOTIFY_SIGNAL - if (!IS_ENABLED(CONFIG_GENERIC_ENTRY)) { - if (test_thread_flag(TIF_NOTIFY_SIGNAL)) - tracehook_notify_signal(); - if (!task_sigpending(current)) - return false; - } --#endif - - if (unlikely(uprobe_deny_signal())) - return false; diff --git a/next-5.10/kernel-stop-masking-signals-in-create_io_thread.patch b/next-5.10/kernel-stop-masking-signals-in-create_io_thread.patch deleted file mode 100644 index 27d9661c8e2..00000000000 --- a/next-5.10/kernel-stop-masking-signals-in-create_io_thread.patch +++ /dev/null @@ -1,56 +0,0 @@ -From 3ba2de08db37474986cabdd4e675b8e3bfc5df5a Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 26 Mar 2021 09:05:22 -0600 -Subject: kernel: stop masking signals in create_io_thread() - -From: Jens Axboe - -[ Upstream commit b16b3855d89fba640996fefdd3a113c0aa0e380d ] - -This is racy - move the blocking into when the task is created and -we're marking it as PF_IO_WORKER anyway. The IO threads are now -prepared to handle signals like SIGSTOP as well, so clear that from -the mask to allow proper stopping of IO threads. - -Acked-by: "Eric W. Biederman" -Reported-by: Oleg Nesterov -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - kernel/fork.c | 16 ++++++++-------- - 1 file changed, 8 insertions(+), 8 deletions(-) - ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -1949,8 +1949,14 @@ static __latent_entropy struct task_stru - p = dup_task_struct(current, node); - if (!p) - goto fork_out; -- if (args->io_thread) -+ if (args->io_thread) { -+ /* -+ * Mark us an IO worker, and block any signal that isn't -+ * fatal or STOP -+ */ - p->flags |= PF_IO_WORKER; -+ siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP)); -+ } - - /* - * This _must_ happen before we call free_task(), i.e. before we jump -@@ -2435,14 +2441,8 @@ struct task_struct *create_io_thread(int - .stack_size = (unsigned long)arg, - .io_thread = 1, - }; -- struct task_struct *tsk; - -- tsk = copy_process(NULL, 0, node, &args); -- if (!IS_ERR(tsk)) { -- sigfillset(&tsk->blocked); -- sigdelsetmask(&tsk->blocked, sigmask(SIGKILL)); -- } -- return tsk; -+ return copy_process(NULL, 0, node, &args); - } - - /* diff --git a/next-5.10/m68k-add-support-for-tif_notify_signal.patch b/next-5.10/m68k-add-support-for-tif_notify_signal.patch deleted file mode 100644 index fcd9445d222..00000000000 --- a/next-5.10/m68k-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,42 +0,0 @@ -From da81a523c0a6278891e020596edc7a32cb593922 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 15:10:55 -0600 -Subject: m68k: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit e660653cd9f2df470d156c249631f68b9dee51ee ] - -Wire up TIF_NOTIFY_SIGNAL handling for m68k. - -Cc: linux-m68k@lists.linux-m68k.org -Acked-by: Geert Uytterhoeven -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/m68k/include/asm/thread_info.h | 1 + - arch/m68k/kernel/signal.c | 3 ++- - 2 files changed, 3 insertions(+), 1 deletion(-) - ---- a/arch/m68k/include/asm/thread_info.h -+++ b/arch/m68k/include/asm/thread_info.h -@@ -60,6 +60,7 @@ static inline struct thread_info *curren - * bits 0-7 are tested at every exception exit - * bits 8-15 are also tested at syscall exit - */ -+#define TIF_NOTIFY_SIGNAL 4 - #define TIF_NOTIFY_RESUME 5 /* callback before returning to user */ - #define TIF_SIGPENDING 6 /* signal pending */ - #define TIF_NEED_RESCHED 7 /* rescheduling necessary */ ---- a/arch/m68k/kernel/signal.c -+++ b/arch/m68k/kernel/signal.c -@@ -1129,7 +1129,8 @@ static void do_signal(struct pt_regs *re - - void do_notify_resume(struct pt_regs *regs) - { -- if (test_thread_flag(TIF_SIGPENDING)) -+ if (test_thread_flag(TIF_NOTIFY_SIGNAL) || -+ test_thread_flag(TIF_SIGPENDING)) - do_signal(regs); - - if (test_thread_flag(TIF_NOTIFY_RESUME)) diff --git a/next-5.10/make-sure-nd-path.mnt-and-nd-path.dentry-are-always-valid-pointers.patch b/next-5.10/make-sure-nd-path.mnt-and-nd-path.dentry-are-always-valid-pointers.patch deleted file mode 100644 index e75795ad5c9..00000000000 --- a/next-5.10/make-sure-nd-path.mnt-and-nd-path.dentry-are-always-valid-pointers.patch +++ /dev/null @@ -1,59 +0,0 @@ -From 597ed7633bde8abc24c0cea5fbbfd824200d3251 Mon Sep 17 00:00:00 2001 -From: Al Viro -Date: Tue, 6 Apr 2021 12:33:07 -0400 -Subject: Make sure nd->path.mnt and nd->path.dentry are always valid pointers - -From: Al Viro - -[ Upstream commit 7d01ef7585c07afaf487759a48486228cd065726 ] - -Initialize them in set_nameidata() and make sure that terminate_walk() clears them -once the pointers become potentially invalid (i.e. we leave RCU mode or drop them -in non-RCU one). Currently we have "path_init() always initializes them and nobody -accesses them outside of path_init()/terminate_walk() segments", which is asking -for trouble. - -With that change we would have nd->path.{mnt,dentry} - 1) always valid - NULL or pointing to currently allocated objects. - 2) non-NULL while we are successfully walking - 3) NULL when we are not walking at all - 4) contributing to refcounts whenever non-NULL outside of RCU mode. - -Fixes: 6c6ec2b0a3e0 ("fs: add support for LOOKUP_CACHED") -Reported-by: syzbot+c88a7030da47945a3cc3@syzkaller.appspotmail.com -Tested-by: Christian Brauner -Signed-off-by: Al Viro -Signed-off-by: Greg Kroah-Hartman ---- - fs/namei.c | 6 ++++-- - 1 file changed, 4 insertions(+), 2 deletions(-) - ---- a/fs/namei.c -+++ b/fs/namei.c -@@ -529,6 +529,8 @@ static void set_nameidata(struct nameida - p->stack = p->internal; - p->dfd = dfd; - p->name = name; -+ p->path.mnt = NULL; -+ p->path.dentry = NULL; - p->total_link_count = old ? old->total_link_count : 0; - p->saved = old; - current->nameidata = p; -@@ -602,6 +604,8 @@ static void terminate_walk(struct nameid - rcu_read_unlock(); - } - nd->depth = 0; -+ nd->path.mnt = NULL; -+ nd->path.dentry = NULL; - } - - /* path_put is needed afterwards regardless of success or failure */ -@@ -2243,8 +2247,6 @@ static const char *path_init(struct name - } - - nd->root.mnt = NULL; -- nd->path.mnt = NULL; -- nd->path.dentry = NULL; - - /* Absolute pathname -- fetch the root (LOOKUP_IN_ROOT uses nd->dfd). */ - if (*s == '/' && !(flags & LOOKUP_IN_ROOT)) { diff --git a/next-5.10/microblaze-add-support-for-tif_notify_signal.patch b/next-5.10/microblaze-add-support-for-tif_notify_signal.patch deleted file mode 100644 index d6d09495221..00000000000 --- a/next-5.10/microblaze-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,49 +0,0 @@ -From ab74702d525c52cbb5dea9fa5169f3f6f5eae2d7 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 15:13:57 -0600 -Subject: microblaze: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit f4ea089e429e0d366cd1a34a2cbe3c7b13d98d75 ] - -Wire up TIF_NOTIFY_SIGNAL handling for microblaze. - -Acked-by: Michal Simek -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/microblaze/include/asm/thread_info.h | 2 ++ - arch/microblaze/kernel/signal.c | 3 ++- - 2 files changed, 4 insertions(+), 1 deletion(-) - ---- a/arch/microblaze/include/asm/thread_info.h -+++ b/arch/microblaze/include/asm/thread_info.h -@@ -107,6 +107,7 @@ static inline struct thread_info *curren - #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ - /* restore singlestep on return to user mode */ - #define TIF_SINGLESTEP 4 -+#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ - #define TIF_MEMDIE 6 /* is terminating due to OOM killer */ - #define TIF_SYSCALL_AUDIT 9 /* syscall auditing active */ - #define TIF_SECCOMP 10 /* secure computing */ -@@ -119,6 +120,7 @@ static inline struct thread_info *curren - #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) - #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) - #define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) - #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) - #define _TIF_SECCOMP (1 << TIF_SECCOMP) ---- a/arch/microblaze/kernel/signal.c -+++ b/arch/microblaze/kernel/signal.c -@@ -313,7 +313,8 @@ static void do_signal(struct pt_regs *re - - asmlinkage void do_notify_resume(struct pt_regs *regs, int in_syscall) - { -- if (test_thread_flag(TIF_SIGPENDING)) -+ if (test_thread_flag(TIF_SIGPENDING) || -+ test_thread_flag(TIF_NOTIFY_SIGNAL)) - do_signal(regs, in_syscall); - - if (test_thread_flag(TIF_NOTIFY_RESUME)) diff --git a/next-5.10/mips-add-support-for-tif_notify_signal.patch b/next-5.10/mips-add-support-for-tif_notify_signal.patch deleted file mode 100644 index 65f197fb496..00000000000 --- a/next-5.10/mips-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,58 +0,0 @@ -From 60559f867bc926e5595c13e4b85ea2fea168ecf8 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 15:16:02 -0600 -Subject: mips: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit f45c184bce15f4a314c0210519bc3b4aab408838 ] - -Wire up TIF_NOTIFY_SIGNAL handling for mips. - -Cc: linux-mips@vger.kernel.org -Acked-By: Thomas Bogendoerfer -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/mips/include/asm/thread_info.h | 4 +++- - arch/mips/kernel/signal.c | 2 +- - 2 files changed, 4 insertions(+), 2 deletions(-) - ---- a/arch/mips/include/asm/thread_info.h -+++ b/arch/mips/include/asm/thread_info.h -@@ -115,6 +115,7 @@ static inline struct thread_info *curren - #define TIF_SECCOMP 4 /* secure computing */ - #define TIF_NOTIFY_RESUME 5 /* callback before returning to user */ - #define TIF_UPROBE 6 /* breakpointed or singlestepping */ -+#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ - #define TIF_RESTORE_SIGMASK 9 /* restore signal mask in do_signal() */ - #define TIF_USEDFPU 16 /* FPU was used by this task this quantum (SMP) */ - #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ -@@ -139,6 +140,7 @@ static inline struct thread_info *curren - #define _TIF_SECCOMP (1< -Date: Fri, 9 Oct 2020 15:18:43 -0600 -Subject: nds32: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit b13e8bf615fe26fb6a6dfe1b5a1c65e1624dfee2 ] - -Wire up TIF_NOTIFY_SIGNAL handling for nds32. - -Cc: Nick Hu -Cc: Greentime Hu -Cc: Vincent Chen -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/nds32/include/asm/thread_info.h | 2 ++ - arch/nds32/kernel/ex-exit.S | 2 +- - arch/nds32/kernel/signal.c | 2 +- - 3 files changed, 4 insertions(+), 2 deletions(-) - ---- a/arch/nds32/include/asm/thread_info.h -+++ b/arch/nds32/include/asm/thread_info.h -@@ -48,6 +48,7 @@ struct thread_info { - #define TIF_NEED_RESCHED 2 - #define TIF_SINGLESTEP 3 - #define TIF_NOTIFY_RESUME 4 /* callback before returning to user */ -+#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ - #define TIF_SYSCALL_TRACE 8 - #define TIF_POLLING_NRFLAG 17 - #define TIF_MEMDIE 18 -@@ -57,6 +58,7 @@ struct thread_info { - #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) - #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) - #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - #define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) - #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) - #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) ---- a/arch/nds32/kernel/ex-exit.S -+++ b/arch/nds32/kernel/ex-exit.S -@@ -120,7 +120,7 @@ work_pending: - andi $p1, $r1, #_TIF_NEED_RESCHED - bnez $p1, work_resched - -- andi $p1, $r1, #_TIF_SIGPENDING|#_TIF_NOTIFY_RESUME -+ andi $p1, $r1, #_TIF_SIGPENDING|#_TIF_NOTIFY_RESUME|#_TIF_NOTIFY_SIGNAL - beqz $p1, no_work_pending - - move $r0, $sp ! 'regs' ---- a/arch/nds32/kernel/signal.c -+++ b/arch/nds32/kernel/signal.c -@@ -376,7 +376,7 @@ static void do_signal(struct pt_regs *re - asmlinkage void - do_notify_resume(struct pt_regs *regs, unsigned int thread_flags) - { -- if (thread_flags & _TIF_SIGPENDING) -+ if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) - do_signal(regs); - - if (thread_flags & _TIF_NOTIFY_RESUME) diff --git a/next-5.10/net-add-accept-helper-not-installing-fd.patch b/next-5.10/net-add-accept-helper-not-installing-fd.patch deleted file mode 100644 index 898eae8e5ff..00000000000 --- a/next-5.10/net-add-accept-helper-not-installing-fd.patch +++ /dev/null @@ -1,142 +0,0 @@ -From ab96e6863470dd3f13f9a894455b802cde9bcefc Mon Sep 17 00:00:00 2001 -From: Pavel Begunkov -Date: Wed, 25 Aug 2021 12:25:44 +0100 -Subject: net: add accept helper not installing fd - -From: Pavel Begunkov - -[ Upstream commit d32f89da7fa8ccc8b3fb8f909d61e42b9bc39329 ] - -Introduce and reuse a helper that acts similarly to __sys_accept4_file() -but returns struct file instead of installing file descriptor. Will be -used by io_uring. - -Signed-off-by: Pavel Begunkov -Acked-by: Jakub Kicinski -Signed-off-by: Jens Axboe -Acked-by: David S. Miller -Link: https://lore.kernel.org/r/c57b9e8e818d93683a3d24f8ca50ca038d1da8c4.1629888991.git.asml.silence@gmail.com -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - include/linux/socket.h | 3 ++ - net/socket.c | 67 ++++++++++++++++++++++++++----------------------- - 2 files changed, 39 insertions(+), 31 deletions(-) - ---- a/include/linux/socket.h -+++ b/include/linux/socket.h -@@ -421,6 +421,9 @@ extern int __sys_accept4_file(struct fil - struct sockaddr __user *upeer_sockaddr, - int __user *upeer_addrlen, int flags, - unsigned long nofile); -+extern struct file *do_accept(struct file *file, unsigned file_flags, -+ struct sockaddr __user *upeer_sockaddr, -+ int __user *upeer_addrlen, int flags); - extern int __sys_accept4(int fd, struct sockaddr __user *upeer_sockaddr, - int __user *upeer_addrlen, int flags); - extern int __sys_socket(int family, int type, int protocol); ---- a/net/socket.c -+++ b/net/socket.c -@@ -1688,30 +1688,22 @@ SYSCALL_DEFINE2(listen, int, fd, int, ba - return __sys_listen(fd, backlog); - } - --int __sys_accept4_file(struct file *file, unsigned file_flags, -+struct file *do_accept(struct file *file, unsigned file_flags, - struct sockaddr __user *upeer_sockaddr, -- int __user *upeer_addrlen, int flags, -- unsigned long nofile) -+ int __user *upeer_addrlen, int flags) - { - struct socket *sock, *newsock; - struct file *newfile; -- int err, len, newfd; -+ int err, len; - struct sockaddr_storage address; - -- if (flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) -- return -EINVAL; -- -- if (SOCK_NONBLOCK != O_NONBLOCK && (flags & SOCK_NONBLOCK)) -- flags = (flags & ~SOCK_NONBLOCK) | O_NONBLOCK; -- - sock = sock_from_file(file, &err); - if (!sock) -- goto out; -+ return ERR_PTR(err); - -- err = -ENFILE; - newsock = sock_alloc(); - if (!newsock) -- goto out; -+ return ERR_PTR(-ENFILE); - - newsock->type = sock->type; - newsock->ops = sock->ops; -@@ -1722,18 +1714,9 @@ int __sys_accept4_file(struct file *file - */ - __module_get(newsock->ops->owner); - -- newfd = __get_unused_fd_flags(flags, nofile); -- if (unlikely(newfd < 0)) { -- err = newfd; -- sock_release(newsock); -- goto out; -- } - newfile = sock_alloc_file(newsock, flags, sock->sk->sk_prot_creator->name); -- if (IS_ERR(newfile)) { -- err = PTR_ERR(newfile); -- put_unused_fd(newfd); -- goto out; -- } -+ if (IS_ERR(newfile)) -+ return newfile; - - err = security_socket_accept(sock, newsock); - if (err) -@@ -1758,16 +1741,38 @@ int __sys_accept4_file(struct file *file - } - - /* File flags are not inherited via accept() unlike another OSes. */ -- -- fd_install(newfd, newfile); -- err = newfd; --out: -- return err; -+ return newfile; - out_fd: - fput(newfile); -- put_unused_fd(newfd); -- goto out; -+ return ERR_PTR(err); -+} -+ -+int __sys_accept4_file(struct file *file, unsigned file_flags, -+ struct sockaddr __user *upeer_sockaddr, -+ int __user *upeer_addrlen, int flags, -+ unsigned long nofile) -+{ -+ struct file *newfile; -+ int newfd; -+ -+ if (flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) -+ return -EINVAL; - -+ if (SOCK_NONBLOCK != O_NONBLOCK && (flags & SOCK_NONBLOCK)) -+ flags = (flags & ~SOCK_NONBLOCK) | O_NONBLOCK; -+ -+ newfd = __get_unused_fd_flags(flags, nofile); -+ if (unlikely(newfd < 0)) -+ return newfd; -+ -+ newfile = do_accept(file, file_flags, upeer_sockaddr, upeer_addrlen, -+ flags); -+ if (IS_ERR(newfile)) { -+ put_unused_fd(newfd); -+ return PTR_ERR(newfile); -+ } -+ fd_install(newfd, newfile); -+ return newfd; - } - - /* diff --git a/next-5.10/net-provide-__sys_shutdown_sock-that-takes-a-socket.patch b/next-5.10/net-provide-__sys_shutdown_sock-that-takes-a-socket.patch deleted file mode 100644 index 2d2a56c0e06..00000000000 --- a/next-5.10/net-provide-__sys_shutdown_sock-that-takes-a-socket.patch +++ /dev/null @@ -1,62 +0,0 @@ -From b7a66ad26bc4d7ba3285345a3bfef88214164df3 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Sat, 5 Sep 2020 11:13:35 -0600 -Subject: net: provide __sys_shutdown_sock() that takes a socket - -From: Jens Axboe - -[ Upstream commit b713c195d59332277a31a59c91f755e53b5b302b ] - -No functional changes in this patch, needed to provide io_uring support -for shutdown(2). - -Cc: netdev@vger.kernel.org -Cc: David S. Miller -Acked-by: Jakub Kicinski -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - include/linux/socket.h | 1 + - net/socket.c | 15 ++++++++++++--- - 2 files changed, 13 insertions(+), 3 deletions(-) - ---- a/include/linux/socket.h -+++ b/include/linux/socket.h -@@ -436,5 +436,6 @@ extern int __sys_getpeername(int fd, str - int __user *usockaddr_len); - extern int __sys_socketpair(int family, int type, int protocol, - int __user *usockvec); -+extern int __sys_shutdown_sock(struct socket *sock, int how); - extern int __sys_shutdown(int fd, int how); - #endif /* _LINUX_SOCKET_H */ ---- a/net/socket.c -+++ b/net/socket.c -@@ -2181,6 +2181,17 @@ SYSCALL_DEFINE5(getsockopt, int, fd, int - * Shutdown a socket. - */ - -+int __sys_shutdown_sock(struct socket *sock, int how) -+{ -+ int err; -+ -+ err = security_socket_shutdown(sock, how); -+ if (!err) -+ err = sock->ops->shutdown(sock, how); -+ -+ return err; -+} -+ - int __sys_shutdown(int fd, int how) - { - int err, fput_needed; -@@ -2188,9 +2199,7 @@ int __sys_shutdown(int fd, int how) - - sock = sockfd_lookup_light(fd, &err, &fput_needed); - if (sock != NULL) { -- err = security_socket_shutdown(sock, how); -- if (!err) -- err = sock->ops->shutdown(sock, how); -+ err = __sys_shutdown_sock(sock, how); - fput_light(sock->file, fput_needed); - } - return err; diff --git a/next-5.10/net-remove-cmsg-restriction-from-io_uring-based-send-recvmsg-calls.patch b/next-5.10/net-remove-cmsg-restriction-from-io_uring-based-send-recvmsg-calls.patch deleted file mode 100644 index a2f9e7957ea..00000000000 --- a/next-5.10/net-remove-cmsg-restriction-from-io_uring-based-send-recvmsg-calls.patch +++ /dev/null @@ -1,90 +0,0 @@ -From 89a29005f595d929d8d784f3422fb6a350f4b05f Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Wed, 17 Feb 2021 10:14:21 -0700 -Subject: net: remove cmsg restriction from io_uring based send/recvmsg calls - -From: Jens Axboe - -[ Upstream commit e54937963fa249595824439dc839c948188dea83 ] - -No need to restrict these anymore, as the worker threads are direct -clones of the original task. Hence we know for a fact that we can -support anything that the regular task can. - -Since the only user of proto_ops->flags was to flag PROTO_CMSG_DATA_ONLY, -kill the member and the flag definition too. - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - include/linux/net.h | 3 --- - net/ipv4/af_inet.c | 1 - - net/ipv6/af_inet6.c | 1 - - net/socket.c | 10 ---------- - 4 files changed, 15 deletions(-) - ---- a/include/linux/net.h -+++ b/include/linux/net.h -@@ -42,8 +42,6 @@ struct net; - #define SOCK_PASSCRED 3 - #define SOCK_PASSSEC 4 - --#define PROTO_CMSG_DATA_ONLY 0x0001 -- - #ifndef ARCH_HAS_SOCKET_TYPES - /** - * enum sock_type - Socket types -@@ -138,7 +136,6 @@ typedef int (*sk_read_actor_t)(read_desc - - struct proto_ops { - int family; -- unsigned int flags; - struct module *owner; - int (*release) (struct socket *sock); - int (*bind) (struct socket *sock, ---- a/net/ipv4/af_inet.c -+++ b/net/ipv4/af_inet.c -@@ -1017,7 +1017,6 @@ static int inet_compat_ioctl(struct sock - - const struct proto_ops inet_stream_ops = { - .family = PF_INET, -- .flags = PROTO_CMSG_DATA_ONLY, - .owner = THIS_MODULE, - .release = inet_release, - .bind = inet_bind, ---- a/net/ipv6/af_inet6.c -+++ b/net/ipv6/af_inet6.c -@@ -661,7 +661,6 @@ int inet6_recvmsg(struct socket *sock, s - - const struct proto_ops inet6_stream_ops = { - .family = PF_INET6, -- .flags = PROTO_CMSG_DATA_ONLY, - .owner = THIS_MODULE, - .release = inet6_release, - .bind = inet6_bind, ---- a/net/socket.c -+++ b/net/socket.c -@@ -2419,10 +2419,6 @@ static int ___sys_sendmsg(struct socket - long __sys_sendmsg_sock(struct socket *sock, struct msghdr *msg, - unsigned int flags) - { -- /* disallow ancillary data requests from this path */ -- if (msg->msg_control || msg->msg_controllen) -- return -EINVAL; -- - return ____sys_sendmsg(sock, msg, flags, NULL, 0); - } - -@@ -2631,12 +2627,6 @@ long __sys_recvmsg_sock(struct socket *s - struct user_msghdr __user *umsg, - struct sockaddr __user *uaddr, unsigned int flags) - { -- if (msg->msg_control || msg->msg_controllen) { -- /* disallow ancillary data reqs unless cmsg is plain data */ -- if (!(sock->ops->flags & PROTO_CMSG_DATA_ONLY)) -- return -EINVAL; -- } -- - return ____sys_recvmsg(sock, msg, umsg, uaddr, flags, 0); - } - diff --git a/next-5.10/nios32-add-support-for-tif_notify_signal.patch b/next-5.10/nios32-add-support-for-tif_notify_signal.patch deleted file mode 100644 index 5b025ddce89..00000000000 --- a/next-5.10/nios32-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,50 +0,0 @@ -From ad18cba50dbe1fbc8e57c4e96f4a952efe212eb3 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 15:21:21 -0600 -Subject: nios32: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 42020064274c235d720d9c4b7d9a678b133e59cf ] - -Wire up TIF_NOTIFY_SIGNAL handling for nios32. - -Cc: Ley Foon Tan -Acked-by: Ley Foon Tan -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/nios2/include/asm/thread_info.h | 2 ++ - arch/nios2/kernel/signal.c | 3 ++- - 2 files changed, 4 insertions(+), 1 deletion(-) - ---- a/arch/nios2/include/asm/thread_info.h -+++ b/arch/nios2/include/asm/thread_info.h -@@ -86,6 +86,7 @@ static inline struct thread_info *curren - #define TIF_MEMDIE 4 /* is terminating due to OOM killer */ - #define TIF_SECCOMP 5 /* secure computing */ - #define TIF_SYSCALL_AUDIT 6 /* syscall auditing active */ -+#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ - #define TIF_RESTORE_SIGMASK 9 /* restore signal mask in do_signal() */ - - #define TIF_POLLING_NRFLAG 16 /* true if poll_idle() is polling -@@ -97,6 +98,7 @@ static inline struct thread_info *curren - #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) - #define _TIF_SECCOMP (1 << TIF_SECCOMP) - #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - #define _TIF_RESTORE_SIGMASK (1 << TIF_RESTORE_SIGMASK) - #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) - ---- a/arch/nios2/kernel/signal.c -+++ b/arch/nios2/kernel/signal.c -@@ -309,7 +309,8 @@ asmlinkage int do_notify_resume(struct p - if (!user_mode(regs)) - return 0; - -- if (test_thread_flag(TIF_SIGPENDING)) { -+ if (test_thread_flag(TIF_SIGPENDING) || -+ test_thread_flag(TIF_NOTIFY_SIGNAL)) { - int restart = do_signal(regs); - - if (unlikely(restart)) { diff --git a/next-5.10/openrisc-add-support-for-tif_notify_signal.patch b/next-5.10/openrisc-add-support-for-tif_notify_signal.patch deleted file mode 100644 index 6c7e4cb7ee7..00000000000 --- a/next-5.10/openrisc-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,49 +0,0 @@ -From 64cef01754a4419e13f0f5f14ec4a9e181bffbe8 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 15:24:46 -0600 -Subject: openrisc: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit e181c0aa2e532af2b17128fbde699f8578cc0562 ] - -Wire up TIF_NOTIFY_SIGNAL handling for openrisc. - -Cc: openrisc@lists.librecores.org -Acked-by: Stafford Horne -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/openrisc/include/asm/thread_info.h | 2 ++ - arch/openrisc/kernel/signal.c | 2 +- - 2 files changed, 3 insertions(+), 1 deletion(-) - ---- a/arch/openrisc/include/asm/thread_info.h -+++ b/arch/openrisc/include/asm/thread_info.h -@@ -98,6 +98,7 @@ register struct thread_info *current_thr - #define TIF_SINGLESTEP 4 /* restore singlestep on return to user - * mode - */ -+#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ - #define TIF_SYSCALL_TRACEPOINT 8 /* for ftrace syscall instrumentation */ - #define TIF_RESTORE_SIGMASK 9 - #define TIF_POLLING_NRFLAG 16 /* true if poll_idle() is polling * TIF_NEED_RESCHED -@@ -109,6 +110,7 @@ register struct thread_info *current_thr - #define _TIF_SIGPENDING (1< -Date: Fri, 9 Oct 2020 15:27:02 -0600 -Subject: parisc: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 18cb3281285d2190c0605d2e53543802319bd1a1 ] - -Wire up TIF_NOTIFY_SIGNAL handling for parisc. - -Cc: linux-parisc@vger.kernel.org -Acked-by: Helge Deller -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/parisc/include/asm/thread_info.h | 4 +++- - arch/parisc/kernel/signal.c | 3 ++- - 2 files changed, 5 insertions(+), 2 deletions(-) - ---- a/arch/parisc/include/asm/thread_info.h -+++ b/arch/parisc/include/asm/thread_info.h -@@ -52,6 +52,7 @@ struct thread_info { - #define TIF_POLLING_NRFLAG 3 /* true if poll_idle() is polling TIF_NEED_RESCHED */ - #define TIF_32BIT 4 /* 32 bit binary */ - #define TIF_MEMDIE 5 /* is terminating due to OOM killer */ -+#define TIF_NOTIFY_SIGNAL 6 /* signal notifications exist */ - #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ - #define TIF_NOTIFY_RESUME 8 /* callback before returning to user */ - #define TIF_SINGLESTEP 9 /* single stepping? */ -@@ -61,6 +62,7 @@ struct thread_info { - - #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) - #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) - #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) - #define _TIF_32BIT (1 << TIF_32BIT) -@@ -72,7 +74,7 @@ struct thread_info { - #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) - - #define _TIF_USER_WORK_MASK (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | \ -- _TIF_NEED_RESCHED) -+ _TIF_NEED_RESCHED | _TIF_NOTIFY_SIGNAL) - #define _TIF_SYSCALL_TRACE_MASK (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP | \ - _TIF_BLOCKSTEP | _TIF_SYSCALL_AUDIT | \ - _TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT) ---- a/arch/parisc/kernel/signal.c -+++ b/arch/parisc/kernel/signal.c -@@ -609,7 +609,8 @@ do_signal(struct pt_regs *regs, long in_ - - void do_notify_resume(struct pt_regs *regs, long in_syscall) - { -- if (test_thread_flag(TIF_SIGPENDING)) -+ if (test_thread_flag(TIF_SIGPENDING) || -+ test_thread_flag(TIF_NOTIFY_SIGNAL)) - do_signal(regs, in_syscall); - - if (test_thread_flag(TIF_NOTIFY_RESUME)) diff --git a/next-5.10/powerpc-add-support-for-tif_notify_signal.patch b/next-5.10/powerpc-add-support-for-tif_notify_signal.patch deleted file mode 100644 index 2b37e4d2e58..00000000000 --- a/next-5.10/powerpc-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,59 +0,0 @@ -From 616d38f834d91a5af2e3afb08f12ecb8b32e4a96 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Thu, 22 Oct 2020 20:11:56 -0600 -Subject: powerpc: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 900f0713fdd730fab0f0bfa4a8ca4db2a8985bbe ] - -Wire up TIF_NOTIFY_SIGNAL handling for powerpc. - -Cc: linuxppc-dev@lists.ozlabs.org -Acked-by: Michael Ellerman -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/powerpc/include/asm/thread_info.h | 5 ++++- - arch/powerpc/kernel/signal.c | 2 +- - 2 files changed, 5 insertions(+), 2 deletions(-) - ---- a/arch/powerpc/include/asm/thread_info.h -+++ b/arch/powerpc/include/asm/thread_info.h -@@ -96,6 +96,7 @@ void arch_setup_new_exec(void); - #define TIF_SYSCALL_TRACE 0 /* syscall trace active */ - #define TIF_SIGPENDING 1 /* signal pending */ - #define TIF_NEED_RESCHED 2 /* rescheduling necessary */ -+#define TIF_NOTIFY_SIGNAL 3 /* signal notifications exist */ - #define TIF_SYSCALL_EMU 4 /* syscall emulation active */ - #define TIF_RESTORE_TM 5 /* need to restore TM FP/VEC/VSX */ - #define TIF_PATCH_PENDING 6 /* pending live patching update */ -@@ -121,6 +122,7 @@ void arch_setup_new_exec(void); - #define _TIF_SYSCALL_TRACE (1<thread.regs); - do_signal(current); - } diff --git a/next-5.10/revert-proc-don-t-allow-async-path-resolution-of-proc-self-components.patch b/next-5.10/revert-proc-don-t-allow-async-path-resolution-of-proc-self-components.patch deleted file mode 100644 index d5b8f40d4df..00000000000 --- a/next-5.10/revert-proc-don-t-allow-async-path-resolution-of-proc-self-components.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 2d7b7e632050476301e4fdfaf7a55751c97e6a83 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Mon, 15 Feb 2021 13:42:42 -0700 -Subject: Revert "proc: don't allow async path resolution of /proc/self components" - -From: Jens Axboe - -[ Upstream commit 9e8d9e829c2142cf1d7756e9ed2e0b4c7569d84c ] - -This reverts commit 8d4c3e76e3be11a64df95ddee52e99092d42fc19. - -No longer needed, as the io-wq worker threads have the right identity. - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - fs/proc/self.c | 7 ------- - 1 file changed, 7 deletions(-) - ---- a/fs/proc/self.c -+++ b/fs/proc/self.c -@@ -16,13 +16,6 @@ static const char *proc_self_get_link(st - pid_t tgid = task_tgid_nr_ns(current, ns); - char *name; - -- /* -- * Not currently supported. Once we can inherit all of struct pid, -- * we can allow this. -- */ -- if (current->flags & PF_KTHREAD) -- return ERR_PTR(-EOPNOTSUPP); -- - if (!tgid) - return ERR_PTR(-ENOENT); - /* max length of unsigned int in decimal + NULL term */ diff --git a/next-5.10/revert-proc-don-t-allow-async-path-resolution-of-proc-thread-self-components.patch b/next-5.10/revert-proc-don-t-allow-async-path-resolution-of-proc-thread-self-components.patch deleted file mode 100644 index c8fa41d9dea..00000000000 --- a/next-5.10/revert-proc-don-t-allow-async-path-resolution-of-proc-thread-self-components.patch +++ /dev/null @@ -1,47 +0,0 @@ -From 361b2a5293b998f8903a5d343ad5cf683a2970fa Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Mon, 15 Feb 2021 13:42:18 -0700 -Subject: Revert "proc: don't allow async path resolution of /proc/thread-self components" - -From: Jens Axboe - -[ Upstream commit 2587890b5e2892dfecaa5e5126bdac8076a4e6f7 ] - -This reverts commit 0d4370cfe36b7f1719123b621a4ec4d9c7a25f89. - -No longer needed, as the io-wq worker threads have the right identity. - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - fs/proc/self.c | 2 +- - fs/proc/thread_self.c | 7 ------- - 2 files changed, 1 insertion(+), 8 deletions(-) - ---- a/fs/proc/self.c -+++ b/fs/proc/self.c -@@ -20,7 +20,7 @@ static const char *proc_self_get_link(st - * Not currently supported. Once we can inherit all of struct pid, - * we can allow this. - */ -- if (current->flags & PF_IO_WORKER) -+ if (current->flags & PF_KTHREAD) - return ERR_PTR(-EOPNOTSUPP); - - if (!tgid) ---- a/fs/proc/thread_self.c -+++ b/fs/proc/thread_self.c -@@ -17,13 +17,6 @@ static const char *proc_thread_self_get_ - pid_t pid = task_pid_nr_ns(current, ns); - char *name; - -- /* -- * Not currently supported. Once we can inherit all of struct pid, -- * we can allow this. -- */ -- if (current->flags & PF_IO_WORKER) -- return ERR_PTR(-EOPNOTSUPP); -- - if (!pid) - return ERR_PTR(-ENOENT); - name = kmalloc(10 + 6 + 10 + 1, dentry ? GFP_KERNEL : GFP_ATOMIC); diff --git a/next-5.10/riscv-add-support-for-tif_notify_signal.patch b/next-5.10/riscv-add-support-for-tif_notify_signal.patch deleted file mode 100644 index 9380226552b..00000000000 --- a/next-5.10/riscv-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,53 +0,0 @@ -From 34a226ec555bf65f3b49bd173889554a04b5882a Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 14:29:17 -0600 -Subject: riscv: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 24a31b81e38309b1604f24520110aae1f83f3cbf ] - -Wire up TIF_NOTIFY_SIGNAL handling for riscv. - -Cc: linux-riscv@lists.infradead.org -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/riscv/include/asm/thread_info.h | 5 ++++- - arch/riscv/kernel/signal.c | 2 +- - 2 files changed, 5 insertions(+), 2 deletions(-) - ---- a/arch/riscv/include/asm/thread_info.h -+++ b/arch/riscv/include/asm/thread_info.h -@@ -80,6 +80,7 @@ struct thread_info { - #define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */ - #define TIF_SYSCALL_AUDIT 7 /* syscall auditing */ - #define TIF_SECCOMP 8 /* syscall secure computing */ -+#define TIF_NOTIFY_SIGNAL 9 /* signal notifications exist */ - - #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) - #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) -@@ -88,9 +89,11 @@ struct thread_info { - #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) - #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) - #define _TIF_SECCOMP (1 << TIF_SECCOMP) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - - #define _TIF_WORK_MASK \ -- (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED) -+ (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \ -+ _TIF_NOTIFY_SIGNAL) - - #define _TIF_SYSCALL_WORK \ - (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT | \ ---- a/arch/riscv/kernel/signal.c -+++ b/arch/riscv/kernel/signal.c -@@ -312,7 +312,7 @@ asmlinkage __visible void do_notify_resu - unsigned long thread_info_flags) - { - /* Handle pending signal delivery */ -- if (thread_info_flags & _TIF_SIGPENDING) -+ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) - do_signal(regs); - - if (thread_info_flags & _TIF_NOTIFY_RESUME) diff --git a/next-5.10/s390-add-support-for-tif_notify_signal.patch b/next-5.10/s390-add-support-for-tif_notify_signal.patch deleted file mode 100644 index a5730bc611e..00000000000 --- a/next-5.10/s390-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,85 +0,0 @@ -From 832028e42ad081b88ee7a72c8c9bcdea6eed3f97 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 15:34:12 -0600 -Subject: s390: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 75309018a24ddfb930c51bad8f4070b9bc2c923b ] - -Wire up TIF_NOTIFY_SIGNAL handling for s390. - -Cc: linux-s390@vger.kernel.org -Acked-by: Heiko Carstens -Acked-by: Sven Schnelle -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/s390/include/asm/thread_info.h | 2 ++ - arch/s390/kernel/entry.S | 11 ++++++----- - arch/s390/kernel/signal.c | 2 +- - 3 files changed, 9 insertions(+), 6 deletions(-) - ---- a/arch/s390/include/asm/thread_info.h -+++ b/arch/s390/include/asm/thread_info.h -@@ -65,6 +65,7 @@ void arch_setup_new_exec(void); - #define TIF_GUARDED_STORAGE 4 /* load guarded storage control block */ - #define TIF_PATCH_PENDING 5 /* pending live patching update */ - #define TIF_PGSTE 6 /* New mm's will use 4K page tables */ -+#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ - #define TIF_ISOLATE_BP 8 /* Run process with isolated BP */ - #define TIF_ISOLATE_BP_GUEST 9 /* Run KVM guests with isolated BP */ - -@@ -82,6 +83,7 @@ void arch_setup_new_exec(void); - #define TIF_SYSCALL_TRACEPOINT 27 /* syscall tracepoint instrumentation */ - - #define _TIF_NOTIFY_RESUME BIT(TIF_NOTIFY_RESUME) -+#define _TIF_NOTIFY_SIGNAL BIT(TIF_NOTIFY_SIGNAL) - #define _TIF_SIGPENDING BIT(TIF_SIGPENDING) - #define _TIF_NEED_RESCHED BIT(TIF_NEED_RESCHED) - #define _TIF_UPROBE BIT(TIF_UPROBE) ---- a/arch/s390/kernel/entry.S -+++ b/arch/s390/kernel/entry.S -@@ -52,7 +52,8 @@ STACK_SIZE = 1 << STACK_SHIFT - STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE - - _TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ -- _TIF_UPROBE | _TIF_GUARDED_STORAGE | _TIF_PATCH_PENDING) -+ _TIF_UPROBE | _TIF_GUARDED_STORAGE | _TIF_PATCH_PENDING | \ -+ _TIF_NOTIFY_SIGNAL) - _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ - _TIF_SYSCALL_TRACEPOINT) - _CIF_WORK = (_CIF_ASCE_PRIMARY | _CIF_ASCE_SECONDARY | _CIF_FPU) -@@ -481,8 +482,8 @@ ENTRY(system_call) - #endif - TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL_RESTART - jo .Lsysc_syscall_restart -- TSTMSK __TI_flags(%r12),_TIF_SIGPENDING -- jo .Lsysc_sigpending -+ TSTMSK __TI_flags(%r12),(_TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL) -+ jnz .Lsysc_sigpending - TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME - jo .Lsysc_notify_resume - TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY) -@@ -863,8 +864,8 @@ ENTRY(io_int_handler) - TSTMSK __TI_flags(%r12),_TIF_PATCH_PENDING - jo .Lio_patch_pending - #endif -- TSTMSK __TI_flags(%r12),_TIF_SIGPENDING -- jo .Lio_sigpending -+ TSTMSK __TI_flags(%r12),(_TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL) -+ jnz .Lio_sigpending - TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME - jo .Lio_notify_resume - TSTMSK __TI_flags(%r12),_TIF_GUARDED_STORAGE ---- a/arch/s390/kernel/signal.c -+++ b/arch/s390/kernel/signal.c -@@ -472,7 +472,7 @@ void do_signal(struct pt_regs *regs) - current->thread.system_call = - test_pt_regs_flag(regs, PIF_SYSCALL) ? regs->int_code : 0; - -- if (get_signal(&ksig)) { -+ if (test_thread_flag(TIF_SIGPENDING) && get_signal(&ksig)) { - /* Whee! Actually deliver the signal. */ - if (current->thread.system_call) { - regs->int_code = current->thread.system_call; diff --git a/next-5.10/saner-calling-conventions-for-unlazy_child.patch b/next-5.10/saner-calling-conventions-for-unlazy_child.patch deleted file mode 100644 index 8078551db8b..00000000000 --- a/next-5.10/saner-calling-conventions-for-unlazy_child.patch +++ /dev/null @@ -1,91 +0,0 @@ -From 2f065a20a2d0b75c25a9125f452991049dcd4205 Mon Sep 17 00:00:00 2001 -From: Al Viro -Date: Mon, 4 Jan 2021 00:08:41 -0500 -Subject: saner calling conventions for unlazy_child() - -From: Al Viro - -[ Upstream commit ae66db45fd309fd1c6d4e846dfc8414dfec7d6ad ] - -same as for the previous commit - instead of 0/-ECHILD make -it return true/false, rename to try_to_unlazy_child(). - -Signed-off-by: Al Viro -Signed-off-by: Greg Kroah-Hartman ---- - fs/namei.c | 30 +++++++++++++++--------------- - 1 file changed, 15 insertions(+), 15 deletions(-) - ---- a/fs/namei.c -+++ b/fs/namei.c -@@ -705,19 +705,19 @@ out: - } - - /** -- * unlazy_child - try to switch to ref-walk mode. -+ * try_to_unlazy_next - try to switch to ref-walk mode. - * @nd: nameidata pathwalk data -- * @dentry: child of nd->path.dentry -- * @seq: seq number to check dentry against -- * Returns: 0 on success, -ECHILD on failure -- * -- * unlazy_child attempts to legitimize the current nd->path, nd->root and dentry -- * for ref-walk mode. @dentry must be a path found by a do_lookup call on -- * @nd. Must be called from rcu-walk context. -- * Nothing should touch nameidata between unlazy_child() failure and -+ * @dentry: next dentry to step into -+ * @seq: seq number to check @dentry against -+ * Returns: true on success, false on failure -+ * -+ * Similar to to try_to_unlazy(), but here we have the next dentry already -+ * picked by rcu-walk and want to legitimize that in addition to the current -+ * nd->path and nd->root for ref-walk mode. Must be called from rcu-walk context. -+ * Nothing should touch nameidata between try_to_unlazy_next() failure and - * terminate_walk(). - */ --static int unlazy_child(struct nameidata *nd, struct dentry *dentry, unsigned seq) -+static bool try_to_unlazy_next(struct nameidata *nd, struct dentry *dentry, unsigned seq) - { - BUG_ON(!(nd->flags & LOOKUP_RCU)); - -@@ -747,7 +747,7 @@ static int unlazy_child(struct nameidata - if (unlikely(!legitimize_root(nd))) - goto out_dput; - rcu_read_unlock(); -- return 0; -+ return true; - - out2: - nd->path.mnt = NULL; -@@ -755,11 +755,11 @@ out1: - nd->path.dentry = NULL; - out: - rcu_read_unlock(); -- return -ECHILD; -+ return false; - out_dput: - rcu_read_unlock(); - dput(dentry); -- return -ECHILD; -+ return false; - } - - static inline int d_revalidate(struct dentry *dentry, unsigned int flags) -@@ -1374,7 +1374,7 @@ static inline int handle_mounts(struct n - return -ENOENT; - if (likely(__follow_mount_rcu(nd, path, inode, seqp))) - return 0; -- if (unlazy_child(nd, dentry, seq)) -+ if (!try_to_unlazy_next(nd, dentry, seq)) - return -ECHILD; - // *path might've been clobbered by __follow_mount_rcu() - path->mnt = nd->path.mnt; -@@ -1495,7 +1495,7 @@ static struct dentry *lookup_fast(struct - status = d_revalidate(dentry, nd->flags); - if (likely(status > 0)) - return dentry; -- if (unlazy_child(nd, dentry, seq)) -+ if (!try_to_unlazy_next(nd, dentry, seq)) - return ERR_PTR(-ECHILD); - if (unlikely(status == -ECHILD)) - /* we'd been told to redo it in non-rcu mode */ diff --git a/next-5.10/sh-add-support-for-tif_notify_signal.patch b/next-5.10/sh-add-support-for-tif_notify_signal.patch deleted file mode 100644 index 95e06a5274d..00000000000 --- a/next-5.10/sh-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,57 +0,0 @@ -From 1469cd034b065cbe06c2382a0d2aace50e3dc854 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 15:36:35 -0600 -Subject: sh: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit 6d3a273355e3c8471ddf9e8ce9a7cc4472bf1ccc ] - -Wire up TIF_NOTIFY_SIGNAL handling for sh. - -Cc: linux-sh@vger.kernel.org -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/sh/include/asm/thread_info.h | 4 +++- - arch/sh/kernel/signal_32.c | 2 +- - 2 files changed, 4 insertions(+), 2 deletions(-) - ---- a/arch/sh/include/asm/thread_info.h -+++ b/arch/sh/include/asm/thread_info.h -@@ -105,6 +105,7 @@ extern void init_thread_xstate(void); - #define TIF_SYSCALL_TRACE 0 /* syscall trace active */ - #define TIF_SIGPENDING 1 /* signal pending */ - #define TIF_NEED_RESCHED 2 /* rescheduling necessary */ -+#define TIF_NOTIFY_SIGNAL 3 /* signal notifications exist */ - #define TIF_SINGLESTEP 4 /* singlestepping active */ - #define TIF_SYSCALL_AUDIT 5 /* syscall auditing active */ - #define TIF_SECCOMP 6 /* secure computing */ -@@ -116,6 +117,7 @@ extern void init_thread_xstate(void); - #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) - #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) - #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - #define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) - #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) - #define _TIF_SECCOMP (1 << TIF_SECCOMP) -@@ -132,7 +134,7 @@ extern void init_thread_xstate(void); - #define _TIF_ALLWORK_MASK (_TIF_SYSCALL_TRACE | _TIF_SIGPENDING | \ - _TIF_NEED_RESCHED | _TIF_SYSCALL_AUDIT | \ - _TIF_SINGLESTEP | _TIF_NOTIFY_RESUME | \ -- _TIF_SYSCALL_TRACEPOINT) -+ _TIF_SYSCALL_TRACEPOINT | _TIF_NOTIFY_SIGNAL) - - /* work to do on interrupt/exception return */ - #define _TIF_WORK_MASK (_TIF_ALLWORK_MASK & ~(_TIF_SYSCALL_TRACE | \ ---- a/arch/sh/kernel/signal_32.c -+++ b/arch/sh/kernel/signal_32.c -@@ -499,7 +499,7 @@ asmlinkage void do_notify_resume(struct - unsigned long thread_info_flags) - { - /* deal with pending signal delivery */ -- if (thread_info_flags & _TIF_SIGPENDING) -+ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) - do_signal(regs, save_r0); - - if (thread_info_flags & _TIF_NOTIFY_RESUME) diff --git a/next-5.10/signal-add-task_sigpending-helper.patch b/next-5.10/signal-add-task_sigpending-helper.patch deleted file mode 100644 index a946e143947..00000000000 --- a/next-5.10/signal-add-task_sigpending-helper.patch +++ /dev/null @@ -1,108 +0,0 @@ -From a70e1337ccd380cbb8b647c1645ded7f3bf7a136 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Mon, 26 Oct 2020 14:32:27 -0600 -Subject: signal: Add task_sigpending() helper - -From: Jens Axboe - -[ Upstream commit 5c251e9dc0e127bac6fc5b8e6696363d2e35f515 ] - -This is in preparation for maintaining signal_pending() as the decider of -whether or not a schedule() loop should be broken, or continue sleeping. -This is different than the core signal use cases, which really need to know -whether an actual signal is pending or not. task_sigpending() returns -non-zero if TIF_SIGPENDING is set. - -Only core kernel use cases should care about the distinction between -the two, make sure those use the task_sigpending() helper. - -Signed-off-by: Jens Axboe -Signed-off-by: Thomas Gleixner -Reviewed-by: Thomas Gleixner -Reviewed-by: Oleg Nesterov -Link: https://lore.kernel.org/r/20201026203230.386348-2-axboe@kernel.dk -Signed-off-by: Greg Kroah-Hartman ---- - include/linux/sched/signal.h | 9 +++++++-- - kernel/events/uprobes.c | 2 +- - kernel/signal.c | 8 ++++---- - 3 files changed, 12 insertions(+), 7 deletions(-) - ---- a/include/linux/sched/signal.h -+++ b/include/linux/sched/signal.h -@@ -354,11 +354,16 @@ static inline int restart_syscall(void) - return -ERESTARTNOINTR; - } - --static inline int signal_pending(struct task_struct *p) -+static inline int task_sigpending(struct task_struct *p) - { - return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING)); - } - -+static inline int signal_pending(struct task_struct *p) -+{ -+ return task_sigpending(p); -+} -+ - static inline int __fatal_signal_pending(struct task_struct *p) - { - return unlikely(sigismember(&p->pending.signal, SIGKILL)); -@@ -366,7 +371,7 @@ static inline int __fatal_signal_pending - - static inline int fatal_signal_pending(struct task_struct *p) - { -- return signal_pending(p) && __fatal_signal_pending(p); -+ return task_sigpending(p) && __fatal_signal_pending(p); - } - - static inline int signal_pending_state(long state, struct task_struct *p) ---- a/kernel/events/uprobes.c -+++ b/kernel/events/uprobes.c -@@ -1973,7 +1973,7 @@ bool uprobe_deny_signal(void) - - WARN_ON_ONCE(utask->state != UTASK_SSTEP); - -- if (signal_pending(t)) { -+ if (task_sigpending(t)) { - spin_lock_irq(&t->sighand->siglock); - clear_tsk_thread_flag(t, TIF_SIGPENDING); - spin_unlock_irq(&t->sighand->siglock); ---- a/kernel/signal.c -+++ b/kernel/signal.c -@@ -984,7 +984,7 @@ static inline bool wants_signal(int sig, - if (task_is_stopped_or_traced(p)) - return false; - -- return task_curr(p) || !signal_pending(p); -+ return task_curr(p) || !task_sigpending(p); - } - - static void complete_signal(int sig, struct task_struct *p, enum pid_type type) -@@ -2813,7 +2813,7 @@ static void retarget_shared_pending(stru - /* Remove the signals this thread can handle. */ - sigandsets(&retarget, &retarget, &t->blocked); - -- if (!signal_pending(t)) -+ if (!task_sigpending(t)) - signal_wake_up(t, 0); - - if (sigisemptyset(&retarget)) -@@ -2847,7 +2847,7 @@ void exit_signals(struct task_struct *ts - - cgroup_threadgroup_change_end(tsk); - -- if (!signal_pending(tsk)) -+ if (!task_sigpending(tsk)) - goto out; - - unblocked = tsk->blocked; -@@ -2891,7 +2891,7 @@ long do_no_restart_syscall(struct restar - - static void __set_task_blocked(struct task_struct *tsk, const sigset_t *newset) - { -- if (signal_pending(tsk) && !thread_group_empty(tsk)) { -+ if (task_sigpending(tsk) && !thread_group_empty(tsk)) { - sigset_t newblocked; - /* A set of now blocked but previously unblocked signals. */ - sigandnsets(&newblocked, newset, ¤t->blocked); diff --git a/next-5.10/signal-kill-jobctl_task_work.patch b/next-5.10/signal-kill-jobctl_task_work.patch deleted file mode 100644 index ba6be83cd84..00000000000 --- a/next-5.10/signal-kill-jobctl_task_work.patch +++ /dev/null @@ -1,69 +0,0 @@ -From 450fb0845804e4245b90eb155d7ac34cda11429f Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 16:03:01 -0600 -Subject: signal: kill JOBCTL_TASK_WORK - -From: Jens Axboe - -[ Upstream commit 98b89b649fce39dacb9dc036d6d0fdb8caff73f7 ] - -It's no longer used, get rid of it. - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - include/linux/sched/jobctl.h | 4 +--- - kernel/signal.c | 20 -------------------- - 2 files changed, 1 insertion(+), 23 deletions(-) - ---- a/include/linux/sched/jobctl.h -+++ b/include/linux/sched/jobctl.h -@@ -19,7 +19,6 @@ struct task_struct; - #define JOBCTL_TRAPPING_BIT 21 /* switching to TRACED */ - #define JOBCTL_LISTENING_BIT 22 /* ptracer is listening for events */ - #define JOBCTL_TRAP_FREEZE_BIT 23 /* trap for cgroup freezer */ --#define JOBCTL_TASK_WORK_BIT 24 /* set by TWA_SIGNAL */ - - #define JOBCTL_STOP_DEQUEUED (1UL << JOBCTL_STOP_DEQUEUED_BIT) - #define JOBCTL_STOP_PENDING (1UL << JOBCTL_STOP_PENDING_BIT) -@@ -29,10 +28,9 @@ struct task_struct; - #define JOBCTL_TRAPPING (1UL << JOBCTL_TRAPPING_BIT) - #define JOBCTL_LISTENING (1UL << JOBCTL_LISTENING_BIT) - #define JOBCTL_TRAP_FREEZE (1UL << JOBCTL_TRAP_FREEZE_BIT) --#define JOBCTL_TASK_WORK (1UL << JOBCTL_TASK_WORK_BIT) - - #define JOBCTL_TRAP_MASK (JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY) --#define JOBCTL_PENDING_MASK (JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK | JOBCTL_TASK_WORK) -+#define JOBCTL_PENDING_MASK (JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK) - - extern bool task_set_jobctl_pending(struct task_struct *task, unsigned long mask); - extern void task_clear_jobctl_trapping(struct task_struct *task); ---- a/kernel/signal.c -+++ b/kernel/signal.c -@@ -2544,26 +2544,6 @@ bool get_signal(struct ksignal *ksig) - - relock: - spin_lock_irq(&sighand->siglock); -- /* -- * Make sure we can safely read ->jobctl() in task_work add. As Oleg -- * states: -- * -- * It pairs with mb (implied by cmpxchg) before READ_ONCE. So we -- * roughly have -- * -- * task_work_add: get_signal: -- * STORE(task->task_works, new_work); STORE(task->jobctl); -- * mb(); mb(); -- * LOAD(task->jobctl); LOAD(task->task_works); -- * -- * and we can rely on STORE-MB-LOAD [ in task_work_add]. -- */ -- smp_store_mb(current->jobctl, current->jobctl & ~JOBCTL_TASK_WORK); -- if (unlikely(current->task_works)) { -- spin_unlock_irq(&sighand->siglock); -- task_work_run(); -- goto relock; -- } - - /* - * Every stopped thread goes here after wakeup. Check to see if diff --git a/next-5.10/sparc-add-support-for-tif_notify_signal.patch b/next-5.10/sparc-add-support-for-tif_notify_signal.patch deleted file mode 100644 index f9112eab075..00000000000 --- a/next-5.10/sparc-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,96 +0,0 @@ -From 12d81984f0d70ca40c14f319ceea586b9351b2cb Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 15:44:37 -0600 -Subject: sparc: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit f50a7052f5e70ee7a6a5e2ed08660994dc3df2a5 ] - -Wire up TIF_NOTIFY_SIGNAL handling for sparc. - -Cc: sparclinux@vger.kernel.org -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/sparc/include/asm/thread_info_32.h | 4 +++- - arch/sparc/include/asm/thread_info_64.h | 6 ++++-- - arch/sparc/kernel/signal_32.c | 2 +- - arch/sparc/kernel/signal_64.c | 2 +- - 4 files changed, 9 insertions(+), 5 deletions(-) - ---- a/arch/sparc/include/asm/thread_info_32.h -+++ b/arch/sparc/include/asm/thread_info_32.h -@@ -104,6 +104,7 @@ register struct thread_info *current_thr - #define TIF_SIGPENDING 2 /* signal pending */ - #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ - #define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */ -+#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ - #define TIF_USEDFPU 8 /* FPU was used by this task - * this quantum (SMP) */ - #define TIF_POLLING_NRFLAG 9 /* true if poll_idle() is polling -@@ -115,11 +116,12 @@ register struct thread_info *current_thr - #define _TIF_NOTIFY_RESUME (1< -Date: Thu, 1 Apr 2021 19:53:29 -0600 -Subject: task_work: add helper for more targeted task_work canceling - -From: Jens Axboe - -[ Upstream commit c7aab1a7c52b82d9afd7e03c398eb03dc2aa0507 ] - -The only exported helper we have right now is task_work_cancel(), which -cancels any task_work from a given task where func matches the queued -work item. This is a bit too coarse for some use cases. Add a -task_work_cancel_match() that allows to more specifically target -individual work items outside of purely the callback function used. - -task_work_cancel() can be trivially implemented on top of that, hence do -so. - -Reviewed-by: Oleg Nesterov -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - include/linux/task_work.h | 2 ++ - kernel/task_work.c | 35 ++++++++++++++++++++++++++++------- - 2 files changed, 30 insertions(+), 7 deletions(-) - ---- a/include/linux/task_work.h -+++ b/include/linux/task_work.h -@@ -22,6 +22,8 @@ enum task_work_notify_mode { - int task_work_add(struct task_struct *task, struct callback_head *twork, - enum task_work_notify_mode mode); - -+struct callback_head *task_work_cancel_match(struct task_struct *task, -+ bool (*match)(struct callback_head *, void *data), void *data); - struct callback_head *task_work_cancel(struct task_struct *, task_work_func_t); - void task_work_run(void); - ---- a/kernel/task_work.c -+++ b/kernel/task_work.c -@@ -59,18 +59,17 @@ int task_work_add(struct task_struct *ta - } - - /** -- * task_work_cancel - cancel a pending work added by task_work_add() -+ * task_work_cancel_match - cancel a pending work added by task_work_add() - * @task: the task which should execute the work -- * @func: identifies the work to remove -- * -- * Find the last queued pending work with ->func == @func and remove -- * it from queue. -+ * @match: match function to call - * - * RETURNS: - * The found work or NULL if not found. - */ - struct callback_head * --task_work_cancel(struct task_struct *task, task_work_func_t func) -+task_work_cancel_match(struct task_struct *task, -+ bool (*match)(struct callback_head *, void *data), -+ void *data) - { - struct callback_head **pprev = &task->task_works; - struct callback_head *work; -@@ -86,7 +85,7 @@ task_work_cancel(struct task_struct *tas - */ - raw_spin_lock_irqsave(&task->pi_lock, flags); - while ((work = READ_ONCE(*pprev))) { -- if (work->func != func) -+ if (!match(work, data)) - pprev = &work->next; - else if (cmpxchg(pprev, work, work->next) == work) - break; -@@ -96,6 +95,28 @@ task_work_cancel(struct task_struct *tas - return work; - } - -+static bool task_work_func_match(struct callback_head *cb, void *data) -+{ -+ return cb->func == data; -+} -+ -+/** -+ * task_work_cancel - cancel a pending work added by task_work_add() -+ * @task: the task which should execute the work -+ * @func: identifies the work to remove -+ * -+ * Find the last queued pending work with ->func == @func and remove -+ * it from queue. -+ * -+ * RETURNS: -+ * The found work or NULL if not found. -+ */ -+struct callback_head * -+task_work_cancel(struct task_struct *task, task_work_func_t func) -+{ -+ return task_work_cancel_match(task, task_work_func_match, func); -+} -+ - /** - * task_work_run - execute the works added by task_work_add() - * diff --git a/next-5.10/task_work-remove-legacy-twa_signal-path.patch b/next-5.10/task_work-remove-legacy-twa_signal-path.patch deleted file mode 100644 index 0b9c01742a9..00000000000 --- a/next-5.10/task_work-remove-legacy-twa_signal-path.patch +++ /dev/null @@ -1,63 +0,0 @@ -From 3e7b055854f9ff08f3c7015c44229147dad5dd08 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 16:01:33 -0600 -Subject: task_work: remove legacy TWA_SIGNAL path - -From: Jens Axboe - -[ Upstream commit 03941ccfda161c2680147fa5ab92aead2a79cac1 ] - -All archs now support TIF_NOTIFY_SIGNAL. - -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - kernel/task_work.c | 30 +----------------------------- - 1 file changed, 1 insertion(+), 29 deletions(-) - ---- a/kernel/task_work.c -+++ b/kernel/task_work.c -@@ -5,34 +5,6 @@ - - static struct callback_head work_exited; /* all we need is ->next == NULL */ - --/* -- * TWA_SIGNAL signaling - use TIF_NOTIFY_SIGNAL, if available, as it's faster -- * than TIF_SIGPENDING as there's no dependency on ->sighand. The latter is -- * shared for threads, and can cause contention on sighand->lock. Even for -- * the non-threaded case TIF_NOTIFY_SIGNAL is more efficient, as no locking -- * or IRQ disabling is involved for notification (or running) purposes. -- */ --static void task_work_notify_signal(struct task_struct *task) --{ --#if defined(TIF_NOTIFY_SIGNAL) -- set_notify_signal(task); --#else -- unsigned long flags; -- -- /* -- * Only grab the sighand lock if we don't already have some -- * task_work pending. This pairs with the smp_store_mb() -- * in get_signal(), see comment there. -- */ -- if (!(READ_ONCE(task->jobctl) & JOBCTL_TASK_WORK) && -- lock_task_sighand(task, &flags)) { -- task->jobctl |= JOBCTL_TASK_WORK; -- signal_wake_up(task, 0); -- unlock_task_sighand(task, &flags); -- } --#endif --} -- - /** - * task_work_add - ask the @task to execute @work->func() - * @task: the task which should run the callback -@@ -76,7 +48,7 @@ int task_work_add(struct task_struct *ta - set_notify_resume(task); - break; - case TWA_SIGNAL: -- task_work_notify_signal(task); -+ set_notify_signal(task); - break; - default: - WARN_ON_ONCE(1); diff --git a/next-5.10/task_work-unconditionally-run-task_work-from-get_signal.patch b/next-5.10/task_work-unconditionally-run-task_work-from-get_signal.patch deleted file mode 100644 index 90a6ab47f89..00000000000 --- a/next-5.10/task_work-unconditionally-run-task_work-from-get_signal.patch +++ /dev/null @@ -1,44 +0,0 @@ -From 0f315fd4143c1591a7f902fa08b71b83764f4fa0 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Tue, 5 Jan 2021 11:32:43 -0700 -Subject: task_work: unconditionally run task_work from get_signal() - -From: Jens Axboe - -[ Upstream commit 35d0b389f3b23439ad15b610d6e43fc72fc75779 ] - -Song reported a boot regression in a kvm image with 5.11-rc, and bisected -it down to the below patch. Debugging this issue, turns out that the boot -stalled when a task is waiting on a pipe being released. As we no longer -run task_work from get_signal() unless it's queued with TWA_SIGNAL, the -task goes idle without running the task_work. This prevents ->release() -from being called on the pipe, which another boot task is waiting on. - -For now, re-instate the unconditional task_work run from get_signal(). -For 5.12, we'll collapse TWA_RESUME and TWA_SIGNAL, as it no longer -makes sense to have a distinction between the two. This will turn -task_work notification into a simple boolean, whether to notify or not. - -Fixes: 98b89b649fce ("signal: kill JOBCTL_TASK_WORK") -Reported-by: Song Liu -Tested-by: John Stultz -Tested-by: Douglas Anderson -Tested-by: Sedat Dilek # LLVM/Clang version 11.0.1 -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - kernel/signal.c | 3 +++ - 1 file changed, 3 insertions(+) - ---- a/kernel/signal.c -+++ b/kernel/signal.c -@@ -2520,6 +2520,9 @@ bool get_signal(struct ksignal *ksig) - struct signal_struct *signal = current->signal; - int signr; - -+ if (unlikely(current->task_works)) -+ task_work_run(); -+ - /* - * For non-generic architectures, check for TIF_NOTIFY_SIGNAL so - * that the arch handlers don't all have to do it. If we get here diff --git a/next-5.10/task_work-use-tif_notify_signal-if-available.patch b/next-5.10/task_work-use-tif_notify_signal-if-available.patch deleted file mode 100644 index 41b068d36db..00000000000 --- a/next-5.10/task_work-use-tif_notify_signal-if-available.patch +++ /dev/null @@ -1,138 +0,0 @@ -From fd7614eb55ed1cd0017baea8a7621586d44089be Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Mon, 26 Oct 2020 14:32:30 -0600 -Subject: task_work: Use TIF_NOTIFY_SIGNAL if available - -From: Jens Axboe - -[ Upstream commit 114518eb6430b832d2f9f5a008043b913ccf0e24 ] - -If the arch supports TIF_NOTIFY_SIGNAL, then use that for TWA_SIGNAL as -it's more efficient than using the signal delivery method. This is -especially true on threaded applications, where ->sighand is shared across -threads, but it's also lighter weight on non-shared cases. - -io_uring is a heavy consumer of TWA_SIGNAL based task_work. A test with -threads shows a nice improvement running an io_uring based echo server. - -stock kernel: -0.01% <= 0.1 milliseconds -95.86% <= 0.2 milliseconds -98.27% <= 0.3 milliseconds -99.71% <= 0.4 milliseconds -100.00% <= 0.5 milliseconds -100.00% <= 0.6 milliseconds -100.00% <= 0.7 milliseconds -100.00% <= 0.8 milliseconds -100.00% <= 0.9 milliseconds -100.00% <= 1.0 milliseconds -100.00% <= 1.1 milliseconds -100.00% <= 2 milliseconds -100.00% <= 3 milliseconds -100.00% <= 3 milliseconds -1378930.00 requests per second -~1600% CPU - -1.38M requests/second, and all 16 CPUs are maxed out. - -patched kernel: -0.01% <= 0.1 milliseconds -98.24% <= 0.2 milliseconds -99.47% <= 0.3 milliseconds -99.99% <= 0.4 milliseconds -100.00% <= 0.5 milliseconds -100.00% <= 0.6 milliseconds -100.00% <= 0.7 milliseconds -100.00% <= 0.8 milliseconds -100.00% <= 0.9 milliseconds -100.00% <= 1.2 milliseconds -1666111.38 requests per second -~1450% CPU - -1.67M requests/second, and we're no longer just hammering on the sighand -lock. The original reporter states: - -"For 5.7.15 my benchmark achieves 1.6M qps and system cpu is at ~80%. - for 5.7.16 or later it achieves only 1M qps and the system cpu is is - at ~100%" - -with the only difference there being that TWA_SIGNAL is used -unconditionally in 5.7.16, since it's required to be able to handle the -inability to run task_work if the application is waiting in the kernel -already on an event that needs task_work run to be satisfied. Also see -commit 0ba9c9edcd15. - -Reported-by: Roman Gershman -Signed-off-by: Jens Axboe -Signed-off-by: Thomas Gleixner -Reviewed-by: Oleg Nesterov -Link: https://lore.kernel.org/r/20201026203230.386348-5-axboe@kernel.dk -Signed-off-by: Greg Kroah-Hartman ---- - kernel/task_work.c | 41 +++++++++++++++++++++++++++++------------ - 1 file changed, 29 insertions(+), 12 deletions(-) - ---- a/kernel/task_work.c -+++ b/kernel/task_work.c -@@ -5,6 +5,34 @@ - - static struct callback_head work_exited; /* all we need is ->next == NULL */ - -+/* -+ * TWA_SIGNAL signaling - use TIF_NOTIFY_SIGNAL, if available, as it's faster -+ * than TIF_SIGPENDING as there's no dependency on ->sighand. The latter is -+ * shared for threads, and can cause contention on sighand->lock. Even for -+ * the non-threaded case TIF_NOTIFY_SIGNAL is more efficient, as no locking -+ * or IRQ disabling is involved for notification (or running) purposes. -+ */ -+static void task_work_notify_signal(struct task_struct *task) -+{ -+#if defined(TIF_NOTIFY_SIGNAL) -+ set_notify_signal(task); -+#else -+ unsigned long flags; -+ -+ /* -+ * Only grab the sighand lock if we don't already have some -+ * task_work pending. This pairs with the smp_store_mb() -+ * in get_signal(), see comment there. -+ */ -+ if (!(READ_ONCE(task->jobctl) & JOBCTL_TASK_WORK) && -+ lock_task_sighand(task, &flags)) { -+ task->jobctl |= JOBCTL_TASK_WORK; -+ signal_wake_up(task, 0); -+ unlock_task_sighand(task, &flags); -+ } -+#endif -+} -+ - /** - * task_work_add - ask the @task to execute @work->func() - * @task: the task which should run the callback -@@ -33,7 +61,6 @@ int task_work_add(struct task_struct *ta - enum task_work_notify_mode notify) - { - struct callback_head *head; -- unsigned long flags; - - do { - head = READ_ONCE(task->task_works); -@@ -49,17 +76,7 @@ int task_work_add(struct task_struct *ta - set_notify_resume(task); - break; - case TWA_SIGNAL: -- /* -- * Only grab the sighand lock if we don't already have some -- * task_work pending. This pairs with the smp_store_mb() -- * in get_signal(), see comment there. -- */ -- if (!(READ_ONCE(task->jobctl) & JOBCTL_TASK_WORK) && -- lock_task_sighand(task, &flags)) { -- task->jobctl |= JOBCTL_TASK_WORK; -- signal_wake_up(task, 0); -- unlock_task_sighand(task, &flags); -- } -+ task_work_notify_signal(task); - break; - default: - WARN_ON_ONCE(1); diff --git a/next-5.10/tools-headers-uapi-sync-openat2.h-with-the-kernel-sources.patch b/next-5.10/tools-headers-uapi-sync-openat2.h-with-the-kernel-sources.patch deleted file mode 100644 index 94d7b7f22b5..00000000000 --- a/next-5.10/tools-headers-uapi-sync-openat2.h-with-the-kernel-sources.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 8e9c2ce1060f14cda30808322d9a8dcbd7a3fd77 Mon Sep 17 00:00:00 2001 -From: Arnaldo Carvalho de Melo -Date: Tue, 23 Feb 2021 09:44:37 -0300 -Subject: tools headers UAPI: Sync openat2.h with the kernel sources - -From: Arnaldo Carvalho de Melo - -[ Upstream commit 1e61463cfcd0b3e7a19ba36b8a98c64ebaac5c6e ] - -To pick the changes in: - - 99668f618062816c ("fs: expose LOOKUP_CACHED through openat2() RESOLVE_CACHED") - -That don't result in any change in tooling, only silences this perf -build warning: - - Warning: Kernel ABI header at 'tools/include/uapi/linux/openat2.h' differs from latest version at 'include/uapi/linux/openat2.h' - diff -u tools/include/uapi/linux/openat2.h include/uapi/linux/openat2.h - -Cc: Al Viro -Cc: Jens Axboe -Signed-off-by: Arnaldo Carvalho de Melo -Signed-off-by: Greg Kroah-Hartman ---- - tools/include/uapi/linux/openat2.h | 4 ++++ - 1 file changed, 4 insertions(+) - ---- a/tools/include/uapi/linux/openat2.h -+++ b/tools/include/uapi/linux/openat2.h -@@ -35,5 +35,9 @@ struct open_how { - #define RESOLVE_IN_ROOT 0x10 /* Make all jumps to "/" and ".." - be scoped inside the dirfd - (similar to chroot(2)). */ -+#define RESOLVE_CACHED 0x20 /* Only complete if resolution can be -+ completed through cached lookup. May -+ return -EAGAIN if that's not -+ possible. */ - - #endif /* _UAPI_LINUX_OPENAT2_H */ diff --git a/next-5.10/um-add-support-for-tif_notify_signal.patch b/next-5.10/um-add-support-for-tif_notify_signal.patch deleted file mode 100644 index d561142d6f7..00000000000 --- a/next-5.10/um-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,50 +0,0 @@ -From 9b4d7860ca092440b774c82827455f53e649ee0f Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 15:47:28 -0600 -Subject: um: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit a5b3cd32ff238b87e94d47b927aff117e22d13c0 ] - -Wire up TIF_NOTIFY_SIGNAL handling for um. - -Cc: linux-um@lists.infradead.org -Acked-By: Anton Ivanov -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/um/include/asm/thread_info.h | 2 ++ - arch/um/kernel/process.c | 3 ++- - 2 files changed, 4 insertions(+), 1 deletion(-) - ---- a/arch/um/include/asm/thread_info.h -+++ b/arch/um/include/asm/thread_info.h -@@ -57,6 +57,7 @@ static inline struct thread_info *curren - #define TIF_SYSCALL_TRACE 0 /* syscall trace active */ - #define TIF_SIGPENDING 1 /* signal pending */ - #define TIF_NEED_RESCHED 2 /* rescheduling necessary */ -+#define TIF_NOTIFY_SIGNAL 3 /* signal notifications exist */ - #define TIF_RESTART_BLOCK 4 - #define TIF_MEMDIE 5 /* is terminating due to OOM killer */ - #define TIF_SYSCALL_AUDIT 6 -@@ -68,6 +69,7 @@ static inline struct thread_info *curren - #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) - #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) - #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - #define _TIF_MEMDIE (1 << TIF_MEMDIE) - #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) - #define _TIF_SECCOMP (1 << TIF_SECCOMP) ---- a/arch/um/kernel/process.c -+++ b/arch/um/kernel/process.c -@@ -99,7 +99,8 @@ void interrupt_end(void) - - if (need_resched()) - schedule(); -- if (test_thread_flag(TIF_SIGPENDING)) -+ if (test_thread_flag(TIF_SIGPENDING) || -+ test_thread_flag(TIF_NOTIFY_SIGNAL)) - do_signal(regs); - if (test_thread_flag(TIF_NOTIFY_RESUME)) - tracehook_notify_resume(regs); diff --git a/next-5.10/x86-process-setup-io_threads-more-like-normal-user-space-threads.patch b/next-5.10/x86-process-setup-io_threads-more-like-normal-user-space-threads.patch deleted file mode 100644 index 7bac5de85ea..00000000000 --- a/next-5.10/x86-process-setup-io_threads-more-like-normal-user-space-threads.patch +++ /dev/null @@ -1,94 +0,0 @@ -From f757cc1efe0ec578f54dc857f1684dcd6541b464 Mon Sep 17 00:00:00 2001 -From: Stefan Metzmacher -Date: Wed, 5 May 2021 13:03:10 +0200 -Subject: x86/process: setup io_threads more like normal user space threads - -From: Stefan Metzmacher - -[ Upstream commit 50b7b6f29de3e18e9d6c09641256a0296361cfee ] - -As io_threads are fully set up USER threads it's clearer to separate the -code path from the KTHREAD logic. - -The only remaining difference to user space threads is that io_threads -never return to user space again. Instead they loop within the given -worker function. - -The fact that they never return to user space means they don't have an -user space thread stack. In order to indicate that to tools like gdb we -reset the stack and instruction pointers to 0. - -This allows gdb attach to user space processes using io-uring, which like -means that they have io_threads, without printing worrying message like -this: - - warning: Selected architecture i386:x86-64 is not compatible with reported target architecture i386 - - warning: Architecture rejected target-supplied description - -The output will be something like this: - - (gdb) info threads - Id Target Id Frame - * 1 LWP 4863 "io_uring-cp-for" syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 - 2 LWP 4864 "iou-mgr-4863" 0x0000000000000000 in ?? () - 3 LWP 4865 "iou-wrk-4863" 0x0000000000000000 in ?? () - (gdb) thread 3 - [Switching to thread 3 (LWP 4865)] - #0 0x0000000000000000 in ?? () - (gdb) bt - #0 0x0000000000000000 in ?? () - Backtrace stopped: Cannot access memory at address 0x0 - -Fixes: 4727dc20e042 ("arch: setup PF_IO_WORKER threads like PF_KTHREAD") -Link: https://lore.kernel.org/io-uring/044d0bad-6888-a211-e1d3-159a4aeed52d@polymtl.ca/T/#m1bbf5727e3d4e839603f6ec7ed79c7eebfba6267 -Signed-off-by: Stefan Metzmacher -cc: Linus Torvalds -cc: Jens Axboe -cc: Andy Lutomirski -cc: linux-kernel@vger.kernel.org -cc: io-uring@vger.kernel.org -cc: x86@kernel.org -Link: https://lore.kernel.org/r/20210505110310.237537-1-metze@samba.org -Reviewed-by: Thomas Gleixner -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/x86/kernel/process.c | 19 ++++++++++++++++++- - 1 file changed, 18 insertions(+), 1 deletion(-) - ---- a/arch/x86/kernel/process.c -+++ b/arch/x86/kernel/process.c -@@ -162,7 +162,7 @@ int copy_thread(unsigned long clone_flag - #endif - - /* Kernel thread ? */ -- if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { -+ if (unlikely(p->flags & PF_KTHREAD)) { - memset(childregs, 0, sizeof(struct pt_regs)); - kthread_frame_init(frame, sp, arg); - return 0; -@@ -178,6 +178,23 @@ int copy_thread(unsigned long clone_flag - task_user_gs(p) = get_user_gs(current_pt_regs()); - #endif - -+ if (unlikely(p->flags & PF_IO_WORKER)) { -+ /* -+ * An IO thread is a user space thread, but it doesn't -+ * return to ret_after_fork(). -+ * -+ * In order to indicate that to tools like gdb, -+ * we reset the stack and instruction pointers. -+ * -+ * It does the same kernel frame setup to return to a kernel -+ * function that a kernel thread does. -+ */ -+ childregs->sp = 0; -+ childregs->ip = 0; -+ kthread_frame_init(frame, sp, arg); -+ return 0; -+ } -+ - /* Set a new TLS for the child thread? */ - if (clone_flags & CLONE_SETTLS) - ret = set_new_tls(p, tls); diff --git a/next-5.10/x86-wire-up-tif_notify_signal.patch b/next-5.10/x86-wire-up-tif_notify_signal.patch deleted file mode 100644 index 15aa0ad1c61..00000000000 --- a/next-5.10/x86-wire-up-tif_notify_signal.patch +++ /dev/null @@ -1,40 +0,0 @@ -From f8f5fdc79f0e480230c501bc5d744bc08defec6c Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Mon, 26 Oct 2020 14:32:29 -0600 -Subject: x86: Wire up TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit c8d5ed67936fddbe2ae845fc80397718006322d7 ] - -The generic entry code has support for TIF_NOTIFY_SIGNAL already. Just -provide the TIF bit. - -[ tglx: Adopted to other TIF changes in x86 ] - -Signed-off-by: Jens Axboe -Signed-off-by: Thomas Gleixner -Link: https://lore.kernel.org/r/20201026203230.386348-4-axboe@kernel.dk -Signed-off-by: Greg Kroah-Hartman ---- - arch/x86/include/asm/thread_info.h | 2 ++ - 1 file changed, 2 insertions(+) - ---- a/arch/x86/include/asm/thread_info.h -+++ b/arch/x86/include/asm/thread_info.h -@@ -93,6 +93,7 @@ struct thread_info { - #define TIF_NOTSC 16 /* TSC is not accessible in userland */ - #define TIF_IA32 17 /* IA32 compatibility process */ - #define TIF_SLD 18 /* Restore split lock detection on context switch */ -+#define TIF_NOTIFY_SIGNAL 19 /* signal notifications exist */ - #define TIF_MEMDIE 20 /* is terminating due to OOM killer */ - #define TIF_POLLING_NRFLAG 21 /* idle is polling for TIF_NEED_RESCHED */ - #define TIF_IO_BITMAP 22 /* uses I/O bitmap */ -@@ -121,6 +122,7 @@ struct thread_info { - #define _TIF_NOCPUID (1 << TIF_NOCPUID) - #define _TIF_NOTSC (1 << TIF_NOTSC) - #define _TIF_IA32 (1 << TIF_IA32) -+#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) - #define _TIF_SLD (1 << TIF_SLD) - #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) - #define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) diff --git a/next-5.10/xtensa-add-support-for-tif_notify_signal.patch b/next-5.10/xtensa-add-support-for-tif_notify_signal.patch deleted file mode 100644 index 851aa45e5fc..00000000000 --- a/next-5.10/xtensa-add-support-for-tif_notify_signal.patch +++ /dev/null @@ -1,72 +0,0 @@ -From 4a20887d142c23ea4f6e0dc1d7c39756ec71d327 Mon Sep 17 00:00:00 2001 -From: Jens Axboe -Date: Fri, 9 Oct 2020 15:56:07 -0600 -Subject: xtensa: add support for TIF_NOTIFY_SIGNAL - -From: Jens Axboe - -[ Upstream commit bec58f40d6c5372d812c93cc3947f3bc97440e57 ] - -Wire up TIF_NOTIFY_SIGNAL handling for xtensa. - -Thanks to Max Filippov for making the asm correct. - -Cc: linux-xtensa@linux-xtensa.org -Signed-off-by: Jens Axboe -Signed-off-by: Greg Kroah-Hartman ---- - arch/xtensa/include/asm/thread_info.h | 5 ++++- - arch/xtensa/kernel/entry.S | 4 ++-- - arch/xtensa/kernel/signal.c | 3 ++- - 3 files changed, 8 insertions(+), 4 deletions(-) - ---- a/arch/xtensa/include/asm/thread_info.h -+++ b/arch/xtensa/include/asm/thread_info.h -@@ -111,18 +111,21 @@ static inline struct thread_info *curren - #define TIF_NEED_RESCHED 2 /* rescheduling necessary */ - #define TIF_SINGLESTEP 3 /* restore singlestep on return to user mode */ - #define TIF_SYSCALL_TRACEPOINT 4 /* syscall tracepoint instrumentation */ --#define TIF_MEMDIE 5 /* is terminating due to OOM killer */ -+#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ - #define TIF_RESTORE_SIGMASK 6 /* restore signal mask in do_signal() */ - #define TIF_NOTIFY_RESUME 7 /* callback before returning to user */ - #define TIF_DB_DISABLED 8 /* debug trap disabled for syscall */ - #define TIF_SYSCALL_AUDIT 9 /* syscall auditing active */ - #define TIF_SECCOMP 10 /* secure computing */ -+#define TIF_MEMDIE 11 /* is terminating due to OOM killer */ - - #define _TIF_SYSCALL_TRACE (1< Signed-off-by: Andrew Morton Signed-off-by: Sasha Levin --- - include/linux/eventfd.h | 2 +- + include/linux/eventfd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -diff --git a/include/linux/eventfd.h b/include/linux/eventfd.h -index dc4fd8a6644d..3482f9365a4d 100644 --- a/include/linux/eventfd.h +++ b/include/linux/eventfd.h -@@ -61,7 +61,7 @@ static inline struct eventfd_ctx *eventfd_ctx_fdget(int fd) +@@ -62,7 +62,7 @@ static inline struct eventfd_ctx *eventf return ERR_PTR(-ENOSYS); } @@ -36,6 +34,3 @@ index dc4fd8a6644d..3482f9365a4d 100644 { return -ENOSYS; } --- -2.35.1 - diff --git a/next-5.10/f2fs-avoid-victim-selection-from-previous-victim-sec.patch b/queue-5.10/f2fs-avoid-victim-selection-from-previous-victim-sec.patch similarity index 100% rename from next-5.10/f2fs-avoid-victim-selection-from-previous-victim-sec.patch rename to queue-5.10/f2fs-avoid-victim-selection-from-previous-victim-sec.patch diff --git a/next-5.10/f2fs-fix-normal-discard-process.patch b/queue-5.10/f2fs-fix-normal-discard-process.patch similarity index 100% rename from next-5.10/f2fs-fix-normal-discard-process.patch rename to queue-5.10/f2fs-fix-normal-discard-process.patch diff --git a/next-5.10/f2fs-fix-the-race-condition-of-resize-flag-between-r.patch b/queue-5.10/f2fs-fix-the-race-condition-of-resize-flag-between-r.patch similarity index 100% rename from next-5.10/f2fs-fix-the-race-condition-of-resize-flag-between-r.patch rename to queue-5.10/f2fs-fix-the-race-condition-of-resize-flag-between-r.patch diff --git a/next-5.10/f2fs-should-put-a-page-when-checking-the-summary-info.patch b/queue-5.10/f2fs-should-put-a-page-when-checking-the-summary-info.patch similarity index 100% rename from next-5.10/f2fs-should-put-a-page-when-checking-the-summary-info.patch rename to queue-5.10/f2fs-should-put-a-page-when-checking-the-summary-info.patch diff --git a/next-5.10/fbdev-pm2fb-fix-missing-pci_disable_device.patch b/queue-5.10/fbdev-pm2fb-fix-missing-pci_disable_device.patch similarity index 100% rename from next-5.10/fbdev-pm2fb-fix-missing-pci_disable_device.patch rename to queue-5.10/fbdev-pm2fb-fix-missing-pci_disable_device.patch diff --git a/next-5.10/fbdev-ssd1307fb-drop-optional-dependency.patch b/queue-5.10/fbdev-ssd1307fb-drop-optional-dependency.patch similarity index 100% rename from next-5.10/fbdev-ssd1307fb-drop-optional-dependency.patch rename to queue-5.10/fbdev-ssd1307fb-drop-optional-dependency.patch diff --git a/next-5.10/fbdev-uvesafb-fixes-an-error-handling-path-in-uvesaf.patch b/queue-5.10/fbdev-uvesafb-fixes-an-error-handling-path-in-uvesaf.patch similarity index 100% rename from next-5.10/fbdev-uvesafb-fixes-an-error-handling-path-in-uvesaf.patch rename to queue-5.10/fbdev-uvesafb-fixes-an-error-handling-path-in-uvesaf.patch diff --git a/next-5.10/fbdev-vermilion-decrease-reference-count-in-error-pa.patch b/queue-5.10/fbdev-vermilion-decrease-reference-count-in-error-pa.patch similarity index 100% rename from next-5.10/fbdev-vermilion-decrease-reference-count-in-error-pa.patch rename to queue-5.10/fbdev-vermilion-decrease-reference-count-in-error-pa.patch diff --git a/next-5.10/fbdev-via-fix-error-in-via_core_init.patch b/queue-5.10/fbdev-via-fix-error-in-via_core_init.patch similarity index 100% rename from next-5.10/fbdev-via-fix-error-in-via_core_init.patch rename to queue-5.10/fbdev-via-fix-error-in-via_core_init.patch diff --git a/next-5.10/firmware-raspberrypi-fix-possible-memory-leak-in-rpi.patch b/queue-5.10/firmware-raspberrypi-fix-possible-memory-leak-in-rpi.patch similarity index 100% rename from next-5.10/firmware-raspberrypi-fix-possible-memory-leak-in-rpi.patch rename to queue-5.10/firmware-raspberrypi-fix-possible-memory-leak-in-rpi.patch diff --git a/next-5.10/fs-don-t-audit-the-capability-check-in-simple_xattr_.patch b/queue-5.10/fs-don-t-audit-the-capability-check-in-simple_xattr_.patch similarity index 100% rename from next-5.10/fs-don-t-audit-the-capability-check-in-simple_xattr_.patch rename to queue-5.10/fs-don-t-audit-the-capability-check-in-simple_xattr_.patch diff --git a/next-5.10/fs-jfs-fix-shift-out-of-bounds-in-dballocag.patch b/queue-5.10/fs-jfs-fix-shift-out-of-bounds-in-dballocag.patch similarity index 100% rename from next-5.10/fs-jfs-fix-shift-out-of-bounds-in-dballocag.patch rename to queue-5.10/fs-jfs-fix-shift-out-of-bounds-in-dballocag.patch diff --git a/next-5.10/fs-jfs-fix-shift-out-of-bounds-in-dbdiscardag.patch b/queue-5.10/fs-jfs-fix-shift-out-of-bounds-in-dbdiscardag.patch similarity index 100% rename from next-5.10/fs-jfs-fix-shift-out-of-bounds-in-dbdiscardag.patch rename to queue-5.10/fs-jfs-fix-shift-out-of-bounds-in-dbdiscardag.patch diff --git a/next-5.10/fs-sysv-fix-sysv_nblocks-returns-wrong-value.patch b/queue-5.10/fs-sysv-fix-sysv_nblocks-returns-wrong-value.patch similarity index 100% rename from next-5.10/fs-sysv-fix-sysv_nblocks-returns-wrong-value.patch rename to queue-5.10/fs-sysv-fix-sysv_nblocks-returns-wrong-value.patch diff --git a/next-5.10/futex-move-to-kernel-futex.patch b/queue-5.10/futex-move-to-kernel-futex.patch similarity index 100% rename from next-5.10/futex-move-to-kernel-futex.patch rename to queue-5.10/futex-move-to-kernel-futex.patch diff --git a/next-5.10/futex-resend-potentially-swallowed-owner-death-notif.patch b/queue-5.10/futex-resend-potentially-swallowed-owner-death-notif.patch similarity index 100% rename from next-5.10/futex-resend-potentially-swallowed-owner-death-notif.patch rename to queue-5.10/futex-resend-potentially-swallowed-owner-death-notif.patch diff --git a/next-5.10/gcov-add-support-for-checksum-field.patch b/queue-5.10/gcov-add-support-for-checksum-field.patch similarity index 100% rename from next-5.10/gcov-add-support-for-checksum-field.patch rename to queue-5.10/gcov-add-support-for-checksum-field.patch diff --git a/next-5.10/genirq-add-irqf_no_autoen-for-request_irq-nmi.patch b/queue-5.10/genirq-add-irqf_no_autoen-for-request_irq-nmi.patch similarity index 100% rename from next-5.10/genirq-add-irqf_no_autoen-for-request_irq-nmi.patch rename to queue-5.10/genirq-add-irqf_no_autoen-for-request_irq-nmi.patch diff --git a/next-5.10/genirq-irqdesc-don-t-try-to-remove-non-existing-sysf.patch b/queue-5.10/genirq-irqdesc-don-t-try-to-remove-non-existing-sysf.patch similarity index 100% rename from next-5.10/genirq-irqdesc-don-t-try-to-remove-non-existing-sysf.patch rename to queue-5.10/genirq-irqdesc-don-t-try-to-remove-non-existing-sysf.patch diff --git a/next-5.10/gpiolib-cdev-fix-null-pointer-dereferences.patch b/queue-5.10/gpiolib-cdev-fix-null-pointer-dereferences.patch similarity index 100% rename from next-5.10/gpiolib-cdev-fix-null-pointer-dereferences.patch rename to queue-5.10/gpiolib-cdev-fix-null-pointer-dereferences.patch diff --git a/next-5.10/gpiolib-get-rid-of-redundant-else.patch b/queue-5.10/gpiolib-get-rid-of-redundant-else.patch similarity index 100% rename from next-5.10/gpiolib-get-rid-of-redundant-else.patch rename to queue-5.10/gpiolib-get-rid-of-redundant-else.patch diff --git a/next-5.10/hamradio-baycom_epp-fix-return-type-of-baycom_send_p.patch b/queue-5.10/hamradio-baycom_epp-fix-return-type-of-baycom_send_p.patch similarity index 100% rename from next-5.10/hamradio-baycom_epp-fix-return-type-of-baycom_send_p.patch rename to queue-5.10/hamradio-baycom_epp-fix-return-type-of-baycom_send_p.patch diff --git a/next-5.10/hamradio-don-t-call-dev_kfree_skb-under-spin_lock_ir.patch b/queue-5.10/hamradio-don-t-call-dev_kfree_skb-under-spin_lock_ir.patch similarity index 100% rename from next-5.10/hamradio-don-t-call-dev_kfree_skb-under-spin_lock_ir.patch rename to queue-5.10/hamradio-don-t-call-dev_kfree_skb-under-spin_lock_ir.patch diff --git a/next-5.10/hfs-fix-oob-read-in-__hfs_brec_find.patch b/queue-5.10/hfs-fix-oob-read-in-__hfs_brec_find.patch similarity index 100% rename from next-5.10/hfs-fix-oob-read-in-__hfs_brec_find.patch rename to queue-5.10/hfs-fix-oob-read-in-__hfs_brec_find.patch diff --git a/next-5.10/hfs-fix-oob-write-in-hfs_asc2mac.patch b/queue-5.10/hfs-fix-oob-write-in-hfs_asc2mac.patch similarity index 100% rename from next-5.10/hfs-fix-oob-write-in-hfs_asc2mac.patch rename to queue-5.10/hfs-fix-oob-write-in-hfs_asc2mac.patch diff --git a/next-5.10/hfsplus-fix-bug-causing-custom-uid-and-gid-being-unable-to-be-assigned-with-mount.patch b/queue-5.10/hfsplus-fix-bug-causing-custom-uid-and-gid-being-unable-to-be-assigned-with-mount.patch similarity index 100% rename from next-5.10/hfsplus-fix-bug-causing-custom-uid-and-gid-being-unable-to-be-assigned-with-mount.patch rename to queue-5.10/hfsplus-fix-bug-causing-custom-uid-and-gid-being-unable-to-be-assigned-with-mount.patch diff --git a/next-5.10/hid-hid-sensor-custom-set-fixed-size-for-custom-attr.patch b/queue-5.10/hid-hid-sensor-custom-set-fixed-size-for-custom-attr.patch similarity index 100% rename from next-5.10/hid-hid-sensor-custom-set-fixed-size-for-custom-attr.patch rename to queue-5.10/hid-hid-sensor-custom-set-fixed-size-for-custom-attr.patch diff --git a/next-5.10/hid-mcp2221-don-t-connect-hidraw.patch b/queue-5.10/hid-mcp2221-don-t-connect-hidraw.patch similarity index 100% rename from next-5.10/hid-mcp2221-don-t-connect-hidraw.patch rename to queue-5.10/hid-mcp2221-don-t-connect-hidraw.patch diff --git a/next-5.10/hid-multitouch-fix-asus-expertbook-p2-p2451fa-trackp.patch b/queue-5.10/hid-multitouch-fix-asus-expertbook-p2-p2451fa-trackp.patch similarity index 100% rename from next-5.10/hid-multitouch-fix-asus-expertbook-p2-p2451fa-trackp.patch rename to queue-5.10/hid-multitouch-fix-asus-expertbook-p2-p2451fa-trackp.patch diff --git a/next-5.10/hid-plantronics-additional-pids-for-double-volume-ke.patch b/queue-5.10/hid-plantronics-additional-pids-for-double-volume-ke.patch similarity index 100% rename from next-5.10/hid-plantronics-additional-pids-for-double-volume-ke.patch rename to queue-5.10/hid-plantronics-additional-pids-for-double-volume-ke.patch diff --git a/next-5.10/hid-wacom-ensure-bootloader-pid-is-usable-in-hidraw-mode.patch b/queue-5.10/hid-wacom-ensure-bootloader-pid-is-usable-in-hidraw-mode.patch similarity index 100% rename from next-5.10/hid-wacom-ensure-bootloader-pid-is-usable-in-hidraw-mode.patch rename to queue-5.10/hid-wacom-ensure-bootloader-pid-is-usable-in-hidraw-mode.patch diff --git a/next-5.10/hsi-omap_ssi_core-fix-error-handling-in-ssi_init.patch b/queue-5.10/hsi-omap_ssi_core-fix-error-handling-in-ssi_init.patch similarity index 100% rename from next-5.10/hsi-omap_ssi_core-fix-error-handling-in-ssi_init.patch rename to queue-5.10/hsi-omap_ssi_core-fix-error-handling-in-ssi_init.patch diff --git a/next-5.10/hsi-omap_ssi_core-fix-possible-memory-leak-in-ssi_pr.patch b/queue-5.10/hsi-omap_ssi_core-fix-possible-memory-leak-in-ssi_pr.patch similarity index 100% rename from next-5.10/hsi-omap_ssi_core-fix-possible-memory-leak-in-ssi_pr.patch rename to queue-5.10/hsi-omap_ssi_core-fix-possible-memory-leak-in-ssi_pr.patch diff --git a/next-5.10/hsi-omap_ssi_core-fix-unbalanced-pm_runtime_disable.patch b/queue-5.10/hsi-omap_ssi_core-fix-unbalanced-pm_runtime_disable.patch similarity index 100% rename from next-5.10/hsi-omap_ssi_core-fix-unbalanced-pm_runtime_disable.patch rename to queue-5.10/hsi-omap_ssi_core-fix-unbalanced-pm_runtime_disable.patch diff --git a/next-5.10/hsr-add-a-rcu-read-lock-to-hsr_forward_skb.patch b/queue-5.10/hsr-add-a-rcu-read-lock-to-hsr_forward_skb.patch similarity index 100% rename from next-5.10/hsr-add-a-rcu-read-lock-to-hsr_forward_skb.patch rename to queue-5.10/hsr-add-a-rcu-read-lock-to-hsr_forward_skb.patch diff --git a/next-5.10/hsr-disable-netpoll.patch b/queue-5.10/hsr-disable-netpoll.patch similarity index 100% rename from next-5.10/hsr-disable-netpoll.patch rename to queue-5.10/hsr-disable-netpoll.patch diff --git a/next-5.10/hsr-synchronize-sending-frames-to-have-always-increm.patch b/queue-5.10/hsr-synchronize-sending-frames-to-have-always-increm.patch similarity index 100% rename from next-5.10/hsr-synchronize-sending-frames-to-have-always-increm.patch rename to queue-5.10/hsr-synchronize-sending-frames-to-have-always-increm.patch diff --git a/next-5.10/hsr-synchronize-sequence-number-updates.patch b/queue-5.10/hsr-synchronize-sequence-number-updates.patch similarity index 100% rename from next-5.10/hsr-synchronize-sequence-number-updates.patch rename to queue-5.10/hsr-synchronize-sequence-number-updates.patch diff --git a/next-5.10/hugetlbfs-fix-null-ptr-deref-in-hugetlbfs_parse_para.patch b/queue-5.10/hugetlbfs-fix-null-ptr-deref-in-hugetlbfs_parse_para.patch similarity index 100% rename from next-5.10/hugetlbfs-fix-null-ptr-deref-in-hugetlbfs_parse_para.patch rename to queue-5.10/hugetlbfs-fix-null-ptr-deref-in-hugetlbfs_parse_para.patch diff --git a/next-5.10/hwmon-jc42-convert-register-access-and-caching-to-re.patch b/queue-5.10/hwmon-jc42-convert-register-access-and-caching-to-re.patch similarity index 100% rename from next-5.10/hwmon-jc42-convert-register-access-and-caching-to-re.patch rename to queue-5.10/hwmon-jc42-convert-register-access-and-caching-to-re.patch diff --git a/next-5.10/hwmon-jc42-fix-missing-unlock-on-error-in-jc42_write.patch b/queue-5.10/hwmon-jc42-fix-missing-unlock-on-error-in-jc42_write.patch similarity index 100% rename from next-5.10/hwmon-jc42-fix-missing-unlock-on-error-in-jc42_write.patch rename to queue-5.10/hwmon-jc42-fix-missing-unlock-on-error-in-jc42_write.patch diff --git a/next-5.10/hwmon-jc42-restore-the-min-max-critical-temperatures.patch b/queue-5.10/hwmon-jc42-restore-the-min-max-critical-temperatures.patch similarity index 100% rename from next-5.10/hwmon-jc42-restore-the-min-max-critical-temperatures.patch rename to queue-5.10/hwmon-jc42-restore-the-min-max-critical-temperatures.patch diff --git a/next-5.10/hwrng-amd-fix-pci-device-refcount-leak.patch b/queue-5.10/hwrng-amd-fix-pci-device-refcount-leak.patch similarity index 100% rename from next-5.10/hwrng-amd-fix-pci-device-refcount-leak.patch rename to queue-5.10/hwrng-amd-fix-pci-device-refcount-leak.patch diff --git a/next-5.10/hwrng-geode-fix-pci-device-refcount-leak.patch b/queue-5.10/hwrng-geode-fix-pci-device-refcount-leak.patch similarity index 100% rename from next-5.10/hwrng-geode-fix-pci-device-refcount-leak.patch rename to queue-5.10/hwrng-geode-fix-pci-device-refcount-leak.patch diff --git a/next-5.10/i2c-ismt-fix-an-out-of-bounds-bug-in-ismt_access.patch b/queue-5.10/i2c-ismt-fix-an-out-of-bounds-bug-in-ismt_access.patch similarity index 100% rename from next-5.10/i2c-ismt-fix-an-out-of-bounds-bug-in-ismt_access.patch rename to queue-5.10/i2c-ismt-fix-an-out-of-bounds-bug-in-ismt_access.patch diff --git a/next-5.10/i2c-mux-reg-check-return-value-after-calling-platfor.patch b/queue-5.10/i2c-mux-reg-check-return-value-after-calling-platfor.patch similarity index 100% rename from next-5.10/i2c-mux-reg-check-return-value-after-calling-platfor.patch rename to queue-5.10/i2c-mux-reg-check-return-value-after-calling-platfor.patch diff --git a/next-5.10/i2c-pxa-pci-fix-missing-pci_disable_device-on-error-.patch b/queue-5.10/i2c-pxa-pci-fix-missing-pci_disable_device-on-error-.patch similarity index 100% rename from next-5.10/i2c-pxa-pci-fix-missing-pci_disable_device-on-error-.patch rename to queue-5.10/i2c-pxa-pci-fix-missing-pci_disable_device-on-error-.patch diff --git a/next-5.10/ib-ipoib-fix-queue-count-inconsistency-for-pkey-chil.patch b/queue-5.10/ib-ipoib-fix-queue-count-inconsistency-for-pkey-chil.patch similarity index 100% rename from next-5.10/ib-ipoib-fix-queue-count-inconsistency-for-pkey-chil.patch rename to queue-5.10/ib-ipoib-fix-queue-count-inconsistency-for-pkey-chil.patch diff --git a/next-5.10/igb-do-not-free-q_vector-unless-new-one-was-allocate.patch b/queue-5.10/igb-do-not-free-q_vector-unless-new-one-was-allocate.patch similarity index 100% rename from next-5.10/igb-do-not-free-q_vector-unless-new-one-was-allocate.patch rename to queue-5.10/igb-do-not-free-q_vector-unless-new-one-was-allocate.patch diff --git a/next-5.10/igc-add-checking-for-basetime-less-than-zero.patch b/queue-5.10/igc-add-checking-for-basetime-less-than-zero.patch similarity index 100% rename from next-5.10/igc-add-checking-for-basetime-less-than-zero.patch rename to queue-5.10/igc-add-checking-for-basetime-less-than-zero.patch diff --git a/next-5.10/igc-enhance-qbv-scheduling-by-using-first-flag-bit.patch b/queue-5.10/igc-enhance-qbv-scheduling-by-using-first-flag-bit.patch similarity index 100% rename from next-5.10/igc-enhance-qbv-scheduling-by-using-first-flag-bit.patch rename to queue-5.10/igc-enhance-qbv-scheduling-by-using-first-flag-bit.patch diff --git a/next-5.10/igc-lift-taprio-schedule-restriction.patch b/queue-5.10/igc-lift-taprio-schedule-restriction.patch similarity index 100% rename from next-5.10/igc-lift-taprio-schedule-restriction.patch rename to queue-5.10/igc-lift-taprio-schedule-restriction.patch diff --git a/next-5.10/igc-recalculate-qbv-end_time-by-considering-cycle-ti.patch b/queue-5.10/igc-recalculate-qbv-end_time-by-considering-cycle-ti.patch similarity index 100% rename from next-5.10/igc-recalculate-qbv-end_time-by-considering-cycle-ti.patch rename to queue-5.10/igc-recalculate-qbv-end_time-by-considering-cycle-ti.patch diff --git a/next-5.10/igc-set-qbv-start_time-and-end_time-to-end_time-if-n.patch b/queue-5.10/igc-set-qbv-start_time-and-end_time-to-end_time-if-n.patch similarity index 100% rename from next-5.10/igc-set-qbv-start_time-and-end_time-to-end_time-if-n.patch rename to queue-5.10/igc-set-qbv-start_time-and-end_time-to-end_time-if-n.patch diff --git a/next-5.10/igc-use-strict-cycles-for-qbv-scheduling.patch b/queue-5.10/igc-use-strict-cycles-for-qbv-scheduling.patch similarity index 100% rename from next-5.10/igc-use-strict-cycles-for-qbv-scheduling.patch rename to queue-5.10/igc-use-strict-cycles-for-qbv-scheduling.patch diff --git a/next-5.10/iio-adc-ad_sigma_delta-do-not-use-internal-iio_dev-lock.patch b/queue-5.10/iio-adc-ad_sigma_delta-do-not-use-internal-iio_dev-lock.patch similarity index 100% rename from next-5.10/iio-adc-ad_sigma_delta-do-not-use-internal-iio_dev-lock.patch rename to queue-5.10/iio-adc-ad_sigma_delta-do-not-use-internal-iio_dev-lock.patch diff --git a/next-5.10/iio-adc128s052-add-proper-.data-members-in-adc128_of_match-table.patch b/queue-5.10/iio-adc128s052-add-proper-.data-members-in-adc128_of_match-table.patch similarity index 100% rename from next-5.10/iio-adc128s052-add-proper-.data-members-in-adc128_of_match-table.patch rename to queue-5.10/iio-adc128s052-add-proper-.data-members-in-adc128_of_match-table.patch diff --git a/next-5.10/iio-adis-add-__adis_enable_irq-implementation.patch b/queue-5.10/iio-adis-add-__adis_enable_irq-implementation.patch similarity index 100% rename from next-5.10/iio-adis-add-__adis_enable_irq-implementation.patch rename to queue-5.10/iio-adis-add-__adis_enable_irq-implementation.patch diff --git a/next-5.10/iio-adis-handle-devices-that-cannot-unmask-the-drdy-.patch b/queue-5.10/iio-adis-handle-devices-that-cannot-unmask-the-drdy-.patch similarity index 100% rename from next-5.10/iio-adis-handle-devices-that-cannot-unmask-the-drdy-.patch rename to queue-5.10/iio-adis-handle-devices-that-cannot-unmask-the-drdy-.patch diff --git a/next-5.10/iio-adis-stylistic-changes.patch b/queue-5.10/iio-adis-stylistic-changes.patch similarity index 100% rename from next-5.10/iio-adis-stylistic-changes.patch rename to queue-5.10/iio-adis-stylistic-changes.patch diff --git a/next-5.10/iio-imu-adis-move-exports-into-iio_adislib-namespace.patch b/queue-5.10/iio-imu-adis-move-exports-into-iio_adislib-namespace.patch similarity index 100% rename from next-5.10/iio-imu-adis-move-exports-into-iio_adislib-namespace.patch rename to queue-5.10/iio-imu-adis-move-exports-into-iio_adislib-namespace.patch diff --git a/next-5.10/iio-imu-adis-use-irqf_no_autoen-instead-of-irq-reque.patch b/queue-5.10/iio-imu-adis-use-irqf_no_autoen-instead-of-irq-reque.patch similarity index 100% rename from next-5.10/iio-imu-adis-use-irqf_no_autoen-instead-of-irq-reque.patch rename to queue-5.10/iio-imu-adis-use-irqf_no_autoen-instead-of-irq-reque.patch diff --git a/next-5.10/iio-temperature-ltc2983-make-bulk-write-buffer-dma-s.patch b/queue-5.10/iio-temperature-ltc2983-make-bulk-write-buffer-dma-s.patch similarity index 100% rename from next-5.10/iio-temperature-ltc2983-make-bulk-write-buffer-dma-s.patch rename to queue-5.10/iio-temperature-ltc2983-make-bulk-write-buffer-dma-s.patch diff --git a/next-5.10/ima-fix-fall-through-warnings-for-clang.patch b/queue-5.10/ima-fix-fall-through-warnings-for-clang.patch similarity index 100% rename from next-5.10/ima-fix-fall-through-warnings-for-clang.patch rename to queue-5.10/ima-fix-fall-through-warnings-for-clang.patch diff --git a/next-5.10/ima-fix-misuse-of-dereference-of-pointer-in-template.patch b/queue-5.10/ima-fix-misuse-of-dereference-of-pointer-in-template.patch similarity index 100% rename from next-5.10/ima-fix-misuse-of-dereference-of-pointer-in-template.patch rename to queue-5.10/ima-fix-misuse-of-dereference-of-pointer-in-template.patch diff --git a/next-5.10/ima-handle-estale-returned-by-ima_filter_rule_match.patch b/queue-5.10/ima-handle-estale-returned-by-ima_filter_rule_match.patch similarity index 100% rename from next-5.10/ima-handle-estale-returned-by-ima_filter_rule_match.patch rename to queue-5.10/ima-handle-estale-returned-by-ima_filter_rule_match.patch diff --git a/next-5.10/ima-simplify-ima_lsm_copy_rule.patch b/queue-5.10/ima-simplify-ima_lsm_copy_rule.patch similarity index 100% rename from next-5.10/ima-simplify-ima_lsm_copy_rule.patch rename to queue-5.10/ima-simplify-ima_lsm_copy_rule.patch diff --git a/next-5.10/include-uapi-linux-swab-fix-potentially-missing-__al.patch b/queue-5.10/include-uapi-linux-swab-fix-potentially-missing-__al.patch similarity index 100% rename from next-5.10/include-uapi-linux-swab-fix-potentially-missing-__al.patch rename to queue-5.10/include-uapi-linux-swab-fix-potentially-missing-__al.patch diff --git a/next-5.10/inet-add-read_once-sk-sk_bound_dev_if-in-inet_csk_bi.patch b/queue-5.10/inet-add-read_once-sk-sk_bound_dev_if-in-inet_csk_bi.patch similarity index 100% rename from next-5.10/inet-add-read_once-sk-sk_bound_dev_if-in-inet_csk_bi.patch rename to queue-5.10/inet-add-read_once-sk-sk_bound_dev_if-in-inet_csk_bi.patch diff --git a/next-5.10/input-elants_i2c-properly-handle-the-reset-gpio-when.patch b/queue-5.10/input-elants_i2c-properly-handle-the-reset-gpio-when.patch similarity index 100% rename from next-5.10/input-elants_i2c-properly-handle-the-reset-gpio-when.patch rename to queue-5.10/input-elants_i2c-properly-handle-the-reset-gpio-when.patch diff --git a/next-5.10/input-joystick-fix-kconfig-warning-for-joystick_adc.patch b/queue-5.10/input-joystick-fix-kconfig-warning-for-joystick_adc.patch similarity index 100% rename from next-5.10/input-joystick-fix-kconfig-warning-for-joystick_adc.patch rename to queue-5.10/input-joystick-fix-kconfig-warning-for-joystick_adc.patch diff --git a/next-5.10/integrity-fix-memory-leakage-in-keyring-allocation-e.patch b/queue-5.10/integrity-fix-memory-leakage-in-keyring-allocation-e.patch similarity index 100% rename from next-5.10/integrity-fix-memory-leakage-in-keyring-allocation-e.patch rename to queue-5.10/integrity-fix-memory-leakage-in-keyring-allocation-e.patch diff --git a/next-5.10/iommu-amd-fix-pci-device-refcount-leak-in-ppr_notifi.patch b/queue-5.10/iommu-amd-fix-pci-device-refcount-leak-in-ppr_notifi.patch similarity index 100% rename from next-5.10/iommu-amd-fix-pci-device-refcount-leak-in-ppr_notifi.patch rename to queue-5.10/iommu-amd-fix-pci-device-refcount-leak-in-ppr_notifi.patch diff --git a/next-5.10/iommu-fsl_pamu-fix-resource-leak-in-fsl_pamu_probe.patch b/queue-5.10/iommu-fsl_pamu-fix-resource-leak-in-fsl_pamu_probe.patch similarity index 100% rename from next-5.10/iommu-fsl_pamu-fix-resource-leak-in-fsl_pamu_probe.patch rename to queue-5.10/iommu-fsl_pamu-fix-resource-leak-in-fsl_pamu_probe.patch diff --git a/next-5.10/iommu-sun50i-consider-all-fault-sources-for-reset.patch b/queue-5.10/iommu-sun50i-consider-all-fault-sources-for-reset.patch similarity index 100% rename from next-5.10/iommu-sun50i-consider-all-fault-sources-for-reset.patch rename to queue-5.10/iommu-sun50i-consider-all-fault-sources-for-reset.patch diff --git a/next-5.10/iommu-sun50i-fix-flush-size.patch b/queue-5.10/iommu-sun50i-fix-flush-size.patch similarity index 100% rename from next-5.10/iommu-sun50i-fix-flush-size.patch rename to queue-5.10/iommu-sun50i-fix-flush-size.patch diff --git a/next-5.10/iommu-sun50i-fix-r-w-permission-check.patch b/queue-5.10/iommu-sun50i-fix-r-w-permission-check.patch similarity index 100% rename from next-5.10/iommu-sun50i-fix-r-w-permission-check.patch rename to queue-5.10/iommu-sun50i-fix-r-w-permission-check.patch diff --git a/next-5.10/iommu-sun50i-fix-reset-release.patch b/queue-5.10/iommu-sun50i-fix-reset-release.patch similarity index 100% rename from next-5.10/iommu-sun50i-fix-reset-release.patch rename to queue-5.10/iommu-sun50i-fix-reset-release.patch diff --git a/next-5.10/iommu-sun50i-remove-iommu_domain_identity.patch b/queue-5.10/iommu-sun50i-remove-iommu_domain_identity.patch similarity index 100% rename from next-5.10/iommu-sun50i-remove-iommu_domain_identity.patch rename to queue-5.10/iommu-sun50i-remove-iommu_domain_identity.patch diff --git a/next-5.10/ipmi-fix-memleak-when-unload-ipmi-driver.patch b/queue-5.10/ipmi-fix-memleak-when-unload-ipmi-driver.patch similarity index 100% rename from next-5.10/ipmi-fix-memleak-when-unload-ipmi-driver.patch rename to queue-5.10/ipmi-fix-memleak-when-unload-ipmi-driver.patch diff --git a/next-5.10/irqchip-gic-pm-use-pm_runtime_resume_and_get-in-gic_.patch b/queue-5.10/irqchip-gic-pm-use-pm_runtime_resume_and_get-in-gic_.patch similarity index 100% rename from next-5.10/irqchip-gic-pm-use-pm_runtime_resume_and_get-in-gic_.patch rename to queue-5.10/irqchip-gic-pm-use-pm_runtime_resume_and_get-in-gic_.patch diff --git a/next-5.10/kbuild-refactor-single-builds-of-.ko.patch b/queue-5.10/kbuild-refactor-single-builds-of-.ko.patch similarity index 100% rename from next-5.10/kbuild-refactor-single-builds-of-.ko.patch rename to queue-5.10/kbuild-refactor-single-builds-of-.ko.patch diff --git a/next-5.10/kbuild-remove-unneeded-mkdir-for-external-modules_in.patch b/queue-5.10/kbuild-remove-unneeded-mkdir-for-external-modules_in.patch similarity index 100% rename from next-5.10/kbuild-remove-unneeded-mkdir-for-external-modules_in.patch rename to queue-5.10/kbuild-remove-unneeded-mkdir-for-external-modules_in.patch diff --git a/next-5.10/kbuild-unify-modules-_install-for-in-tree-and-extern.patch b/queue-5.10/kbuild-unify-modules-_install-for-in-tree-and-extern.patch similarity index 100% rename from next-5.10/kbuild-unify-modules-_install-for-in-tree-and-extern.patch rename to queue-5.10/kbuild-unify-modules-_install-for-in-tree-and-extern.patch diff --git a/next-5.10/kcsan-instrument-memcpy-memset-memmove-with-newer-clang.patch b/queue-5.10/kcsan-instrument-memcpy-memset-memmove-with-newer-clang.patch similarity index 100% rename from next-5.10/kcsan-instrument-memcpy-memset-memmove-with-newer-clang.patch rename to queue-5.10/kcsan-instrument-memcpy-memset-memmove-with-newer-clang.patch diff --git a/next-5.10/lib-debugobjects-fix-stat-count-and-optimize-debug_o.patch b/queue-5.10/lib-debugobjects-fix-stat-count-and-optimize-debug_o.patch similarity index 100% rename from next-5.10/lib-debugobjects-fix-stat-count-and-optimize-debug_o.patch rename to queue-5.10/lib-debugobjects-fix-stat-count-and-optimize-debug_o.patch diff --git a/next-5.10/lib-fonts-fix-undefined-behavior-in-bit-shift-for-ge.patch b/queue-5.10/lib-fonts-fix-undefined-behavior-in-bit-shift-for-ge.patch similarity index 100% rename from next-5.10/lib-fonts-fix-undefined-behavior-in-bit-shift-for-ge.patch rename to queue-5.10/lib-fonts-fix-undefined-behavior-in-bit-shift-for-ge.patch diff --git a/next-5.10/lib-notifier-error-inject-fix-error-when-writing-err.patch b/queue-5.10/lib-notifier-error-inject-fix-error-when-writing-err.patch similarity index 100% rename from next-5.10/lib-notifier-error-inject-fix-error-when-writing-err.patch rename to queue-5.10/lib-notifier-error-inject-fix-error-when-writing-err.patch diff --git a/next-5.10/libbpf-avoid-enum-forward-declarations-in-public-api.patch b/queue-5.10/libbpf-avoid-enum-forward-declarations-in-public-api.patch similarity index 100% rename from next-5.10/libbpf-avoid-enum-forward-declarations-in-public-api.patch rename to queue-5.10/libbpf-avoid-enum-forward-declarations-in-public-api.patch diff --git a/next-5.10/libbpf-fix-null-pointer-dereference-in-find_prog_by_.patch b/queue-5.10/libbpf-fix-null-pointer-dereference-in-find_prog_by_.patch similarity index 100% rename from next-5.10/libbpf-fix-null-pointer-dereference-in-find_prog_by_.patch rename to queue-5.10/libbpf-fix-null-pointer-dereference-in-find_prog_by_.patch diff --git a/next-5.10/libbpf-fix-use-after-free-in-btf_dump_name_dups.patch b/queue-5.10/libbpf-fix-use-after-free-in-btf_dump_name_dups.patch similarity index 100% rename from next-5.10/libbpf-fix-use-after-free-in-btf_dump_name_dups.patch rename to queue-5.10/libbpf-fix-use-after-free-in-btf_dump_name_dups.patch diff --git a/next-5.10/libfs-add-define_simple_attribute_signed-for-signed-.patch b/queue-5.10/libfs-add-define_simple_attribute_signed-for-signed-.patch similarity index 100% rename from next-5.10/libfs-add-define_simple_attribute_signed-for-signed-.patch rename to queue-5.10/libfs-add-define_simple_attribute_signed-for-signed-.patch diff --git a/next-5.10/loadpin-ignore-the-contents-argument-of-the-lsm-hook.patch b/queue-5.10/loadpin-ignore-the-contents-argument-of-the-lsm-hook.patch similarity index 100% rename from next-5.10/loadpin-ignore-the-contents-argument-of-the-lsm-hook.patch rename to queue-5.10/loadpin-ignore-the-contents-argument-of-the-lsm-hook.patch diff --git a/next-5.10/macintosh-fix-possible-memory-leak-in-macio_add_one_.patch b/queue-5.10/macintosh-fix-possible-memory-leak-in-macio_add_one_.patch similarity index 100% rename from next-5.10/macintosh-fix-possible-memory-leak-in-macio_add_one_.patch rename to queue-5.10/macintosh-fix-possible-memory-leak-in-macio_add_one_.patch diff --git a/next-5.10/macintosh-macio-adb-check-the-return-value-of-iorema.patch b/queue-5.10/macintosh-macio-adb-check-the-return-value-of-iorema.patch similarity index 100% rename from next-5.10/macintosh-macio-adb-check-the-return-value-of-iorema.patch rename to queue-5.10/macintosh-macio-adb-check-the-return-value-of-iorema.patch diff --git a/next-5.10/mailbox-zynq-ipi-fix-error-handling-while-device_reg.patch b/queue-5.10/mailbox-zynq-ipi-fix-error-handling-while-device_reg.patch similarity index 100% rename from next-5.10/mailbox-zynq-ipi-fix-error-handling-while-device_reg.patch rename to queue-5.10/mailbox-zynq-ipi-fix-error-handling-while-device_reg.patch diff --git a/next-5.10/mcb-mcb-parse-fix-error-handing-in-chameleon_parse_g.patch b/queue-5.10/mcb-mcb-parse-fix-error-handing-in-chameleon_parse_g.patch similarity index 100% rename from next-5.10/mcb-mcb-parse-fix-error-handing-in-chameleon_parse_g.patch rename to queue-5.10/mcb-mcb-parse-fix-error-handing-in-chameleon_parse_g.patch diff --git a/next-5.10/md-fix-a-crash-in-mempool_free.patch b/queue-5.10/md-fix-a-crash-in-mempool_free.patch similarity index 100% rename from next-5.10/md-fix-a-crash-in-mempool_free.patch rename to queue-5.10/md-fix-a-crash-in-mempool_free.patch diff --git a/next-5.10/md-raid1-stop-mdx_raid1-thread-when-raid1-array-run-.patch b/queue-5.10/md-raid1-stop-mdx_raid1-thread-when-raid1-array-run-.patch similarity index 100% rename from next-5.10/md-raid1-stop-mdx_raid1-thread-when-raid1-array-run-.patch rename to queue-5.10/md-raid1-stop-mdx_raid1-thread-when-raid1-array-run-.patch diff --git a/next-5.10/media-c8sectpfe-add-of_node_put-when-breaking-out-of.patch b/queue-5.10/media-c8sectpfe-add-of_node_put-when-breaking-out-of.patch similarity index 100% rename from next-5.10/media-c8sectpfe-add-of_node_put-when-breaking-out-of.patch rename to queue-5.10/media-c8sectpfe-add-of_node_put-when-breaking-out-of.patch diff --git a/next-5.10/media-camss-clean-up-received-buffers-on-failed-star.patch b/queue-5.10/media-camss-clean-up-received-buffers-on-failed-star.patch similarity index 100% rename from next-5.10/media-camss-clean-up-received-buffers-on-failed-star.patch rename to queue-5.10/media-camss-clean-up-received-buffers-on-failed-star.patch diff --git a/next-5.10/media-coda-add-check-for-dcoda_iram_alloc.patch b/queue-5.10/media-coda-add-check-for-dcoda_iram_alloc.patch similarity index 100% rename from next-5.10/media-coda-add-check-for-dcoda_iram_alloc.patch rename to queue-5.10/media-coda-add-check-for-dcoda_iram_alloc.patch diff --git a/next-5.10/media-coda-add-check-for-kmalloc.patch b/queue-5.10/media-coda-add-check-for-kmalloc.patch similarity index 100% rename from next-5.10/media-coda-add-check-for-kmalloc.patch rename to queue-5.10/media-coda-add-check-for-kmalloc.patch diff --git a/next-5.10/media-coda-jpeg-add-check-for-kmalloc.patch b/queue-5.10/media-coda-jpeg-add-check-for-kmalloc.patch similarity index 100% rename from next-5.10/media-coda-jpeg-add-check-for-kmalloc.patch rename to queue-5.10/media-coda-jpeg-add-check-for-kmalloc.patch diff --git a/next-5.10/media-dvb-core-fix-ignored-return-value-in-dvb_regis.patch b/queue-5.10/media-dvb-core-fix-ignored-return-value-in-dvb_regis.patch similarity index 100% rename from next-5.10/media-dvb-core-fix-ignored-return-value-in-dvb_regis.patch rename to queue-5.10/media-dvb-core-fix-ignored-return-value-in-dvb_regis.patch diff --git a/next-5.10/media-dvb-frontends-fix-leak-of-memory-fw.patch b/queue-5.10/media-dvb-frontends-fix-leak-of-memory-fw.patch similarity index 100% rename from next-5.10/media-dvb-frontends-fix-leak-of-memory-fw.patch rename to queue-5.10/media-dvb-frontends-fix-leak-of-memory-fw.patch diff --git a/next-5.10/media-dvb-usb-az6027-fix-null-ptr-deref-in-az6027_i2.patch b/queue-5.10/media-dvb-usb-az6027-fix-null-ptr-deref-in-az6027_i2.patch similarity index 100% rename from next-5.10/media-dvb-usb-az6027-fix-null-ptr-deref-in-az6027_i2.patch rename to queue-5.10/media-dvb-usb-az6027-fix-null-ptr-deref-in-az6027_i2.patch diff --git a/next-5.10/media-dvb-usb-fix-memory-leak-in-dvb_usb_adapter_ini.patch b/queue-5.10/media-dvb-usb-fix-memory-leak-in-dvb_usb_adapter_ini.patch similarity index 100% rename from next-5.10/media-dvb-usb-fix-memory-leak-in-dvb_usb_adapter_ini.patch rename to queue-5.10/media-dvb-usb-fix-memory-leak-in-dvb_usb_adapter_ini.patch diff --git a/next-5.10/media-dvbdev-adopts-refcnt-to-avoid-uaf.patch b/queue-5.10/media-dvbdev-adopts-refcnt-to-avoid-uaf.patch similarity index 100% rename from next-5.10/media-dvbdev-adopts-refcnt-to-avoid-uaf.patch rename to queue-5.10/media-dvbdev-adopts-refcnt-to-avoid-uaf.patch diff --git a/next-5.10/media-dvbdev-fix-build-warning-due-to-comments.patch b/queue-5.10/media-dvbdev-fix-build-warning-due-to-comments.patch similarity index 100% rename from next-5.10/media-dvbdev-fix-build-warning-due-to-comments.patch rename to queue-5.10/media-dvbdev-fix-build-warning-due-to-comments.patch diff --git a/next-5.10/media-dvbdev-fix-refcnt-bug.patch b/queue-5.10/media-dvbdev-fix-refcnt-bug.patch similarity index 100% rename from next-5.10/media-dvbdev-fix-refcnt-bug.patch rename to queue-5.10/media-dvbdev-fix-refcnt-bug.patch diff --git a/next-5.10/media-exynos4-is-don-t-rely-on-the-v4l2_async_subdev.patch b/queue-5.10/media-exynos4-is-don-t-rely-on-the-v4l2_async_subdev.patch similarity index 100% rename from next-5.10/media-exynos4-is-don-t-rely-on-the-v4l2_async_subdev.patch rename to queue-5.10/media-exynos4-is-don-t-rely-on-the-v4l2_async_subdev.patch diff --git a/next-5.10/media-exynos4-is-use-v4l2_async_notifier_add_fwnode_.patch b/queue-5.10/media-exynos4-is-use-v4l2_async_notifier_add_fwnode_.patch similarity index 100% rename from next-5.10/media-exynos4-is-use-v4l2_async_notifier_add_fwnode_.patch rename to queue-5.10/media-exynos4-is-use-v4l2_async_notifier_add_fwnode_.patch diff --git a/next-5.10/media-i2c-ad5820-fix-error-path.patch b/queue-5.10/media-i2c-ad5820-fix-error-path.patch similarity index 100% rename from next-5.10/media-i2c-ad5820-fix-error-path.patch rename to queue-5.10/media-i2c-ad5820-fix-error-path.patch diff --git a/next-5.10/media-imon-fix-a-race-condition-in-send_packet.patch b/queue-5.10/media-imon-fix-a-race-condition-in-send_packet.patch similarity index 100% rename from next-5.10/media-imon-fix-a-race-condition-in-send_packet.patch rename to queue-5.10/media-imon-fix-a-race-condition-in-send_packet.patch diff --git a/next-5.10/media-platform-exynos4-is-fix-error-handling-in-fimc.patch b/queue-5.10/media-platform-exynos4-is-fix-error-handling-in-fimc.patch similarity index 100% rename from next-5.10/media-platform-exynos4-is-fix-error-handling-in-fimc.patch rename to queue-5.10/media-platform-exynos4-is-fix-error-handling-in-fimc.patch diff --git a/next-5.10/media-platform-exynos4-is-fix-return-value-check-in-.patch b/queue-5.10/media-platform-exynos4-is-fix-return-value-check-in-.patch similarity index 100% rename from next-5.10/media-platform-exynos4-is-fix-return-value-check-in-.patch rename to queue-5.10/media-platform-exynos4-is-fix-return-value-check-in-.patch diff --git a/next-5.10/media-s5p-mfc-add-variant-data-for-mfc-v7-hardware-f.patch b/queue-5.10/media-s5p-mfc-add-variant-data-for-mfc-v7-hardware-f.patch similarity index 100% rename from next-5.10/media-s5p-mfc-add-variant-data-for-mfc-v7-hardware-f.patch rename to queue-5.10/media-s5p-mfc-add-variant-data-for-mfc-v7-hardware-f.patch diff --git a/next-5.10/media-saa7164-fix-missing-pci_disable_device.patch b/queue-5.10/media-saa7164-fix-missing-pci_disable_device.patch similarity index 100% rename from next-5.10/media-saa7164-fix-missing-pci_disable_device.patch rename to queue-5.10/media-saa7164-fix-missing-pci_disable_device.patch diff --git a/next-5.10/media-si470x-fix-use-after-free-in-si470x_int_in_cal.patch b/queue-5.10/media-si470x-fix-use-after-free-in-si470x_int_in_cal.patch similarity index 100% rename from next-5.10/media-si470x-fix-use-after-free-in-si470x_int_in_cal.patch rename to queue-5.10/media-si470x-fix-use-after-free-in-si470x_int_in_cal.patch diff --git a/next-5.10/media-solo6x10-fix-possible-memory-leak-in-solo_sysf.patch b/queue-5.10/media-solo6x10-fix-possible-memory-leak-in-solo_sysf.patch similarity index 100% rename from next-5.10/media-solo6x10-fix-possible-memory-leak-in-solo_sysf.patch rename to queue-5.10/media-solo6x10-fix-possible-memory-leak-in-solo_sysf.patch diff --git a/next-5.10/media-videobuf-dma-contig-use-dma_mmap_coherent.patch b/queue-5.10/media-videobuf-dma-contig-use-dma_mmap_coherent.patch similarity index 100% rename from next-5.10/media-videobuf-dma-contig-use-dma_mmap_coherent.patch rename to queue-5.10/media-videobuf-dma-contig-use-dma_mmap_coherent.patch diff --git a/next-5.10/media-vidtv-fix-use-after-free-in-vidtv_bridge_dvb_i.patch b/queue-5.10/media-vidtv-fix-use-after-free-in-vidtv_bridge_dvb_i.patch similarity index 100% rename from next-5.10/media-vidtv-fix-use-after-free-in-vidtv_bridge_dvb_i.patch rename to queue-5.10/media-vidtv-fix-use-after-free-in-vidtv_bridge_dvb_i.patch diff --git a/next-5.10/media-vimc-fix-wrong-function-called-when-vimc_init-.patch b/queue-5.10/media-vimc-fix-wrong-function-called-when-vimc_init-.patch similarity index 100% rename from next-5.10/media-vimc-fix-wrong-function-called-when-vimc_init-.patch rename to queue-5.10/media-vimc-fix-wrong-function-called-when-vimc_init-.patch diff --git a/next-5.10/media-vivid-fix-compose-size-exceed-boundary.patch b/queue-5.10/media-vivid-fix-compose-size-exceed-boundary.patch similarity index 100% rename from next-5.10/media-vivid-fix-compose-size-exceed-boundary.patch rename to queue-5.10/media-vivid-fix-compose-size-exceed-boundary.patch diff --git a/next-5.10/mips-bcm63xx-add-check-for-null-for-clk-in-clk_enabl.patch b/queue-5.10/mips-bcm63xx-add-check-for-null-for-clk-in-clk_enabl.patch similarity index 100% rename from next-5.10/mips-bcm63xx-add-check-for-null-for-clk-in-clk_enabl.patch rename to queue-5.10/mips-bcm63xx-add-check-for-null-for-clk-in-clk_enabl.patch diff --git a/next-5.10/mips-octeon-warn-only-once-if-deprecated-link-status.patch b/queue-5.10/mips-octeon-warn-only-once-if-deprecated-link-status.patch similarity index 100% rename from next-5.10/mips-octeon-warn-only-once-if-deprecated-link-status.patch rename to queue-5.10/mips-octeon-warn-only-once-if-deprecated-link-status.patch diff --git a/next-5.10/mips-vpe-cmp-fix-possible-memory-leak-while-module-e.patch b/queue-5.10/mips-vpe-cmp-fix-possible-memory-leak-while-module-e.patch similarity index 100% rename from next-5.10/mips-vpe-cmp-fix-possible-memory-leak-while-module-e.patch rename to queue-5.10/mips-vpe-cmp-fix-possible-memory-leak-while-module-e.patch diff --git a/next-5.10/mips-vpe-mt-fix-possible-memory-leak-while-module-ex.patch b/queue-5.10/mips-vpe-mt-fix-possible-memory-leak-while-module-ex.patch similarity index 100% rename from next-5.10/mips-vpe-mt-fix-possible-memory-leak-while-module-ex.patch rename to queue-5.10/mips-vpe-mt-fix-possible-memory-leak-while-module-ex.patch diff --git a/next-5.10/misc-ocxl-fix-possible-name-leak-in-ocxl_file_regist.patch b/queue-5.10/misc-ocxl-fix-possible-name-leak-in-ocxl_file_regist.patch similarity index 100% rename from next-5.10/misc-ocxl-fix-possible-name-leak-in-ocxl_file_regist.patch rename to queue-5.10/misc-ocxl-fix-possible-name-leak-in-ocxl_file_regist.patch diff --git a/next-5.10/misc-sgi-gru-fix-use-after-free-error-in-gru_set_con.patch b/queue-5.10/misc-sgi-gru-fix-use-after-free-error-in-gru_set_con.patch similarity index 100% rename from next-5.10/misc-sgi-gru-fix-use-after-free-error-in-gru_set_con.patch rename to queue-5.10/misc-sgi-gru-fix-use-after-free-error-in-gru_set_con.patch diff --git a/next-5.10/misc-tifm-fix-possible-memory-leak-in-tifm_7xx1_swit.patch b/queue-5.10/misc-tifm-fix-possible-memory-leak-in-tifm_7xx1_swit.patch similarity index 100% rename from next-5.10/misc-tifm-fix-possible-memory-leak-in-tifm_7xx1_swit.patch rename to queue-5.10/misc-tifm-fix-possible-memory-leak-in-tifm_7xx1_swit.patch diff --git a/next-5.10/misdn-hfcmulti-don-t-call-dev_kfree_skb-kfree_skb-un.patch b/queue-5.10/misdn-hfcmulti-don-t-call-dev_kfree_skb-kfree_skb-un.patch similarity index 100% rename from next-5.10/misdn-hfcmulti-don-t-call-dev_kfree_skb-kfree_skb-un.patch rename to queue-5.10/misdn-hfcmulti-don-t-call-dev_kfree_skb-kfree_skb-un.patch diff --git a/next-5.10/misdn-hfcpci-don-t-call-dev_kfree_skb-kfree_skb-unde.patch b/queue-5.10/misdn-hfcpci-don-t-call-dev_kfree_skb-kfree_skb-unde.patch similarity index 100% rename from next-5.10/misdn-hfcpci-don-t-call-dev_kfree_skb-kfree_skb-unde.patch rename to queue-5.10/misdn-hfcpci-don-t-call-dev_kfree_skb-kfree_skb-unde.patch diff --git a/next-5.10/misdn-hfcsusb-don-t-call-dev_kfree_skb-kfree_skb-und.patch b/queue-5.10/misdn-hfcsusb-don-t-call-dev_kfree_skb-kfree_skb-und.patch similarity index 100% rename from next-5.10/misdn-hfcsusb-don-t-call-dev_kfree_skb-kfree_skb-und.patch rename to queue-5.10/misdn-hfcsusb-don-t-call-dev_kfree_skb-kfree_skb-und.patch diff --git a/next-5.10/mm-compaction-fix-fast_isolate_around-to-stay-within-boundaries.patch b/queue-5.10/mm-compaction-fix-fast_isolate_around-to-stay-within-boundaries.patch similarity index 100% rename from next-5.10/mm-compaction-fix-fast_isolate_around-to-stay-within-boundaries.patch rename to queue-5.10/mm-compaction-fix-fast_isolate_around-to-stay-within-boundaries.patch diff --git a/next-5.10/mmc-alcor-fix-return-value-check-of-mmc_add_host.patch b/queue-5.10/mmc-alcor-fix-return-value-check-of-mmc_add_host.patch similarity index 100% rename from next-5.10/mmc-alcor-fix-return-value-check-of-mmc_add_host.patch rename to queue-5.10/mmc-alcor-fix-return-value-check-of-mmc_add_host.patch diff --git a/next-5.10/mmc-atmel-mci-fix-return-value-check-of-mmc_add_host.patch b/queue-5.10/mmc-atmel-mci-fix-return-value-check-of-mmc_add_host.patch similarity index 100% rename from next-5.10/mmc-atmel-mci-fix-return-value-check-of-mmc_add_host.patch rename to queue-5.10/mmc-atmel-mci-fix-return-value-check-of-mmc_add_host.patch diff --git a/next-5.10/mmc-f-sdh30-add-quirks-for-broken-timeout-clock-capa.patch b/queue-5.10/mmc-f-sdh30-add-quirks-for-broken-timeout-clock-capa.patch similarity index 100% rename from next-5.10/mmc-f-sdh30-add-quirks-for-broken-timeout-clock-capa.patch rename to queue-5.10/mmc-f-sdh30-add-quirks-for-broken-timeout-clock-capa.patch diff --git a/next-5.10/mmc-meson-gx-fix-return-value-check-of-mmc_add_host.patch b/queue-5.10/mmc-meson-gx-fix-return-value-check-of-mmc_add_host.patch similarity index 100% rename from next-5.10/mmc-meson-gx-fix-return-value-check-of-mmc_add_host.patch rename to queue-5.10/mmc-meson-gx-fix-return-value-check-of-mmc_add_host.patch diff --git a/next-5.10/mmc-mmci-fix-return-value-check-of-mmc_add_host.patch b/queue-5.10/mmc-mmci-fix-return-value-check-of-mmc_add_host.patch similarity index 100% rename from next-5.10/mmc-mmci-fix-return-value-check-of-mmc_add_host.patch rename to queue-5.10/mmc-mmci-fix-return-value-check-of-mmc_add_host.patch diff --git a/next-5.10/mmc-moxart-fix-return-value-check-of-mmc_add_host.patch b/queue-5.10/mmc-moxart-fix-return-value-check-of-mmc_add_host.patch similarity index 100% rename from next-5.10/mmc-moxart-fix-return-value-check-of-mmc_add_host.patch rename to queue-5.10/mmc-moxart-fix-return-value-check-of-mmc_add_host.patch diff --git a/next-5.10/mmc-mxcmmc-fix-return-value-check-of-mmc_add_host.patch b/queue-5.10/mmc-mxcmmc-fix-return-value-check-of-mmc_add_host.patch similarity index 100% rename from next-5.10/mmc-mxcmmc-fix-return-value-check-of-mmc_add_host.patch rename to queue-5.10/mmc-mxcmmc-fix-return-value-check-of-mmc_add_host.patch diff --git a/next-5.10/mmc-omap_hsmmc-fix-return-value-check-of-mmc_add_hos.patch b/queue-5.10/mmc-omap_hsmmc-fix-return-value-check-of-mmc_add_hos.patch similarity index 100% rename from next-5.10/mmc-omap_hsmmc-fix-return-value-check-of-mmc_add_hos.patch rename to queue-5.10/mmc-omap_hsmmc-fix-return-value-check-of-mmc_add_hos.patch diff --git a/next-5.10/mmc-pxamci-fix-return-value-check-of-mmc_add_host.patch b/queue-5.10/mmc-pxamci-fix-return-value-check-of-mmc_add_host.patch similarity index 100% rename from next-5.10/mmc-pxamci-fix-return-value-check-of-mmc_add_host.patch rename to queue-5.10/mmc-pxamci-fix-return-value-check-of-mmc_add_host.patch diff --git a/next-5.10/mmc-renesas_sdhi-better-reset-from-hs400-mode.patch b/queue-5.10/mmc-renesas_sdhi-better-reset-from-hs400-mode.patch similarity index 100% rename from next-5.10/mmc-renesas_sdhi-better-reset-from-hs400-mode.patch rename to queue-5.10/mmc-renesas_sdhi-better-reset-from-hs400-mode.patch diff --git a/next-5.10/mmc-rtsx_usb_sdmmc-fix-return-value-check-of-mmc_add.patch b/queue-5.10/mmc-rtsx_usb_sdmmc-fix-return-value-check-of-mmc_add.patch similarity index 100% rename from next-5.10/mmc-rtsx_usb_sdmmc-fix-return-value-check-of-mmc_add.patch rename to queue-5.10/mmc-rtsx_usb_sdmmc-fix-return-value-check-of-mmc_add.patch diff --git a/next-5.10/mmc-toshsd-fix-return-value-check-of-mmc_add_host.patch b/queue-5.10/mmc-toshsd-fix-return-value-check-of-mmc_add_host.patch similarity index 100% rename from next-5.10/mmc-toshsd-fix-return-value-check-of-mmc_add_host.patch rename to queue-5.10/mmc-toshsd-fix-return-value-check-of-mmc_add_host.patch diff --git a/next-5.10/mmc-via-sdmmc-fix-return-value-check-of-mmc_add_host.patch b/queue-5.10/mmc-via-sdmmc-fix-return-value-check-of-mmc_add_host.patch similarity index 100% rename from next-5.10/mmc-via-sdmmc-fix-return-value-check-of-mmc_add_host.patch rename to queue-5.10/mmc-via-sdmmc-fix-return-value-check-of-mmc_add_host.patch diff --git a/next-5.10/mmc-vub300-fix-return-value-check-of-mmc_add_host.patch b/queue-5.10/mmc-vub300-fix-return-value-check-of-mmc_add_host.patch similarity index 100% rename from next-5.10/mmc-vub300-fix-return-value-check-of-mmc_add_host.patch rename to queue-5.10/mmc-vub300-fix-return-value-check-of-mmc_add_host.patch diff --git a/next-5.10/mmc-vub300-fix-warning-do-not-call-blocking-ops-when-task_running.patch b/queue-5.10/mmc-vub300-fix-warning-do-not-call-blocking-ops-when-task_running.patch similarity index 100% rename from next-5.10/mmc-vub300-fix-warning-do-not-call-blocking-ops-when-task_running.patch rename to queue-5.10/mmc-vub300-fix-warning-do-not-call-blocking-ops-when-task_running.patch diff --git a/next-5.10/mmc-wbsd-fix-return-value-check-of-mmc_add_host.patch b/queue-5.10/mmc-wbsd-fix-return-value-check-of-mmc_add_host.patch similarity index 100% rename from next-5.10/mmc-wbsd-fix-return-value-check-of-mmc_add_host.patch rename to queue-5.10/mmc-wbsd-fix-return-value-check-of-mmc_add_host.patch diff --git a/next-5.10/mmc-wmt-sdmmc-fix-return-value-check-of-mmc_add_host.patch b/queue-5.10/mmc-wmt-sdmmc-fix-return-value-check-of-mmc_add_host.patch similarity index 100% rename from next-5.10/mmc-wmt-sdmmc-fix-return-value-check-of-mmc_add_host.patch rename to queue-5.10/mmc-wmt-sdmmc-fix-return-value-check-of-mmc_add_host.patch diff --git a/next-5.10/mrp-introduce-active-flags-to-prevent-uaf-when-appli.patch b/queue-5.10/mrp-introduce-active-flags-to-prevent-uaf-when-appli.patch similarity index 100% rename from next-5.10/mrp-introduce-active-flags-to-prevent-uaf-when-appli.patch rename to queue-5.10/mrp-introduce-active-flags-to-prevent-uaf-when-appli.patch diff --git a/next-5.10/mtd-fix-device-name-leak-when-register-device-failed.patch b/queue-5.10/mtd-fix-device-name-leak-when-register-device-failed.patch similarity index 100% rename from next-5.10/mtd-fix-device-name-leak-when-register-device-failed.patch rename to queue-5.10/mtd-fix-device-name-leak-when-register-device-failed.patch diff --git a/next-5.10/mtd-lpddr2_nvm-fix-possible-null-ptr-deref.patch b/queue-5.10/mtd-lpddr2_nvm-fix-possible-null-ptr-deref.patch similarity index 100% rename from next-5.10/mtd-lpddr2_nvm-fix-possible-null-ptr-deref.patch rename to queue-5.10/mtd-lpddr2_nvm-fix-possible-null-ptr-deref.patch diff --git a/next-5.10/mtd-maps-pxa2xx-flash-fix-memory-leak-in-probe.patch b/queue-5.10/mtd-maps-pxa2xx-flash-fix-memory-leak-in-probe.patch similarity index 100% rename from next-5.10/mtd-maps-pxa2xx-flash-fix-memory-leak-in-probe.patch rename to queue-5.10/mtd-maps-pxa2xx-flash-fix-memory-leak-in-probe.patch diff --git a/next-5.10/myri10ge-fix-an-error-handling-path-in-myri10ge_prob.patch b/queue-5.10/myri10ge-fix-an-error-handling-path-in-myri10ge_prob.patch similarity index 100% rename from next-5.10/myri10ge-fix-an-error-handling-path-in-myri10ge_prob.patch rename to queue-5.10/myri10ge-fix-an-error-handling-path-in-myri10ge_prob.patch diff --git a/next-5.10/net-add-a-helper-to-avoid-issues-with-hw-tx-timestam.patch b/queue-5.10/net-add-a-helper-to-avoid-issues-with-hw-tx-timestam.patch similarity index 100% rename from next-5.10/net-add-a-helper-to-avoid-issues-with-hw-tx-timestam.patch rename to queue-5.10/net-add-a-helper-to-avoid-issues-with-hw-tx-timestam.patch diff --git a/next-5.10/net-add-atomic_long_t-to-net_device_stats-fields.patch b/queue-5.10/net-add-atomic_long_t-to-net_device_stats-fields.patch similarity index 100% rename from next-5.10/net-add-atomic_long_t-to-net_device_stats-fields.patch rename to queue-5.10/net-add-atomic_long_t-to-net_device_stats-fields.patch diff --git a/next-5.10/net-add-inline-function-skb_csum_is_sctp.patch b/queue-5.10/net-add-inline-function-skb_csum_is_sctp.patch similarity index 100% rename from next-5.10/net-add-inline-function-skb_csum_is_sctp.patch rename to queue-5.10/net-add-inline-function-skb_csum_is_sctp.patch diff --git a/next-5.10/net-amd-lance-don-t-call-dev_kfree_skb-under-spin_lo.patch b/queue-5.10/net-amd-lance-don-t-call-dev_kfree_skb-under-spin_lo.patch similarity index 100% rename from next-5.10/net-amd-lance-don-t-call-dev_kfree_skb-under-spin_lo.patch rename to queue-5.10/net-amd-lance-don-t-call-dev_kfree_skb-under-spin_lo.patch diff --git a/next-5.10/net-amd-xgbe-check-only-the-minimum-speed-for-active.patch b/queue-5.10/net-amd-xgbe-check-only-the-minimum-speed-for-active.patch similarity index 100% rename from next-5.10/net-amd-xgbe-check-only-the-minimum-speed-for-active.patch rename to queue-5.10/net-amd-xgbe-check-only-the-minimum-speed-for-active.patch diff --git a/next-5.10/net-amd-xgbe-fix-logic-around-active-and-passive-cab.patch b/queue-5.10/net-amd-xgbe-fix-logic-around-active-and-passive-cab.patch similarity index 100% rename from next-5.10/net-amd-xgbe-fix-logic-around-active-and-passive-cab.patch rename to queue-5.10/net-amd-xgbe-fix-logic-around-active-and-passive-cab.patch diff --git a/next-5.10/net-apple-bmac-don-t-call-dev_kfree_skb-under-spin_l.patch b/queue-5.10/net-apple-bmac-don-t-call-dev_kfree_skb-under-spin_l.patch similarity index 100% rename from next-5.10/net-apple-bmac-don-t-call-dev_kfree_skb-under-spin_l.patch rename to queue-5.10/net-apple-bmac-don-t-call-dev_kfree_skb-under-spin_l.patch diff --git a/next-5.10/net-apple-mace-don-t-call-dev_kfree_skb-under-spin_l.patch b/queue-5.10/net-apple-mace-don-t-call-dev_kfree_skb-under-spin_l.patch similarity index 100% rename from next-5.10/net-apple-mace-don-t-call-dev_kfree_skb-under-spin_l.patch rename to queue-5.10/net-apple-mace-don-t-call-dev_kfree_skb-under-spin_l.patch diff --git a/next-5.10/net-defxx-fix-missing-err-handling-in-dfx_init.patch b/queue-5.10/net-defxx-fix-missing-err-handling-in-dfx_init.patch similarity index 100% rename from next-5.10/net-defxx-fix-missing-err-handling-in-dfx_init.patch rename to queue-5.10/net-defxx-fix-missing-err-handling-in-dfx_init.patch diff --git a/next-5.10/net-emaclite-don-t-call-dev_kfree_skb-under-spin_loc.patch b/queue-5.10/net-emaclite-don-t-call-dev_kfree_skb-under-spin_loc.patch similarity index 100% rename from next-5.10/net-emaclite-don-t-call-dev_kfree_skb-under-spin_loc.patch rename to queue-5.10/net-emaclite-don-t-call-dev_kfree_skb-under-spin_loc.patch diff --git a/next-5.10/net-ethernet-dnet-don-t-call-dev_kfree_skb-under-spi.patch b/queue-5.10/net-ethernet-dnet-don-t-call-dev_kfree_skb-under-spi.patch similarity index 100% rename from next-5.10/net-ethernet-dnet-don-t-call-dev_kfree_skb-under-spi.patch rename to queue-5.10/net-ethernet-dnet-don-t-call-dev_kfree_skb-under-spi.patch diff --git a/next-5.10/net-ethernet-ti-fix-return-type-of-netcp_ndo_start_x.patch b/queue-5.10/net-ethernet-ti-fix-return-type-of-netcp_ndo_start_x.patch similarity index 100% rename from next-5.10/net-ethernet-ti-fix-return-type-of-netcp_ndo_start_x.patch rename to queue-5.10/net-ethernet-ti-fix-return-type-of-netcp_ndo_start_x.patch diff --git a/next-5.10/net-farsync-fix-kmemleak-when-rmmods-farsync.patch b/queue-5.10/net-farsync-fix-kmemleak-when-rmmods-farsync.patch similarity index 100% rename from next-5.10/net-farsync-fix-kmemleak-when-rmmods-farsync.patch rename to queue-5.10/net-farsync-fix-kmemleak-when-rmmods-farsync.patch diff --git a/next-5.10/net-hsr-generate-supervision-frame-without-hsr-prp-t.patch b/queue-5.10/net-hsr-generate-supervision-frame-without-hsr-prp-t.patch similarity index 100% rename from next-5.10/net-hsr-generate-supervision-frame-without-hsr-prp-t.patch rename to queue-5.10/net-hsr-generate-supervision-frame-without-hsr-prp-t.patch diff --git a/next-5.10/net-igc-use-skb_csum_is_sctp-instead-of-protocol-che.patch b/queue-5.10/net-igc-use-skb_csum_is_sctp-instead-of-protocol-che.patch similarity index 100% rename from next-5.10/net-igc-use-skb_csum_is_sctp-instead-of-protocol-che.patch rename to queue-5.10/net-igc-use-skb_csum_is_sctp-instead-of-protocol-che.patch diff --git a/next-5.10/net-lan9303-fix-read-error-execution-path.patch b/queue-5.10/net-lan9303-fix-read-error-execution-path.patch similarity index 100% rename from next-5.10/net-lan9303-fix-read-error-execution-path.patch rename to queue-5.10/net-lan9303-fix-read-error-execution-path.patch diff --git a/next-5.10/net-macsec-fix-net-device-access-prior-to-holding-a-.patch b/queue-5.10/net-macsec-fix-net-device-access-prior-to-holding-a-.patch similarity index 100% rename from next-5.10/net-macsec-fix-net-device-access-prior-to-holding-a-.patch rename to queue-5.10/net-macsec-fix-net-device-access-prior-to-holding-a-.patch diff --git a/next-5.10/net-proc-provide-proc_fs-n-fallback-for-proc_create_.patch b/queue-5.10/net-proc-provide-proc_fs-n-fallback-for-proc_create_.patch similarity index 100% rename from next-5.10/net-proc-provide-proc_fs-n-fallback-for-proc_create_.patch rename to queue-5.10/net-proc-provide-proc_fs-n-fallback-for-proc_create_.patch diff --git a/next-5.10/net-stmmac-selftests-fix-potential-memleak-in-stmmac.patch b/queue-5.10/net-stmmac-selftests-fix-potential-memleak-in-stmmac.patch similarity index 100% rename from next-5.10/net-stmmac-selftests-fix-potential-memleak-in-stmmac.patch rename to queue-5.10/net-stmmac-selftests-fix-potential-memleak-in-stmmac.patch diff --git a/next-5.10/net-stream-purge-sk_error_queue-in-sk_stream_kill_qu.patch b/queue-5.10/net-stream-purge-sk_error_queue-in-sk_stream_kill_qu.patch similarity index 100% rename from next-5.10/net-stream-purge-sk_error_queue-in-sk_stream_kill_qu.patch rename to queue-5.10/net-stream-purge-sk_error_queue-in-sk_stream_kill_qu.patch diff --git a/next-5.10/net-switch-to-storing-kcov-handle-directly-in-sk_buf.patch b/queue-5.10/net-switch-to-storing-kcov-handle-directly-in-sk_buf.patch similarity index 100% rename from next-5.10/net-switch-to-storing-kcov-handle-directly-in-sk_buf.patch rename to queue-5.10/net-switch-to-storing-kcov-handle-directly-in-sk_buf.patch diff --git a/next-5.10/net-tunnel-wait-until-all-sk_user_data-reader-finish.patch b/queue-5.10/net-tunnel-wait-until-all-sk_user_data-reader-finish.patch similarity index 100% rename from next-5.10/net-tunnel-wait-until-all-sk_user_data-reader-finish.patch rename to queue-5.10/net-tunnel-wait-until-all-sk_user_data-reader-finish.patch diff --git a/next-5.10/net-vmw_vsock-vmci-check-memcpy_from_msg.patch b/queue-5.10/net-vmw_vsock-vmci-check-memcpy_from_msg.patch similarity index 100% rename from next-5.10/net-vmw_vsock-vmci-check-memcpy_from_msg.patch rename to queue-5.10/net-vmw_vsock-vmci-check-memcpy_from_msg.patch diff --git a/next-5.10/net_sched-reject-tcf_em_simple-case-for-complex-emat.patch b/queue-5.10/net_sched-reject-tcf_em_simple-case-for-complex-emat.patch similarity index 100% rename from next-5.10/net_sched-reject-tcf_em_simple-case-for-complex-emat.patch rename to queue-5.10/net_sched-reject-tcf_em_simple-case-for-complex-emat.patch diff --git a/next-5.10/netfilter-conntrack-set-icmpv6-redirects-as-related.patch b/queue-5.10/netfilter-conntrack-set-icmpv6-redirects-as-related.patch similarity index 100% rename from next-5.10/netfilter-conntrack-set-icmpv6-redirects-as-related.patch rename to queue-5.10/netfilter-conntrack-set-icmpv6-redirects-as-related.patch diff --git a/next-5.10/netfilter-flowtable-really-fix-nat-ipv6-offload.patch b/queue-5.10/netfilter-flowtable-really-fix-nat-ipv6-offload.patch similarity index 100% rename from next-5.10/netfilter-flowtable-really-fix-nat-ipv6-offload.patch rename to queue-5.10/netfilter-flowtable-really-fix-nat-ipv6-offload.patch diff --git a/next-5.10/nfc-pn533-clear-nfc_target-before-being-used.patch b/queue-5.10/nfc-pn533-clear-nfc_target-before-being-used.patch similarity index 100% rename from next-5.10/nfc-pn533-clear-nfc_target-before-being-used.patch rename to queue-5.10/nfc-pn533-clear-nfc_target-before-being-used.patch diff --git a/next-5.10/nfs-fix-an-oops-in-nfs_d_automount.patch b/queue-5.10/nfs-fix-an-oops-in-nfs_d_automount.patch similarity index 100% rename from next-5.10/nfs-fix-an-oops-in-nfs_d_automount.patch rename to queue-5.10/nfs-fix-an-oops-in-nfs_d_automount.patch diff --git a/next-5.10/nfsd-don-t-call-nfsd_file_put-from-client-states-seq.patch b/queue-5.10/nfsd-don-t-call-nfsd_file_put-from-client-states-seq.patch similarity index 100% rename from next-5.10/nfsd-don-t-call-nfsd_file_put-from-client-states-seq.patch rename to queue-5.10/nfsd-don-t-call-nfsd_file_put-from-client-states-seq.patch diff --git a/next-5.10/nfsd-remove-spurious-cb_setup_err-tracepoint.patch b/queue-5.10/nfsd-remove-spurious-cb_setup_err-tracepoint.patch similarity index 100% rename from next-5.10/nfsd-remove-spurious-cb_setup_err-tracepoint.patch rename to queue-5.10/nfsd-remove-spurious-cb_setup_err-tracepoint.patch diff --git a/next-5.10/nfsd-under-nfsv4.1-fix-double-svc_xprt_put-on-rpc_cr.patch b/queue-5.10/nfsd-under-nfsv4.1-fix-double-svc_xprt_put-on-rpc_cr.patch similarity index 100% rename from next-5.10/nfsd-under-nfsv4.1-fix-double-svc_xprt_put-on-rpc_cr.patch rename to queue-5.10/nfsd-under-nfsv4.1-fix-double-svc_xprt_put-on-rpc_cr.patch diff --git a/next-5.10/nfsv4-fix-a-deadlock-between-nfs4_open_recover_helpe.patch b/queue-5.10/nfsv4-fix-a-deadlock-between-nfs4_open_recover_helpe.patch similarity index 100% rename from next-5.10/nfsv4-fix-a-deadlock-between-nfs4_open_recover_helpe.patch rename to queue-5.10/nfsv4-fix-a-deadlock-between-nfs4_open_recover_helpe.patch diff --git a/next-5.10/nfsv4.2-clear-fattr4_word2_security_label-when-done-.patch b/queue-5.10/nfsv4.2-clear-fattr4_word2_security_label-when-done-.patch similarity index 100% rename from next-5.10/nfsv4.2-clear-fattr4_word2_security_label-when-done-.patch rename to queue-5.10/nfsv4.2-clear-fattr4_word2_security_label-when-done-.patch diff --git a/next-5.10/nfsv4.2-fix-a-memory-stomp-in-decode_attr_security_l.patch b/queue-5.10/nfsv4.2-fix-a-memory-stomp-in-decode_attr_security_l.patch similarity index 100% rename from next-5.10/nfsv4.2-fix-a-memory-stomp-in-decode_attr_security_l.patch rename to queue-5.10/nfsv4.2-fix-a-memory-stomp-in-decode_attr_security_l.patch diff --git a/next-5.10/nfsv4.2-fix-initialisation-of-struct-nfs4_label.patch b/queue-5.10/nfsv4.2-fix-initialisation-of-struct-nfs4_label.patch similarity index 100% rename from next-5.10/nfsv4.2-fix-initialisation-of-struct-nfs4_label.patch rename to queue-5.10/nfsv4.2-fix-initialisation-of-struct-nfs4_label.patch diff --git a/next-5.10/nfsv4.x-fail-client-initialisation-if-state-manager-.patch b/queue-5.10/nfsv4.x-fail-client-initialisation-if-state-manager-.patch similarity index 100% rename from next-5.10/nfsv4.x-fail-client-initialisation-if-state-manager-.patch rename to queue-5.10/nfsv4.x-fail-client-initialisation-if-state-manager-.patch diff --git a/next-5.10/nilfs2-fix-shift-out-of-bounds-due-to-too-large-expo.patch b/queue-5.10/nilfs2-fix-shift-out-of-bounds-due-to-too-large-expo.patch similarity index 100% rename from next-5.10/nilfs2-fix-shift-out-of-bounds-due-to-too-large-expo.patch rename to queue-5.10/nilfs2-fix-shift-out-of-bounds-due-to-too-large-expo.patch diff --git a/next-5.10/nilfs2-fix-shift-out-of-bounds-overflow-in-nilfs_sb2.patch b/queue-5.10/nilfs2-fix-shift-out-of-bounds-overflow-in-nilfs_sb2.patch similarity index 100% rename from next-5.10/nilfs2-fix-shift-out-of-bounds-overflow-in-nilfs_sb2.patch rename to queue-5.10/nilfs2-fix-shift-out-of-bounds-overflow-in-nilfs_sb2.patch diff --git a/next-5.10/ntb_netdev-use-dev_kfree_skb_any-in-interrupt-contex.patch b/queue-5.10/ntb_netdev-use-dev_kfree_skb_any-in-interrupt-contex.patch similarity index 100% rename from next-5.10/ntb_netdev-use-dev_kfree_skb_any-in-interrupt-contex.patch rename to queue-5.10/ntb_netdev-use-dev_kfree_skb_any-in-interrupt-contex.patch diff --git a/next-5.10/nvme-fix-the-nvme_cmd_effects_cse_mask-definition.patch b/queue-5.10/nvme-fix-the-nvme_cmd_effects_cse_mask-definition.patch similarity index 100% rename from next-5.10/nvme-fix-the-nvme_cmd_effects_cse_mask-definition.patch rename to queue-5.10/nvme-fix-the-nvme_cmd_effects_cse_mask-definition.patch diff --git a/next-5.10/nvme-pci-fix-doorbell-buffer-value-endianness.patch b/queue-5.10/nvme-pci-fix-doorbell-buffer-value-endianness.patch similarity index 100% rename from next-5.10/nvme-pci-fix-doorbell-buffer-value-endianness.patch rename to queue-5.10/nvme-pci-fix-doorbell-buffer-value-endianness.patch diff --git a/next-5.10/nvme-pci-fix-mempool-alloc-size.patch b/queue-5.10/nvme-pci-fix-mempool-alloc-size.patch similarity index 100% rename from next-5.10/nvme-pci-fix-mempool-alloc-size.patch rename to queue-5.10/nvme-pci-fix-mempool-alloc-size.patch diff --git a/next-5.10/nvme-pci-fix-page-size-checks.patch b/queue-5.10/nvme-pci-fix-page-size-checks.patch similarity index 100% rename from next-5.10/nvme-pci-fix-page-size-checks.patch rename to queue-5.10/nvme-pci-fix-page-size-checks.patch diff --git a/next-5.10/nvmet-don-t-defer-passthrough-commands-with-trivial-.patch b/queue-5.10/nvmet-don-t-defer-passthrough-commands-with-trivial-.patch similarity index 100% rename from next-5.10/nvmet-don-t-defer-passthrough-commands-with-trivial-.patch rename to queue-5.10/nvmet-don-t-defer-passthrough-commands-with-trivial-.patch diff --git a/next-5.10/objtool-fix-segfault.patch b/queue-5.10/objtool-fix-segfault.patch similarity index 100% rename from next-5.10/objtool-fix-segfault.patch rename to queue-5.10/objtool-fix-segfault.patch diff --git a/next-5.10/objtool-kcsan-add-volatile-read-write-instrumentatio.patch b/queue-5.10/objtool-kcsan-add-volatile-read-write-instrumentatio.patch similarity index 100% rename from next-5.10/objtool-kcsan-add-volatile-read-write-instrumentatio.patch rename to queue-5.10/objtool-kcsan-add-volatile-read-write-instrumentatio.patch diff --git a/next-5.10/ocfs2-fix-memory-leak-in-ocfs2_mount_volume.patch b/queue-5.10/ocfs2-fix-memory-leak-in-ocfs2_mount_volume.patch similarity index 100% rename from next-5.10/ocfs2-fix-memory-leak-in-ocfs2_mount_volume.patch rename to queue-5.10/ocfs2-fix-memory-leak-in-ocfs2_mount_volume.patch diff --git a/next-5.10/ocfs2-fix-memory-leak-in-ocfs2_stack_glue_init.patch b/queue-5.10/ocfs2-fix-memory-leak-in-ocfs2_stack_glue_init.patch similarity index 100% rename from next-5.10/ocfs2-fix-memory-leak-in-ocfs2_stack_glue_init.patch rename to queue-5.10/ocfs2-fix-memory-leak-in-ocfs2_stack_glue_init.patch diff --git a/next-5.10/ocfs2-ocfs2_mount_volume-does-cleanup-job-before-ret.patch b/queue-5.10/ocfs2-ocfs2_mount_volume-does-cleanup-job-before-ret.patch similarity index 100% rename from next-5.10/ocfs2-ocfs2_mount_volume-does-cleanup-job-before-ret.patch rename to queue-5.10/ocfs2-ocfs2_mount_volume-does-cleanup-job-before-ret.patch diff --git a/next-5.10/ocfs2-rewrite-error-handling-of-ocfs2_fill_super.patch b/queue-5.10/ocfs2-rewrite-error-handling-of-ocfs2_fill_super.patch similarity index 100% rename from next-5.10/ocfs2-rewrite-error-handling-of-ocfs2_fill_super.patch rename to queue-5.10/ocfs2-rewrite-error-handling-of-ocfs2_fill_super.patch diff --git a/next-5.10/ocxl-fix-pci-device-refcount-leak-when-calling-get_f.patch b/queue-5.10/ocxl-fix-pci-device-refcount-leak-when-calling-get_f.patch similarity index 100% rename from next-5.10/ocxl-fix-pci-device-refcount-leak-when-calling-get_f.patch rename to queue-5.10/ocxl-fix-pci-device-refcount-leak-when-calling-get_f.patch diff --git a/next-5.10/of-overlay-fix-null-pointer-dereferencing-in-find_du.patch b/queue-5.10/of-overlay-fix-null-pointer-dereferencing-in-find_du.patch similarity index 100% rename from next-5.10/of-overlay-fix-null-pointer-dereferencing-in-find_du.patch rename to queue-5.10/of-overlay-fix-null-pointer-dereferencing-in-find_du.patch diff --git a/next-5.10/openvswitch-fix-flow-lookup-to-use-unmasked-key.patch b/queue-5.10/openvswitch-fix-flow-lookup-to-use-unmasked-key.patch similarity index 100% rename from next-5.10/openvswitch-fix-flow-lookup-to-use-unmasked-key.patch rename to queue-5.10/openvswitch-fix-flow-lookup-to-use-unmasked-key.patch diff --git a/next-5.10/orangefs-fix-kmemleak-in-orangefs_-kernel-client-_de.patch b/queue-5.10/orangefs-fix-kmemleak-in-orangefs_-kernel-client-_de.patch similarity index 100% rename from next-5.10/orangefs-fix-kmemleak-in-orangefs_-kernel-client-_de.patch rename to queue-5.10/orangefs-fix-kmemleak-in-orangefs_-kernel-client-_de.patch diff --git a/next-5.10/orangefs-fix-kmemleak-in-orangefs_prepare_debugfs_he.patch b/queue-5.10/orangefs-fix-kmemleak-in-orangefs_prepare_debugfs_he.patch similarity index 100% rename from next-5.10/orangefs-fix-kmemleak-in-orangefs_prepare_debugfs_he.patch rename to queue-5.10/orangefs-fix-kmemleak-in-orangefs_prepare_debugfs_he.patch diff --git a/next-5.10/orangefs-fix-sysfs-not-cleanup-when-dev-init-failed.patch b/queue-5.10/orangefs-fix-sysfs-not-cleanup-when-dev-init-failed.patch similarity index 100% rename from next-5.10/orangefs-fix-sysfs-not-cleanup-when-dev-init-failed.patch rename to queue-5.10/orangefs-fix-sysfs-not-cleanup-when-dev-init-failed.patch diff --git a/next-5.10/ovl-fix-use-inode-directly-in-rcu-walk-mode.patch b/queue-5.10/ovl-fix-use-inode-directly-in-rcu-walk-mode.patch similarity index 100% rename from next-5.10/ovl-fix-use-inode-directly-in-rcu-walk-mode.patch rename to queue-5.10/ovl-fix-use-inode-directly-in-rcu-walk-mode.patch diff --git a/next-5.10/ovl-use-ovl-mounter-s-fsuid-and-fsgid-in-ovl_link.patch b/queue-5.10/ovl-use-ovl-mounter-s-fsuid-and-fsgid-in-ovl_link.patch similarity index 100% rename from next-5.10/ovl-use-ovl-mounter-s-fsuid-and-fsgid-in-ovl_link.patch rename to queue-5.10/ovl-use-ovl-mounter-s-fsuid-and-fsgid-in-ovl_link.patch diff --git a/next-5.10/padata-always-leave-bhs-disabled-when-running-parall.patch b/queue-5.10/padata-always-leave-bhs-disabled-when-running-parall.patch similarity index 100% rename from next-5.10/padata-always-leave-bhs-disabled-when-running-parall.patch rename to queue-5.10/padata-always-leave-bhs-disabled-when-running-parall.patch diff --git a/next-5.10/padata-fix-list-iterator-in-padata_do_serial.patch b/queue-5.10/padata-fix-list-iterator-in-padata_do_serial.patch similarity index 100% rename from next-5.10/padata-fix-list-iterator-in-padata_do_serial.patch rename to queue-5.10/padata-fix-list-iterator-in-padata_do_serial.patch diff --git a/next-5.10/pata_ipx4xx_cf-fix-unsigned-comparison-with-less-tha.patch b/queue-5.10/pata_ipx4xx_cf-fix-unsigned-comparison-with-less-tha.patch similarity index 100% rename from next-5.10/pata_ipx4xx_cf-fix-unsigned-comparison-with-less-tha.patch rename to queue-5.10/pata_ipx4xx_cf-fix-unsigned-comparison-with-less-tha.patch diff --git a/next-5.10/pci-check-for-alloc-failure-in-pci_request_irq.patch b/queue-5.10/pci-check-for-alloc-failure-in-pci_request_irq.patch similarity index 100% rename from next-5.10/pci-check-for-alloc-failure-in-pci_request_irq.patch rename to queue-5.10/pci-check-for-alloc-failure-in-pci_request_irq.patch diff --git a/next-5.10/pci-dwc-fix-n_fts-array-overrun.patch b/queue-5.10/pci-dwc-fix-n_fts-array-overrun.patch similarity index 100% rename from next-5.10/pci-dwc-fix-n_fts-array-overrun.patch rename to queue-5.10/pci-dwc-fix-n_fts-array-overrun.patch diff --git a/next-5.10/pci-pci-epf-test-register-notifier-if-only-core_init.patch b/queue-5.10/pci-pci-epf-test-register-notifier-if-only-core_init.patch similarity index 100% rename from next-5.10/pci-pci-epf-test-register-notifier-if-only-core_init.patch rename to queue-5.10/pci-pci-epf-test-register-notifier-if-only-core_init.patch diff --git a/next-5.10/perf-arm_dsu-fix-hotplug-callback-leak-in-dsu_pmu_in.patch b/queue-5.10/perf-arm_dsu-fix-hotplug-callback-leak-in-dsu_pmu_in.patch similarity index 100% rename from next-5.10/perf-arm_dsu-fix-hotplug-callback-leak-in-dsu_pmu_in.patch rename to queue-5.10/perf-arm_dsu-fix-hotplug-callback-leak-in-dsu_pmu_in.patch diff --git a/next-5.10/perf-debug-set-debug_peo_args-and-redirect_to_stderr.patch b/queue-5.10/perf-debug-set-debug_peo_args-and-redirect_to_stderr.patch similarity index 100% rename from next-5.10/perf-debug-set-debug_peo_args-and-redirect_to_stderr.patch rename to queue-5.10/perf-debug-set-debug_peo_args-and-redirect_to_stderr.patch diff --git a/next-5.10/perf-fix-possible-memleak-in-pmu_dev_alloc.patch b/queue-5.10/perf-fix-possible-memleak-in-pmu_dev_alloc.patch similarity index 100% rename from next-5.10/perf-fix-possible-memleak-in-pmu_dev_alloc.patch rename to queue-5.10/perf-fix-possible-memleak-in-pmu_dev_alloc.patch diff --git a/next-5.10/perf-smmuv3-fix-hotplug-callback-leak-in-arm_smmu_pm.patch b/queue-5.10/perf-smmuv3-fix-hotplug-callback-leak-in-arm_smmu_pm.patch similarity index 100% rename from next-5.10/perf-smmuv3-fix-hotplug-callback-leak-in-arm_smmu_pm.patch rename to queue-5.10/perf-smmuv3-fix-hotplug-callback-leak-in-arm_smmu_pm.patch diff --git a/next-5.10/perf-symbol-correction-while-adjusting-symbol.patch b/queue-5.10/perf-symbol-correction-while-adjusting-symbol.patch similarity index 100% rename from next-5.10/perf-symbol-correction-while-adjusting-symbol.patch rename to queue-5.10/perf-symbol-correction-while-adjusting-symbol.patch diff --git a/next-5.10/perf-trace-handle-failure-when-trace-point-folder-is.patch b/queue-5.10/perf-trace-handle-failure-when-trace-point-folder-is.patch similarity index 100% rename from next-5.10/perf-trace-handle-failure-when-trace-point-folder-is.patch rename to queue-5.10/perf-trace-handle-failure-when-trace-point-folder-is.patch diff --git a/next-5.10/perf-trace-return-error-if-a-system-call-doesn-t-exi.patch b/queue-5.10/perf-trace-return-error-if-a-system-call-doesn-t-exi.patch similarity index 100% rename from next-5.10/perf-trace-return-error-if-a-system-call-doesn-t-exi.patch rename to queue-5.10/perf-trace-return-error-if-a-system-call-doesn-t-exi.patch diff --git a/next-5.10/perf-trace-use-macro-raw_syscall_args_num-to-replace.patch b/queue-5.10/perf-trace-use-macro-raw_syscall_args_num-to-replace.patch similarity index 100% rename from next-5.10/perf-trace-use-macro-raw_syscall_args_num-to-replace.patch rename to queue-5.10/perf-trace-use-macro-raw_syscall_args_num-to-replace.patch diff --git a/next-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-__.patch b/queue-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-__.patch similarity index 100% rename from next-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-__.patch rename to queue-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-__.patch diff --git a/next-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-hs.patch b/queue-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-hs.patch similarity index 100% rename from next-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-hs.patch rename to queue-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-hs.patch diff --git a/next-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-sn.patch b/queue-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-sn.patch similarity index 100% rename from next-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-sn.patch rename to queue-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-sn.patch diff --git a/next-5.10/phy-usb-s2-wol-wakeup_count-not-incremented-for-usb-.patch b/queue-5.10/phy-usb-s2-wol-wakeup_count-not-incremented-for-usb-.patch similarity index 100% rename from next-5.10/phy-usb-s2-wol-wakeup_count-not-incremented-for-usb-.patch rename to queue-5.10/phy-usb-s2-wol-wakeup_count-not-incremented-for-usb-.patch diff --git a/next-5.10/pinctrl-pinconf-generic-add-missing-of_node_put.patch b/queue-5.10/pinctrl-pinconf-generic-add-missing-of_node_put.patch similarity index 100% rename from next-5.10/pinctrl-pinconf-generic-add-missing-of_node_put.patch rename to queue-5.10/pinctrl-pinconf-generic-add-missing-of_node_put.patch diff --git a/next-5.10/platform-chrome-cros_usbpd_notify-fix-error-handling.patch b/queue-5.10/platform-chrome-cros_usbpd_notify-fix-error-handling.patch similarity index 100% rename from next-5.10/platform-chrome-cros_usbpd_notify-fix-error-handling.patch rename to queue-5.10/platform-chrome-cros_usbpd_notify-fix-error-handling.patch diff --git a/next-5.10/platform-x86-huawei-wmi-fix-return-value-calculation.patch b/queue-5.10/platform-x86-huawei-wmi-fix-return-value-calculation.patch similarity index 100% rename from next-5.10/platform-x86-huawei-wmi-fix-return-value-calculation.patch rename to queue-5.10/platform-x86-huawei-wmi-fix-return-value-calculation.patch diff --git a/next-5.10/platform-x86-intel_scu_ipc-fix-possible-name-leak-in.patch b/queue-5.10/platform-x86-intel_scu_ipc-fix-possible-name-leak-in.patch similarity index 100% rename from next-5.10/platform-x86-intel_scu_ipc-fix-possible-name-leak-in.patch rename to queue-5.10/platform-x86-intel_scu_ipc-fix-possible-name-leak-in.patch diff --git a/next-5.10/platform-x86-mxm-wmi-fix-memleak-in-mxm_wmi_call_mx-.patch b/queue-5.10/platform-x86-mxm-wmi-fix-memleak-in-mxm_wmi_call_mx-.patch similarity index 100% rename from next-5.10/platform-x86-mxm-wmi-fix-memleak-in-mxm_wmi_call_mx-.patch rename to queue-5.10/platform-x86-mxm-wmi-fix-memleak-in-mxm_wmi_call_mx-.patch diff --git a/next-5.10/pm-hibernate-fix-mistake-in-kerneldoc-comment.patch b/queue-5.10/pm-hibernate-fix-mistake-in-kerneldoc-comment.patch similarity index 100% rename from next-5.10/pm-hibernate-fix-mistake-in-kerneldoc-comment.patch rename to queue-5.10/pm-hibernate-fix-mistake-in-kerneldoc-comment.patch diff --git a/next-5.10/pm-runtime-do-not-call-__rpm_callback-from-rpm_idle.patch b/queue-5.10/pm-runtime-do-not-call-__rpm_callback-from-rpm_idle.patch similarity index 100% rename from next-5.10/pm-runtime-do-not-call-__rpm_callback-from-rpm_idle.patch rename to queue-5.10/pm-runtime-do-not-call-__rpm_callback-from-rpm_idle.patch diff --git a/next-5.10/pm-runtime-improve-path-in-rpm_idle-when-no-callback.patch b/queue-5.10/pm-runtime-improve-path-in-rpm_idle-when-no-callback.patch similarity index 100% rename from next-5.10/pm-runtime-improve-path-in-rpm_idle-when-no-callback.patch rename to queue-5.10/pm-runtime-improve-path-in-rpm_idle-when-no-callback.patch diff --git a/next-5.10/pnode-terminate-at-peers-of-source.patch b/queue-5.10/pnode-terminate-at-peers-of-source.patch similarity index 100% rename from next-5.10/pnode-terminate-at-peers-of-source.patch rename to queue-5.10/pnode-terminate-at-peers-of-source.patch diff --git a/next-5.10/pnp-fix-name-memory-leak-in-pnp_alloc_dev.patch b/queue-5.10/pnp-fix-name-memory-leak-in-pnp_alloc_dev.patch similarity index 100% rename from next-5.10/pnp-fix-name-memory-leak-in-pnp_alloc_dev.patch rename to queue-5.10/pnp-fix-name-memory-leak-in-pnp_alloc_dev.patch diff --git a/next-5.10/power-supply-fix-null-pointer-dereferencing-in-power.patch b/queue-5.10/power-supply-fix-null-pointer-dereferencing-in-power.patch similarity index 100% rename from next-5.10/power-supply-fix-null-pointer-dereferencing-in-power.patch rename to queue-5.10/power-supply-fix-null-pointer-dereferencing-in-power.patch diff --git a/next-5.10/power-supply-fix-residue-sysfs-file-in-error-handle-.patch b/queue-5.10/power-supply-fix-residue-sysfs-file-in-error-handle-.patch similarity index 100% rename from next-5.10/power-supply-fix-residue-sysfs-file-in-error-handle-.patch rename to queue-5.10/power-supply-fix-residue-sysfs-file-in-error-handle-.patch diff --git a/next-5.10/powerpc-52xx-fix-a-resource-leak-in-an-error-handlin.patch b/queue-5.10/powerpc-52xx-fix-a-resource-leak-in-an-error-handlin.patch similarity index 100% rename from next-5.10/powerpc-52xx-fix-a-resource-leak-in-an-error-handlin.patch rename to queue-5.10/powerpc-52xx-fix-a-resource-leak-in-an-error-handlin.patch diff --git a/next-5.10/powerpc-83xx-mpc832x_rdb-call-platform_device_put-in.patch b/queue-5.10/powerpc-83xx-mpc832x_rdb-call-platform_device_put-in.patch similarity index 100% rename from next-5.10/powerpc-83xx-mpc832x_rdb-call-platform_device_put-in.patch rename to queue-5.10/powerpc-83xx-mpc832x_rdb-call-platform_device_put-in.patch diff --git a/next-5.10/powerpc-eeh-drop-redundant-spinlock-initialization.patch b/queue-5.10/powerpc-eeh-drop-redundant-spinlock-initialization.patch similarity index 100% rename from next-5.10/powerpc-eeh-drop-redundant-spinlock-initialization.patch rename to queue-5.10/powerpc-eeh-drop-redundant-spinlock-initialization.patch diff --git a/next-5.10/powerpc-hv-gpci-fix-hv_gpci-event-list.patch b/queue-5.10/powerpc-hv-gpci-fix-hv_gpci-event-list.patch similarity index 100% rename from next-5.10/powerpc-hv-gpci-fix-hv_gpci-event-list.patch rename to queue-5.10/powerpc-hv-gpci-fix-hv_gpci-event-list.patch diff --git a/next-5.10/powerpc-perf-callchain-validate-kernel-stack-pointer.patch b/queue-5.10/powerpc-perf-callchain-validate-kernel-stack-pointer.patch similarity index 100% rename from next-5.10/powerpc-perf-callchain-validate-kernel-stack-pointer.patch rename to queue-5.10/powerpc-perf-callchain-validate-kernel-stack-pointer.patch diff --git a/next-5.10/powerpc-pseries-eeh-use-correct-api-for-error-log-si.patch b/queue-5.10/powerpc-pseries-eeh-use-correct-api-for-error-log-si.patch similarity index 100% rename from next-5.10/powerpc-pseries-eeh-use-correct-api-for-error-log-si.patch rename to queue-5.10/powerpc-pseries-eeh-use-correct-api-for-error-log-si.patch diff --git a/next-5.10/powerpc-rtas-avoid-device-tree-lookups-in-rtas_os_te.patch b/queue-5.10/powerpc-rtas-avoid-device-tree-lookups-in-rtas_os_te.patch similarity index 100% rename from next-5.10/powerpc-rtas-avoid-device-tree-lookups-in-rtas_os_te.patch rename to queue-5.10/powerpc-rtas-avoid-device-tree-lookups-in-rtas_os_te.patch diff --git a/next-5.10/powerpc-rtas-avoid-scheduling-in-rtas_os_term.patch b/queue-5.10/powerpc-rtas-avoid-scheduling-in-rtas_os_term.patch similarity index 100% rename from next-5.10/powerpc-rtas-avoid-scheduling-in-rtas_os_term.patch rename to queue-5.10/powerpc-rtas-avoid-scheduling-in-rtas_os_term.patch diff --git a/next-5.10/powerpc-xive-add-missing-iounmap-in-error-path-in-xi.patch b/queue-5.10/powerpc-xive-add-missing-iounmap-in-error-path-in-xi.patch similarity index 100% rename from next-5.10/powerpc-xive-add-missing-iounmap-in-error-path-in-xi.patch rename to queue-5.10/powerpc-xive-add-missing-iounmap-in-error-path-in-xi.patch diff --git a/next-5.10/powerpc-xmon-enable-breakpoints-on-8xx.patch b/queue-5.10/powerpc-xmon-enable-breakpoints-on-8xx.patch similarity index 100% rename from next-5.10/powerpc-xmon-enable-breakpoints-on-8xx.patch rename to queue-5.10/powerpc-xmon-enable-breakpoints-on-8xx.patch diff --git a/next-5.10/powerpc-xmon-fix-wswitch-unreachable-warning-in-bpt_.patch b/queue-5.10/powerpc-xmon-fix-wswitch-unreachable-warning-in-bpt_.patch similarity index 100% rename from next-5.10/powerpc-xmon-fix-wswitch-unreachable-warning-in-bpt_.patch rename to queue-5.10/powerpc-xmon-fix-wswitch-unreachable-warning-in-bpt_.patch diff --git a/next-5.10/ppp-associate-skb-with-a-device-at-tx.patch b/queue-5.10/ppp-associate-skb-with-a-device-at-tx.patch similarity index 100% rename from next-5.10/ppp-associate-skb-with-a-device-at-tx.patch rename to queue-5.10/ppp-associate-skb-with-a-device-at-tx.patch diff --git a/next-5.10/proc-fixup-uptime-selftest.patch b/queue-5.10/proc-fixup-uptime-selftest.patch similarity index 100% rename from next-5.10/proc-fixup-uptime-selftest.patch rename to queue-5.10/proc-fixup-uptime-selftest.patch diff --git a/next-5.10/pstore-avoid-kcore-oops-by-vmap-ing-with-vm_ioremap.patch b/queue-5.10/pstore-avoid-kcore-oops-by-vmap-ing-with-vm_ioremap.patch similarity index 100% rename from next-5.10/pstore-avoid-kcore-oops-by-vmap-ing-with-vm_ioremap.patch rename to queue-5.10/pstore-avoid-kcore-oops-by-vmap-ing-with-vm_ioremap.patch diff --git a/next-5.10/pstore-make-sure-config_pstore_pmsg-selects-config_r.patch b/queue-5.10/pstore-make-sure-config_pstore_pmsg-selects-config_r.patch similarity index 100% rename from next-5.10/pstore-make-sure-config_pstore_pmsg-selects-config_r.patch rename to queue-5.10/pstore-make-sure-config_pstore_pmsg-selects-config_r.patch diff --git a/next-5.10/pstore-ram-fix-error-return-code-in-ramoops_probe.patch b/queue-5.10/pstore-ram-fix-error-return-code-in-ramoops_probe.patch similarity index 100% rename from next-5.10/pstore-ram-fix-error-return-code-in-ramoops_probe.patch rename to queue-5.10/pstore-ram-fix-error-return-code-in-ramoops_probe.patch diff --git a/next-5.10/pstore-switch-pmsg_lock-to-an-rt_mutex-to-avoid-prio.patch b/queue-5.10/pstore-switch-pmsg_lock-to-an-rt_mutex-to-avoid-prio.patch similarity index 100% rename from next-5.10/pstore-switch-pmsg_lock-to-an-rt_mutex-to-avoid-prio.patch rename to queue-5.10/pstore-switch-pmsg_lock-to-an-rt_mutex-to-avoid-prio.patch diff --git a/next-5.10/pstore-zone-use-gfp_atomic-to-allocate-zone-buffer.patch b/queue-5.10/pstore-zone-use-gfp_atomic-to-allocate-zone-buffer.patch similarity index 100% rename from next-5.10/pstore-zone-use-gfp_atomic-to-allocate-zone-buffer.patch rename to queue-5.10/pstore-zone-use-gfp_atomic-to-allocate-zone-buffer.patch diff --git a/next-5.10/pwm-sifive-call-pwm_sifive_update_clock-while-mutex-.patch b/queue-5.10/pwm-sifive-call-pwm_sifive_update_clock-while-mutex-.patch similarity index 100% rename from next-5.10/pwm-sifive-call-pwm_sifive_update_clock-while-mutex-.patch rename to queue-5.10/pwm-sifive-call-pwm_sifive_update_clock-while-mutex-.patch diff --git a/next-5.10/pwm-tegra-fix-32-bit-build.patch b/queue-5.10/pwm-tegra-fix-32-bit-build.patch similarity index 100% rename from next-5.10/pwm-tegra-fix-32-bit-build.patch rename to queue-5.10/pwm-tegra-fix-32-bit-build.patch diff --git a/next-5.10/pwm-tegra-improve-required-rate-calculation.patch b/queue-5.10/pwm-tegra-improve-required-rate-calculation.patch similarity index 100% rename from next-5.10/pwm-tegra-improve-required-rate-calculation.patch rename to queue-5.10/pwm-tegra-improve-required-rate-calculation.patch diff --git a/next-5.10/qed-gcc13-use-u16-for-fid-to-be-big-enough.patch b/queue-5.10/qed-gcc13-use-u16-for-fid-to-be-big-enough.patch similarity index 100% rename from next-5.10/qed-gcc13-use-u16-for-fid-to-be-big-enough.patch rename to queue-5.10/qed-gcc13-use-u16-for-fid-to-be-big-enough.patch diff --git a/next-5.10/r6040-fix-kmemleak-in-probe-and-remove.patch b/queue-5.10/r6040-fix-kmemleak-in-probe-and-remove.patch similarity index 100% rename from next-5.10/r6040-fix-kmemleak-in-probe-and-remove.patch rename to queue-5.10/r6040-fix-kmemleak-in-probe-and-remove.patch diff --git a/next-5.10/rapidio-devices-fix-missing-put_device-in-mport_cdev.patch b/queue-5.10/rapidio-devices-fix-missing-put_device-in-mport_cdev.patch similarity index 100% rename from next-5.10/rapidio-devices-fix-missing-put_device-in-mport_cdev.patch rename to queue-5.10/rapidio-devices-fix-missing-put_device-in-mport_cdev.patch diff --git a/next-5.10/rapidio-fix-possible-name-leaks-when-rio_add_device-.patch b/queue-5.10/rapidio-fix-possible-name-leaks-when-rio_add_device-.patch similarity index 100% rename from next-5.10/rapidio-fix-possible-name-leaks-when-rio_add_device-.patch rename to queue-5.10/rapidio-fix-possible-name-leaks-when-rio_add_device-.patch diff --git a/next-5.10/rapidio-fix-possible-uaf-when-kfifo_alloc-fails.patch b/queue-5.10/rapidio-fix-possible-uaf-when-kfifo_alloc-fails.patch similarity index 100% rename from next-5.10/rapidio-fix-possible-uaf-when-kfifo_alloc-fails.patch rename to queue-5.10/rapidio-fix-possible-uaf-when-kfifo_alloc-fails.patch diff --git a/next-5.10/rapidio-rio-fix-possible-name-leak-in-rio_register_m.patch b/queue-5.10/rapidio-rio-fix-possible-name-leak-in-rio_register_m.patch similarity index 100% rename from next-5.10/rapidio-rio-fix-possible-name-leak-in-rio_register_m.patch rename to queue-5.10/rapidio-rio-fix-possible-name-leak-in-rio_register_m.patch diff --git a/next-5.10/rcu-fix-__this_cpu_read-lockdep-warning-in-rcu_force.patch b/queue-5.10/rcu-fix-__this_cpu_read-lockdep-warning-in-rcu_force.patch similarity index 100% rename from next-5.10/rcu-fix-__this_cpu_read-lockdep-warning-in-rcu_force.patch rename to queue-5.10/rcu-fix-__this_cpu_read-lockdep-warning-in-rcu_force.patch diff --git a/next-5.10/rdma-core-fix-order-of-nldev_exit-call.patch b/queue-5.10/rdma-core-fix-order-of-nldev_exit-call.patch similarity index 100% rename from next-5.10/rdma-core-fix-order-of-nldev_exit-call.patch rename to queue-5.10/rdma-core-fix-order-of-nldev_exit-call.patch diff --git a/next-5.10/rdma-hfi-decrease-pci-device-reference-count-in-erro.patch b/queue-5.10/rdma-hfi-decrease-pci-device-reference-count-in-erro.patch similarity index 100% rename from next-5.10/rdma-hfi-decrease-pci-device-reference-count-in-erro.patch rename to queue-5.10/rdma-hfi-decrease-pci-device-reference-count-in-erro.patch diff --git a/next-5.10/rdma-hfi1-fix-error-return-code-in-parse_platform_co.patch b/queue-5.10/rdma-hfi1-fix-error-return-code-in-parse_platform_co.patch similarity index 100% rename from next-5.10/rdma-hfi1-fix-error-return-code-in-parse_platform_co.patch rename to queue-5.10/rdma-hfi1-fix-error-return-code-in-parse_platform_co.patch diff --git a/next-5.10/rdma-hns-fix-ext_sge-num-error-when-post-send.patch b/queue-5.10/rdma-hns-fix-ext_sge-num-error-when-post-send.patch similarity index 100% rename from next-5.10/rdma-hns-fix-ext_sge-num-error-when-post-send.patch rename to queue-5.10/rdma-hns-fix-ext_sge-num-error-when-post-send.patch diff --git a/next-5.10/rdma-hns-fix-memory-leak-in-hns_roce_alloc_mr.patch b/queue-5.10/rdma-hns-fix-memory-leak-in-hns_roce_alloc_mr.patch similarity index 100% rename from next-5.10/rdma-hns-fix-memory-leak-in-hns_roce_alloc_mr.patch rename to queue-5.10/rdma-hns-fix-memory-leak-in-hns_roce_alloc_mr.patch diff --git a/next-5.10/rdma-hns-fix-page-size-cap-from-firmware.patch b/queue-5.10/rdma-hns-fix-page-size-cap-from-firmware.patch similarity index 100% rename from next-5.10/rdma-hns-fix-page-size-cap-from-firmware.patch rename to queue-5.10/rdma-hns-fix-page-size-cap-from-firmware.patch diff --git a/next-5.10/rdma-hns-fix-pbl-page-mtr-find.patch b/queue-5.10/rdma-hns-fix-pbl-page-mtr-find.patch similarity index 100% rename from next-5.10/rdma-hns-fix-pbl-page-mtr-find.patch rename to queue-5.10/rdma-hns-fix-pbl-page-mtr-find.patch diff --git a/next-5.10/rdma-hns-repacing-dseg_len-by-macros-in-fill_ext_sge.patch b/queue-5.10/rdma-hns-repacing-dseg_len-by-macros-in-fill_ext_sge.patch similarity index 100% rename from next-5.10/rdma-hns-repacing-dseg_len-by-macros-in-fill_ext_sge.patch rename to queue-5.10/rdma-hns-repacing-dseg_len-by-macros-in-fill_ext_sge.patch diff --git a/next-5.10/rdma-nldev-add-checks-for-nla_nest_start-in-fill_sta.patch b/queue-5.10/rdma-nldev-add-checks-for-nla_nest_start-in-fill_sta.patch similarity index 100% rename from next-5.10/rdma-nldev-add-checks-for-nla_nest_start-in-fill_sta.patch rename to queue-5.10/rdma-nldev-add-checks-for-nla_nest_start-in-fill_sta.patch diff --git a/next-5.10/rdma-nldev-fix-failure-to-send-large-messages.patch b/queue-5.10/rdma-nldev-fix-failure-to-send-large-messages.patch similarity index 100% rename from next-5.10/rdma-nldev-fix-failure-to-send-large-messages.patch rename to queue-5.10/rdma-nldev-fix-failure-to-send-large-messages.patch diff --git a/next-5.10/rdma-nldev-return-eagain-if-the-cm_id-isn-t-from-exp.patch b/queue-5.10/rdma-nldev-return-eagain-if-the-cm_id-isn-t-from-exp.patch similarity index 100% rename from next-5.10/rdma-nldev-return-eagain-if-the-cm_id-isn-t-from-exp.patch rename to queue-5.10/rdma-nldev-return-eagain-if-the-cm_id-isn-t-from-exp.patch diff --git a/next-5.10/rdma-rxe-fix-null-ptr-deref-in-rxe_qp_do_cleanup-whe.patch b/queue-5.10/rdma-rxe-fix-null-ptr-deref-in-rxe_qp_do_cleanup-whe.patch similarity index 100% rename from next-5.10/rdma-rxe-fix-null-ptr-deref-in-rxe_qp_do_cleanup-whe.patch rename to queue-5.10/rdma-rxe-fix-null-ptr-deref-in-rxe_qp_do_cleanup-whe.patch diff --git a/next-5.10/rdma-siw-fix-immediate-work-request-flush-to-complet.patch b/queue-5.10/rdma-siw-fix-immediate-work-request-flush-to-complet.patch similarity index 100% rename from next-5.10/rdma-siw-fix-immediate-work-request-flush-to-complet.patch rename to queue-5.10/rdma-siw-fix-immediate-work-request-flush-to-complet.patch diff --git a/next-5.10/rdma-siw-fix-pointer-cast-warning.patch b/queue-5.10/rdma-siw-fix-pointer-cast-warning.patch similarity index 100% rename from next-5.10/rdma-siw-fix-pointer-cast-warning.patch rename to queue-5.10/rdma-siw-fix-pointer-cast-warning.patch diff --git a/next-5.10/rdma-siw-set-defined-status-for-work-completion-with.patch b/queue-5.10/rdma-siw-set-defined-status-for-work-completion-with.patch similarity index 100% rename from next-5.10/rdma-siw-set-defined-status-for-work-completion-with.patch rename to queue-5.10/rdma-siw-set-defined-status-for-work-completion-with.patch diff --git a/next-5.10/rdma-srp-fix-error-return-code-in-srp_parse_options.patch b/queue-5.10/rdma-srp-fix-error-return-code-in-srp_parse_options.patch similarity index 100% rename from next-5.10/rdma-srp-fix-error-return-code-in-srp_parse_options.patch rename to queue-5.10/rdma-srp-fix-error-return-code-in-srp_parse_options.patch diff --git a/next-5.10/regulator-core-fix-deadlock-on-regulator-enable.patch b/queue-5.10/regulator-core-fix-deadlock-on-regulator-enable.patch similarity index 100% rename from next-5.10/regulator-core-fix-deadlock-on-regulator-enable.patch rename to queue-5.10/regulator-core-fix-deadlock-on-regulator-enable.patch diff --git a/next-5.10/regulator-core-fix-module-refcount-leak-in-set_suppl.patch b/queue-5.10/regulator-core-fix-module-refcount-leak-in-set_suppl.patch similarity index 100% rename from next-5.10/regulator-core-fix-module-refcount-leak-in-set_suppl.patch rename to queue-5.10/regulator-core-fix-module-refcount-leak-in-set_suppl.patch diff --git a/next-5.10/regulator-core-fix-resource-leak-in-regulator_regist.patch b/queue-5.10/regulator-core-fix-resource-leak-in-regulator_regist.patch similarity index 100% rename from next-5.10/regulator-core-fix-resource-leak-in-regulator_regist.patch rename to queue-5.10/regulator-core-fix-resource-leak-in-regulator_regist.patch diff --git a/next-5.10/regulator-core-fix-unbalanced-of-node-refcount-in-re.patch b/queue-5.10/regulator-core-fix-unbalanced-of-node-refcount-in-re.patch similarity index 100% rename from next-5.10/regulator-core-fix-unbalanced-of-node-refcount-in-re.patch rename to queue-5.10/regulator-core-fix-unbalanced-of-node-refcount-in-re.patch diff --git a/next-5.10/regulator-core-fix-use_count-leakage-when-handling-b.patch b/queue-5.10/regulator-core-fix-use_count-leakage-when-handling-b.patch similarity index 100% rename from next-5.10/regulator-core-fix-use_count-leakage-when-handling-b.patch rename to queue-5.10/regulator-core-fix-use_count-leakage-when-handling-b.patch diff --git a/next-5.10/regulator-core-use-kfree_const-to-free-space-conditi.patch b/queue-5.10/regulator-core-use-kfree_const-to-free-space-conditi.patch similarity index 100% rename from next-5.10/regulator-core-use-kfree_const-to-free-space-conditi.patch rename to queue-5.10/regulator-core-use-kfree_const-to-free-space-conditi.patch diff --git a/next-5.10/reiserfs-add-missing-calls-to-reiserfs_security_free.patch b/queue-5.10/reiserfs-add-missing-calls-to-reiserfs_security_free.patch similarity index 100% rename from next-5.10/reiserfs-add-missing-calls-to-reiserfs_security_free.patch rename to queue-5.10/reiserfs-add-missing-calls-to-reiserfs_security_free.patch diff --git a/next-5.10/relay-fix-type-mismatch-when-allocating-memory-in-re.patch b/queue-5.10/relay-fix-type-mismatch-when-allocating-memory-in-re.patch similarity index 100% rename from next-5.10/relay-fix-type-mismatch-when-allocating-memory-in-re.patch rename to queue-5.10/relay-fix-type-mismatch-when-allocating-memory-in-re.patch diff --git a/next-5.10/remoteproc-qcom_q6v5_pas-detach-power-domains-on-rem.patch b/queue-5.10/remoteproc-qcom_q6v5_pas-detach-power-domains-on-rem.patch similarity index 100% rename from next-5.10/remoteproc-qcom_q6v5_pas-detach-power-domains-on-rem.patch rename to queue-5.10/remoteproc-qcom_q6v5_pas-detach-power-domains-on-rem.patch diff --git a/next-5.10/remoteproc-qcom_q6v5_pas-disable-wakeup-on-probe-fai.patch b/queue-5.10/remoteproc-qcom_q6v5_pas-disable-wakeup-on-probe-fai.patch similarity index 100% rename from next-5.10/remoteproc-qcom_q6v5_pas-disable-wakeup-on-probe-fai.patch rename to queue-5.10/remoteproc-qcom_q6v5_pas-disable-wakeup-on-probe-fai.patch diff --git a/next-5.10/remoteproc-qcom_q6v5_pas-fix-missing-of_node_put-in-.patch b/queue-5.10/remoteproc-qcom_q6v5_pas-fix-missing-of_node_put-in-.patch similarity index 100% rename from next-5.10/remoteproc-qcom_q6v5_pas-fix-missing-of_node_put-in-.patch rename to queue-5.10/remoteproc-qcom_q6v5_pas-fix-missing-of_node_put-in-.patch diff --git a/next-5.10/remoteproc-sysmon-fix-memory-leak-in-qcom_add_sysmon.patch b/queue-5.10/remoteproc-sysmon-fix-memory-leak-in-qcom_add_sysmon.patch similarity index 100% rename from next-5.10/remoteproc-sysmon-fix-memory-leak-in-qcom_add_sysmon.patch rename to queue-5.10/remoteproc-sysmon-fix-memory-leak-in-qcom_add_sysmon.patch diff --git a/next-5.10/riscv-mm-add-arch-hook-arch_clear_hugepage_flags.patch b/queue-5.10/riscv-mm-add-arch-hook-arch_clear_hugepage_flags.patch similarity index 100% rename from next-5.10/riscv-mm-add-arch-hook-arch_clear_hugepage_flags.patch rename to queue-5.10/riscv-mm-add-arch-hook-arch_clear_hugepage_flags.patch diff --git a/next-5.10/rtc-cmos-call-cmos_wake_setup-from-cmos_do_probe.patch b/queue-5.10/rtc-cmos-call-cmos_wake_setup-from-cmos_do_probe.patch similarity index 100% rename from next-5.10/rtc-cmos-call-cmos_wake_setup-from-cmos_do_probe.patch rename to queue-5.10/rtc-cmos-call-cmos_wake_setup-from-cmos_do_probe.patch diff --git a/next-5.10/rtc-cmos-call-rtc_wake_setup-from-cmos_do_probe.patch b/queue-5.10/rtc-cmos-call-rtc_wake_setup-from-cmos_do_probe.patch similarity index 100% rename from next-5.10/rtc-cmos-call-rtc_wake_setup-from-cmos_do_probe.patch rename to queue-5.10/rtc-cmos-call-rtc_wake_setup-from-cmos_do_probe.patch diff --git a/next-5.10/rtc-cmos-disable-acpi-rtc-event-on-removal.patch b/queue-5.10/rtc-cmos-disable-acpi-rtc-event-on-removal.patch similarity index 100% rename from next-5.10/rtc-cmos-disable-acpi-rtc-event-on-removal.patch rename to queue-5.10/rtc-cmos-disable-acpi-rtc-event-on-removal.patch diff --git a/next-5.10/rtc-cmos-eliminate-forward-declarations-of-some-func.patch b/queue-5.10/rtc-cmos-eliminate-forward-declarations-of-some-func.patch similarity index 100% rename from next-5.10/rtc-cmos-eliminate-forward-declarations-of-some-func.patch rename to queue-5.10/rtc-cmos-eliminate-forward-declarations-of-some-func.patch diff --git a/next-5.10/rtc-cmos-fix-build-on-non-acpi-platforms.patch b/queue-5.10/rtc-cmos-fix-build-on-non-acpi-platforms.patch similarity index 100% rename from next-5.10/rtc-cmos-fix-build-on-non-acpi-platforms.patch rename to queue-5.10/rtc-cmos-fix-build-on-non-acpi-platforms.patch diff --git a/next-5.10/rtc-cmos-fix-event-handler-registration-ordering-iss.patch b/queue-5.10/rtc-cmos-fix-event-handler-registration-ordering-iss.patch similarity index 100% rename from next-5.10/rtc-cmos-fix-event-handler-registration-ordering-iss.patch rename to queue-5.10/rtc-cmos-fix-event-handler-registration-ordering-iss.patch diff --git a/next-5.10/rtc-cmos-fix-wake-alarm-breakage.patch b/queue-5.10/rtc-cmos-fix-wake-alarm-breakage.patch similarity index 100% rename from next-5.10/rtc-cmos-fix-wake-alarm-breakage.patch rename to queue-5.10/rtc-cmos-fix-wake-alarm-breakage.patch diff --git a/next-5.10/rtc-cmos-rename-acpi-related-functions.patch b/queue-5.10/rtc-cmos-rename-acpi-related-functions.patch similarity index 100% rename from next-5.10/rtc-cmos-rename-acpi-related-functions.patch rename to queue-5.10/rtc-cmos-rename-acpi-related-functions.patch diff --git a/next-5.10/rtc-mxc_v2-add-missing-clk_disable_unprepare.patch b/queue-5.10/rtc-mxc_v2-add-missing-clk_disable_unprepare.patch similarity index 100% rename from next-5.10/rtc-mxc_v2-add-missing-clk_disable_unprepare.patch rename to queue-5.10/rtc-mxc_v2-add-missing-clk_disable_unprepare.patch diff --git a/next-5.10/rtc-pcf85063-fix-pcf85063_clkout_control.patch b/queue-5.10/rtc-pcf85063-fix-pcf85063_clkout_control.patch similarity index 100% rename from next-5.10/rtc-pcf85063-fix-pcf85063_clkout_control.patch rename to queue-5.10/rtc-pcf85063-fix-pcf85063_clkout_control.patch diff --git a/next-5.10/rtc-pcf85063-fix-reading-alarm.patch b/queue-5.10/rtc-pcf85063-fix-reading-alarm.patch similarity index 100% rename from next-5.10/rtc-pcf85063-fix-reading-alarm.patch rename to queue-5.10/rtc-pcf85063-fix-reading-alarm.patch diff --git a/next-5.10/rtc-pic32-move-devm_rtc_allocate_device-earlier-in-p.patch b/queue-5.10/rtc-pic32-move-devm_rtc_allocate_device-earlier-in-p.patch similarity index 100% rename from next-5.10/rtc-pic32-move-devm_rtc_allocate_device-earlier-in-p.patch rename to queue-5.10/rtc-pic32-move-devm_rtc_allocate_device-earlier-in-p.patch diff --git a/next-5.10/rtc-rtc-cmos-do-not-check-acpi_fadt_low_power_s0.patch b/queue-5.10/rtc-rtc-cmos-do-not-check-acpi_fadt_low_power_s0.patch similarity index 100% rename from next-5.10/rtc-rtc-cmos-do-not-check-acpi_fadt_low_power_s0.patch rename to queue-5.10/rtc-rtc-cmos-do-not-check-acpi_fadt_low_power_s0.patch diff --git a/next-5.10/rtc-snvs-allow-a-time-difference-on-clock-register-r.patch b/queue-5.10/rtc-snvs-allow-a-time-difference-on-clock-register-r.patch similarity index 100% rename from next-5.10/rtc-snvs-allow-a-time-difference-on-clock-register-r.patch rename to queue-5.10/rtc-snvs-allow-a-time-difference-on-clock-register-r.patch diff --git a/next-5.10/rtc-st-lpc-add-missing-clk_disable_unprepare-in-st_r.patch b/queue-5.10/rtc-st-lpc-add-missing-clk_disable_unprepare-in-st_r.patch similarity index 100% rename from next-5.10/rtc-st-lpc-add-missing-clk_disable_unprepare-in-st_r.patch rename to queue-5.10/rtc-st-lpc-add-missing-clk_disable_unprepare-in-st_r.patch diff --git a/next-5.10/rxrpc-fix-ack.buffersize-to-be-0-when-generating-an-.patch b/queue-5.10/rxrpc-fix-ack.buffersize-to-be-0-when-generating-an-.patch similarity index 100% rename from next-5.10/rxrpc-fix-ack.buffersize-to-be-0-when-generating-an-.patch rename to queue-5.10/rxrpc-fix-ack.buffersize-to-be-0-when-generating-an-.patch diff --git a/next-5.10/rxrpc-fix-missing-unlock-in-rxrpc_do_sendmsg.patch b/queue-5.10/rxrpc-fix-missing-unlock-in-rxrpc_do_sendmsg.patch similarity index 100% rename from next-5.10/rxrpc-fix-missing-unlock-in-rxrpc_do_sendmsg.patch rename to queue-5.10/rxrpc-fix-missing-unlock-in-rxrpc_do_sendmsg.patch diff --git a/next-5.10/s390-ctcm-fix-return-type-of-ctc-mp-m_tx.patch b/queue-5.10/s390-ctcm-fix-return-type-of-ctc-mp-m_tx.patch similarity index 100% rename from next-5.10/s390-ctcm-fix-return-type-of-ctc-mp-m_tx.patch rename to queue-5.10/s390-ctcm-fix-return-type-of-ctc-mp-m_tx.patch diff --git a/next-5.10/s390-lcs-fix-return-type-of-lcs_start_xmit.patch b/queue-5.10/s390-lcs-fix-return-type-of-lcs_start_xmit.patch similarity index 100% rename from next-5.10/s390-lcs-fix-return-type-of-lcs_start_xmit.patch rename to queue-5.10/s390-lcs-fix-return-type-of-lcs_start_xmit.patch diff --git a/next-5.10/s390-netiucv-fix-return-type-of-netiucv_tx.patch b/queue-5.10/s390-netiucv-fix-return-type-of-netiucv_tx.patch similarity index 100% rename from next-5.10/s390-netiucv-fix-return-type-of-netiucv_tx.patch rename to queue-5.10/s390-netiucv-fix-return-type-of-netiucv_tx.patch diff --git a/next-5.10/samples-vfio-mdev-fix-missing-pci_disable_device-in-.patch b/queue-5.10/samples-vfio-mdev-fix-missing-pci_disable_device-in-.patch similarity index 100% rename from next-5.10/samples-vfio-mdev-fix-missing-pci_disable_device-in-.patch rename to queue-5.10/samples-vfio-mdev-fix-missing-pci_disable_device-in-.patch diff --git a/next-5.10/sched-fair-cleanup-task_util-and-capacity-type.patch b/queue-5.10/sched-fair-cleanup-task_util-and-capacity-type.patch similarity index 100% rename from next-5.10/sched-fair-cleanup-task_util-and-capacity-type.patch rename to queue-5.10/sched-fair-cleanup-task_util-and-capacity-type.patch diff --git a/next-5.10/sched-uclamp-fix-relationship-between-uclamp-and-mig.patch b/queue-5.10/sched-uclamp-fix-relationship-between-uclamp-and-mig.patch similarity index 100% rename from next-5.10/sched-uclamp-fix-relationship-between-uclamp-and-mig.patch rename to queue-5.10/sched-uclamp-fix-relationship-between-uclamp-and-mig.patch diff --git a/next-5.10/scsi-core-fix-a-race-between-scsi_done-and-scsi_time.patch b/queue-5.10/scsi-core-fix-a-race-between-scsi_done-and-scsi_time.patch similarity index 100% rename from next-5.10/scsi-core-fix-a-race-between-scsi_done-and-scsi_time.patch rename to queue-5.10/scsi-core-fix-a-race-between-scsi_done-and-scsi_time.patch diff --git a/next-5.10/scsi-fcoe-fix-possible-name-leak-when-device_registe.patch b/queue-5.10/scsi-fcoe-fix-possible-name-leak-when-device_registe.patch similarity index 100% rename from next-5.10/scsi-fcoe-fix-possible-name-leak-when-device_registe.patch rename to queue-5.10/scsi-fcoe-fix-possible-name-leak-when-device_registe.patch diff --git a/next-5.10/scsi-fcoe-fix-transport-not-deattached-when-fcoe_if_.patch b/queue-5.10/scsi-fcoe-fix-transport-not-deattached-when-fcoe_if_.patch similarity index 100% rename from next-5.10/scsi-fcoe-fix-transport-not-deattached-when-fcoe_if_.patch rename to queue-5.10/scsi-fcoe-fix-transport-not-deattached-when-fcoe_if_.patch diff --git a/next-5.10/scsi-hpsa-fix-error-handling-in-hpsa_add_sas_host.patch b/queue-5.10/scsi-hpsa-fix-error-handling-in-hpsa_add_sas_host.patch similarity index 100% rename from next-5.10/scsi-hpsa-fix-error-handling-in-hpsa_add_sas_host.patch rename to queue-5.10/scsi-hpsa-fix-error-handling-in-hpsa_add_sas_host.patch diff --git a/next-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_add_sas_d.patch b/queue-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_add_sas_d.patch similarity index 100% rename from next-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_add_sas_d.patch rename to queue-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_add_sas_d.patch diff --git a/next-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_init_one.patch b/queue-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_init_one.patch similarity index 100% rename from next-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_init_one.patch rename to queue-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_init_one.patch diff --git a/next-5.10/scsi-ipr-fix-warning-in-ipr_init.patch b/queue-5.10/scsi-ipr-fix-warning-in-ipr_init.patch similarity index 100% rename from next-5.10/scsi-ipr-fix-warning-in-ipr_init.patch rename to queue-5.10/scsi-ipr-fix-warning-in-ipr_init.patch diff --git a/next-5.10/scsi-mpt3sas-fix-possible-resource-leaks-in-mpt3sas_.patch b/queue-5.10/scsi-mpt3sas-fix-possible-resource-leaks-in-mpt3sas_.patch similarity index 100% rename from next-5.10/scsi-mpt3sas-fix-possible-resource-leaks-in-mpt3sas_.patch rename to queue-5.10/scsi-mpt3sas-fix-possible-resource-leaks-in-mpt3sas_.patch diff --git a/next-5.10/scsi-scsi_debug-fix-a-warning-in-resp_report_zones.patch b/queue-5.10/scsi-scsi_debug-fix-a-warning-in-resp_report_zones.patch similarity index 100% rename from next-5.10/scsi-scsi_debug-fix-a-warning-in-resp_report_zones.patch rename to queue-5.10/scsi-scsi_debug-fix-a-warning-in-resp_report_zones.patch diff --git a/next-5.10/scsi-scsi_debug-fix-a-warning-in-resp_verify.patch b/queue-5.10/scsi-scsi_debug-fix-a-warning-in-resp_verify.patch similarity index 100% rename from next-5.10/scsi-scsi_debug-fix-a-warning-in-resp_verify.patch rename to queue-5.10/scsi-scsi_debug-fix-a-warning-in-resp_verify.patch diff --git a/next-5.10/scsi-scsi_debug-fix-a-warning-in-resp_write_scat.patch b/queue-5.10/scsi-scsi_debug-fix-a-warning-in-resp_write_scat.patch similarity index 100% rename from next-5.10/scsi-scsi_debug-fix-a-warning-in-resp_write_scat.patch rename to queue-5.10/scsi-scsi_debug-fix-a-warning-in-resp_write_scat.patch diff --git a/next-5.10/scsi-scsi_debug-fix-possible-name-leak-in-sdebug_add.patch b/queue-5.10/scsi-scsi_debug-fix-possible-name-leak-in-sdebug_add.patch similarity index 100% rename from next-5.10/scsi-scsi_debug-fix-possible-name-leak-in-sdebug_add.patch rename to queue-5.10/scsi-scsi_debug-fix-possible-name-leak-in-sdebug_add.patch diff --git a/next-5.10/scsi-snic-fix-possible-uaf-in-snic_tgt_create.patch b/queue-5.10/scsi-snic-fix-possible-uaf-in-snic_tgt_create.patch similarity index 100% rename from next-5.10/scsi-snic-fix-possible-uaf-in-snic_tgt_create.patch rename to queue-5.10/scsi-snic-fix-possible-uaf-in-snic_tgt_create.patch diff --git a/next-5.10/sctp-sysctl-make-extra-pointers-netns-aware.patch b/queue-5.10/sctp-sysctl-make-extra-pointers-netns-aware.patch similarity index 100% rename from next-5.10/sctp-sysctl-make-extra-pointers-netns-aware.patch rename to queue-5.10/sctp-sysctl-make-extra-pointers-netns-aware.patch diff --git a/next-5.10/selftests-devlink-fix-the-fd-redirect-in-dummy_repor.patch b/queue-5.10/selftests-devlink-fix-the-fd-redirect-in-dummy_repor.patch similarity index 100% rename from next-5.10/selftests-devlink-fix-the-fd-redirect-in-dummy_repor.patch rename to queue-5.10/selftests-devlink-fix-the-fd-redirect-in-dummy_repor.patch diff --git a/next-5.10/selftests-efivarfs-add-checking-of-the-test-return-v.patch b/queue-5.10/selftests-efivarfs-add-checking-of-the-test-return-v.patch similarity index 100% rename from next-5.10/selftests-efivarfs-add-checking-of-the-test-return-v.patch rename to queue-5.10/selftests-efivarfs-add-checking-of-the-test-return-v.patch diff --git a/next-5.10/selftests-ftrace-event_triggers-wait-longer-for-test.patch b/queue-5.10/selftests-ftrace-event_triggers-wait-longer-for-test.patch similarity index 100% rename from next-5.10/selftests-ftrace-event_triggers-wait-longer-for-test.patch rename to queue-5.10/selftests-ftrace-event_triggers-wait-longer-for-test.patch diff --git a/next-5.10/selftests-powerpc-fix-resource-leaks.patch b/queue-5.10/selftests-powerpc-fix-resource-leaks.patch similarity index 100% rename from next-5.10/selftests-powerpc-fix-resource-leaks.patch rename to queue-5.10/selftests-powerpc-fix-resource-leaks.patch diff --git a/next-5.10/serial-altera_uart-fix-locking-in-polling-mode.patch b/queue-5.10/serial-altera_uart-fix-locking-in-polling-mode.patch similarity index 100% rename from next-5.10/serial-altera_uart-fix-locking-in-polling-mode.patch rename to queue-5.10/serial-altera_uart-fix-locking-in-polling-mode.patch diff --git a/next-5.10/serial-amba-pl011-avoid-sbsa-uart-accessing-dmacr-re.patch b/queue-5.10/serial-amba-pl011-avoid-sbsa-uart-accessing-dmacr-re.patch similarity index 100% rename from next-5.10/serial-amba-pl011-avoid-sbsa-uart-accessing-dmacr-re.patch rename to queue-5.10/serial-amba-pl011-avoid-sbsa-uart-accessing-dmacr-re.patch diff --git a/next-5.10/serial-pch-fix-pci-device-refcount-leak-in-pch_reque.patch b/queue-5.10/serial-pch-fix-pci-device-refcount-leak-in-pch_reque.patch similarity index 100% rename from next-5.10/serial-pch-fix-pci-device-refcount-leak-in-pch_reque.patch rename to queue-5.10/serial-pch-fix-pci-device-refcount-leak-in-pch_reque.patch diff --git a/next-5.10/serial-pl011-do-not-clear-rx-fifo-rx-interrupt-in-un.patch b/queue-5.10/serial-pl011-do-not-clear-rx-fifo-rx-interrupt-in-un.patch similarity index 100% rename from next-5.10/serial-pl011-do-not-clear-rx-fifo-rx-interrupt-in-un.patch rename to queue-5.10/serial-pl011-do-not-clear-rx-fifo-rx-interrupt-in-un.patch diff --git a/next-5.10/serial-sunsab-fix-error-handling-in-sunsab_init.patch b/queue-5.10/serial-sunsab-fix-error-handling-in-sunsab_init.patch similarity index 100% rename from next-5.10/serial-sunsab-fix-error-handling-in-sunsab_init.patch rename to queue-5.10/serial-sunsab-fix-error-handling-in-sunsab_init.patch diff --git a/next-5.10/serial-tegra-read-dma-status-before-terminating.patch b/queue-5.10/serial-tegra-read-dma-status-before-terminating.patch similarity index 100% rename from next-5.10/serial-tegra-read-dma-status-before-terminating.patch rename to queue-5.10/serial-tegra-read-dma-status-before-terminating.patch diff --git a/next-5.10/series b/queue-5.10/series similarity index 90% rename from next-5.10/series rename to queue-5.10/series index ba4de0dadb8..5cf91469d0d 100644 --- a/next-5.10/series +++ b/queue-5.10/series @@ -598,66 +598,3 @@ tpm-tpm_crb-add-the-missed-acpi_put_table-to-fix-memory-leak.patch tpm-tpm_tis-add-the-missed-acpi_put_table-to-fix-memory-leak.patch sunrpc-don-t-leak-netobj-memory-when-gss_read_proxy_verf-fails.patch kcsan-instrument-memcpy-memset-memmove-with-newer-clang.patch -kernel-provide-create_io_thread-helper.patch -iov_iter-add-helper-to-save-iov_iter-state.patch -saner-calling-conventions-for-unlazy_child.patch -fs-add-support-for-lookup_cached.patch -fix-handling-of-nd-depth-on-lookup_cached-failures-in-try_to_unlazy.patch -make-sure-nd-path.mnt-and-nd-path.dentry-are-always-valid-pointers.patch -fs-expose-lookup_cached-through-openat2-resolve_cached.patch -tools-headers-uapi-sync-openat2.h-with-the-kernel-sources.patch -net-provide-__sys_shutdown_sock-that-takes-a-socket.patch -net-add-accept-helper-not-installing-fd.patch -signal-add-task_sigpending-helper.patch -fs-make-do_renameat2-take-struct-filename.patch -file-rename-__close_fd_get_file-close_fd_get_file.patch -fs-provide-locked-helper-variant-of-close_fd_get_file.patch -entry-add-support-for-tif_notify_signal.patch -task_work-use-tif_notify_signal-if-available.patch -x86-wire-up-tif_notify_signal.patch -arc-add-support-for-tif_notify_signal.patch -arm64-add-support-for-tif_notify_signal.patch -m68k-add-support-for-tif_notify_signal.patch -nios32-add-support-for-tif_notify_signal.patch -parisc-add-support-for-tif_notify_signal.patch -powerpc-add-support-for-tif_notify_signal.patch -mips-add-support-for-tif_notify_signal.patch -s390-add-support-for-tif_notify_signal.patch -um-add-support-for-tif_notify_signal.patch -sh-add-support-for-tif_notify_signal.patch -openrisc-add-support-for-tif_notify_signal.patch -csky-add-support-for-tif_notify_signal.patch -hexagon-add-support-for-tif_notify_signal.patch -microblaze-add-support-for-tif_notify_signal.patch -arm-add-support-for-tif_notify_signal.patch -xtensa-add-support-for-tif_notify_signal.patch -alpha-add-support-for-tif_notify_signal.patch -c6x-add-support-for-tif_notify_signal.patch -h8300-add-support-for-tif_notify_signal.patch -ia64-add-support-for-tif_notify_signal.patch -nds32-add-support-for-tif_notify_signal.patch -riscv-add-support-for-tif_notify_signal.patch -sparc-add-support-for-tif_notify_signal.patch -ia64-don-t-call-handle_signal-unless-there-s-actually-a-signal-queued.patch -arc-unbork-5.11-bootup-fix-snafu-in-_tif_notify_signal-handling.patch -alpha-fix-tif_notify_signal-handling.patch -task_work-remove-legacy-twa_signal-path.patch -kernel-remove-checking-for-tif_notify_signal.patch -coredump-limit-what-can-interrupt-coredumps.patch -kernel-allow-fork-with-tif_notify_signal-pending.patch -entry-kvm-exit-to-user-mode-when-tif_notify_signal-is-set.patch -arch-setup-pf_io_worker-threads-like-pf_kthread.patch -arch-ensure-parisc-powerpc-handle-pf_io_worker-in-copy_thread.patch -x86-process-setup-io_threads-more-like-normal-user-space-threads.patch -kernel-stop-masking-signals-in-create_io_thread.patch -kernel-don-t-call-do_exit-for-pf_io_worker-threads.patch -task_work-add-helper-for-more-targeted-task_work-canceling.patch -io_uring-import-5.15-stable-io_uring.patch -signal-kill-jobctl_task_work.patch -task_work-unconditionally-run-task_work-from-get_signal.patch -net-remove-cmsg-restriction-from-io_uring-based-send-recvmsg-calls.patch -revert-proc-don-t-allow-async-path-resolution-of-proc-thread-self-components.patch -revert-proc-don-t-allow-async-path-resolution-of-proc-self-components.patch -eventpoll-add-epoll_uring_wake-poll-wakeup-flag.patch -eventfd-provide-a-eventfd_signal_mask-helper.patch -io_uring-pass-in-epoll_uring_wake-for-eventfd-signaling-and-wakeups.patch diff --git a/next-5.10/skbuff-account-for-tail-adjustment-during-pull-opera.patch b/queue-5.10/skbuff-account-for-tail-adjustment-during-pull-opera.patch similarity index 100% rename from next-5.10/skbuff-account-for-tail-adjustment-during-pull-opera.patch rename to queue-5.10/skbuff-account-for-tail-adjustment-during-pull-opera.patch diff --git a/next-5.10/soc-qcom-apr-add-check-for-idr_alloc-and-of_property.patch b/queue-5.10/soc-qcom-apr-add-check-for-idr_alloc-and-of_property.patch similarity index 100% rename from next-5.10/soc-qcom-apr-add-check-for-idr_alloc-and-of_property.patch rename to queue-5.10/soc-qcom-apr-add-check-for-idr_alloc-and-of_property.patch diff --git a/next-5.10/soc-qcom-apr-make-code-more-reuseable.patch b/queue-5.10/soc-qcom-apr-make-code-more-reuseable.patch similarity index 100% rename from next-5.10/soc-qcom-apr-make-code-more-reuseable.patch rename to queue-5.10/soc-qcom-apr-make-code-more-reuseable.patch diff --git a/next-5.10/soc-qcom-llcc-make-irq-truly-optional.patch b/queue-5.10/soc-qcom-llcc-make-irq-truly-optional.patch similarity index 100% rename from next-5.10/soc-qcom-llcc-make-irq-truly-optional.patch rename to queue-5.10/soc-qcom-llcc-make-irq-truly-optional.patch diff --git a/next-5.10/soc-ti-knav_qmss_queue-fix-pm-disable-depth-imbalanc.patch b/queue-5.10/soc-ti-knav_qmss_queue-fix-pm-disable-depth-imbalanc.patch similarity index 100% rename from next-5.10/soc-ti-knav_qmss_queue-fix-pm-disable-depth-imbalanc.patch rename to queue-5.10/soc-ti-knav_qmss_queue-fix-pm-disable-depth-imbalanc.patch diff --git a/next-5.10/soc-ti-knav_qmss_queue-use-pm_runtime_resume_and_get.patch b/queue-5.10/soc-ti-knav_qmss_queue-use-pm_runtime_resume_and_get.patch similarity index 100% rename from next-5.10/soc-ti-knav_qmss_queue-use-pm_runtime_resume_and_get.patch rename to queue-5.10/soc-ti-knav_qmss_queue-use-pm_runtime_resume_and_get.patch diff --git a/next-5.10/soc-ti-smartreflex-fix-pm-disable-depth-imbalance-in.patch b/queue-5.10/soc-ti-smartreflex-fix-pm-disable-depth-imbalance-in.patch similarity index 100% rename from next-5.10/soc-ti-smartreflex-fix-pm-disable-depth-imbalance-in.patch rename to queue-5.10/soc-ti-smartreflex-fix-pm-disable-depth-imbalance-in.patch diff --git a/next-5.10/spi-spi-gpio-don-t-set-mosi-as-an-input-if-not-3wire.patch b/queue-5.10/spi-spi-gpio-don-t-set-mosi-as-an-input-if-not-3wire.patch similarity index 100% rename from next-5.10/spi-spi-gpio-don-t-set-mosi-as-an-input-if-not-3wire.patch rename to queue-5.10/spi-spi-gpio-don-t-set-mosi-as-an-input-if-not-3wire.patch diff --git a/next-5.10/spi-spidev-mask-spi_cs_high-in-spi_ioc_rd_mode.patch b/queue-5.10/spi-spidev-mask-spi_cs_high-in-spi_ioc_rd_mode.patch similarity index 100% rename from next-5.10/spi-spidev-mask-spi_cs_high-in-spi_ioc_rd_mode.patch rename to queue-5.10/spi-spidev-mask-spi_cs_high-in-spi_ioc_rd_mode.patch diff --git a/next-5.10/spi-update-reference-to-struct-spi_controller.patch b/queue-5.10/spi-update-reference-to-struct-spi_controller.patch similarity index 100% rename from next-5.10/spi-update-reference-to-struct-spi_controller.patch rename to queue-5.10/spi-update-reference-to-struct-spi_controller.patch diff --git a/next-5.10/staging-rtl8192e-fix-potential-use-after-free-in-rtl.patch b/queue-5.10/staging-rtl8192e-fix-potential-use-after-free-in-rtl.patch similarity index 100% rename from next-5.10/staging-rtl8192e-fix-potential-use-after-free-in-rtl.patch rename to queue-5.10/staging-rtl8192e-fix-potential-use-after-free-in-rtl.patch diff --git a/next-5.10/staging-rtl8192u-fix-use-after-free-in-ieee80211_rx.patch b/queue-5.10/staging-rtl8192u-fix-use-after-free-in-ieee80211_rx.patch similarity index 100% rename from next-5.10/staging-rtl8192u-fix-use-after-free-in-ieee80211_rx.patch rename to queue-5.10/staging-rtl8192u-fix-use-after-free-in-ieee80211_rx.patch diff --git a/next-5.10/staging-vme_user-fix-possible-uaf-in-tsi148_dma_list.patch b/queue-5.10/staging-vme_user-fix-possible-uaf-in-tsi148_dma_list.patch similarity index 100% rename from next-5.10/staging-vme_user-fix-possible-uaf-in-tsi148_dma_list.patch rename to queue-5.10/staging-vme_user-fix-possible-uaf-in-tsi148_dma_list.patch diff --git a/next-5.10/stmmac-fix-potential-division-by-0.patch b/queue-5.10/stmmac-fix-potential-division-by-0.patch similarity index 100% rename from next-5.10/stmmac-fix-potential-division-by-0.patch rename to queue-5.10/stmmac-fix-potential-division-by-0.patch diff --git a/next-5.10/sunrpc-don-t-leak-netobj-memory-when-gss_read_proxy_verf-fails.patch b/queue-5.10/sunrpc-don-t-leak-netobj-memory-when-gss_read_proxy_verf-fails.patch similarity index 100% rename from next-5.10/sunrpc-don-t-leak-netobj-memory-when-gss_read_proxy_verf-fails.patch rename to queue-5.10/sunrpc-don-t-leak-netobj-memory-when-gss_read_proxy_verf-fails.patch diff --git a/next-5.10/sunrpc-fix-missing-release-socket-in-rpc_sockname.patch b/queue-5.10/sunrpc-fix-missing-release-socket-in-rpc_sockname.patch similarity index 100% rename from next-5.10/sunrpc-fix-missing-release-socket-in-rpc_sockname.patch rename to queue-5.10/sunrpc-fix-missing-release-socket-in-rpc_sockname.patch diff --git a/next-5.10/test_firmware-fix-memory-leak-in-test_firmware_init.patch b/queue-5.10/test_firmware-fix-memory-leak-in-test_firmware_init.patch similarity index 100% rename from next-5.10/test_firmware-fix-memory-leak-in-test_firmware_init.patch rename to queue-5.10/test_firmware-fix-memory-leak-in-test_firmware_init.patch diff --git a/next-5.10/thermal-drivers-imx8mm_thermal-validate-temperature-.patch b/queue-5.10/thermal-drivers-imx8mm_thermal-validate-temperature-.patch similarity index 100% rename from next-5.10/thermal-drivers-imx8mm_thermal-validate-temperature-.patch rename to queue-5.10/thermal-drivers-imx8mm_thermal-validate-temperature-.patch diff --git a/next-5.10/timerqueue-use-rb_entry_safe-in-timerqueue_getnext.patch b/queue-5.10/timerqueue-use-rb_entry_safe-in-timerqueue_getnext.patch similarity index 100% rename from next-5.10/timerqueue-use-rb_entry_safe-in-timerqueue_getnext.patch rename to queue-5.10/timerqueue-use-rb_entry_safe-in-timerqueue_getnext.patch diff --git a/next-5.10/tpm-acpi-call-acpi_put_table-to-fix-memory-leak.patch b/queue-5.10/tpm-acpi-call-acpi_put_table-to-fix-memory-leak.patch similarity index 100% rename from next-5.10/tpm-acpi-call-acpi_put_table-to-fix-memory-leak.patch rename to queue-5.10/tpm-acpi-call-acpi_put_table-to-fix-memory-leak.patch diff --git a/next-5.10/tpm-tpm_crb-add-the-missed-acpi_put_table-to-fix-memory-leak.patch b/queue-5.10/tpm-tpm_crb-add-the-missed-acpi_put_table-to-fix-memory-leak.patch similarity index 100% rename from next-5.10/tpm-tpm_crb-add-the-missed-acpi_put_table-to-fix-memory-leak.patch rename to queue-5.10/tpm-tpm_crb-add-the-missed-acpi_put_table-to-fix-memory-leak.patch diff --git a/next-5.10/tpm-tpm_crb-fix-error-message-in-__crb_relinquish_lo.patch b/queue-5.10/tpm-tpm_crb-fix-error-message-in-__crb_relinquish_lo.patch similarity index 100% rename from next-5.10/tpm-tpm_crb-fix-error-message-in-__crb_relinquish_lo.patch rename to queue-5.10/tpm-tpm_crb-fix-error-message-in-__crb_relinquish_lo.patch diff --git a/next-5.10/tpm-tpm_ftpm_tee-fix-error-handling-in-ftpm_mod_init.patch b/queue-5.10/tpm-tpm_ftpm_tee-fix-error-handling-in-ftpm_mod_init.patch similarity index 100% rename from next-5.10/tpm-tpm_ftpm_tee-fix-error-handling-in-ftpm_mod_init.patch rename to queue-5.10/tpm-tpm_ftpm_tee-fix-error-handling-in-ftpm_mod_init.patch diff --git a/next-5.10/tpm-tpm_tis-add-the-missed-acpi_put_table-to-fix-memory-leak.patch b/queue-5.10/tpm-tpm_tis-add-the-missed-acpi_put_table-to-fix-memory-leak.patch similarity index 100% rename from next-5.10/tpm-tpm_tis-add-the-missed-acpi_put_table-to-fix-memory-leak.patch rename to queue-5.10/tpm-tpm_tis-add-the-missed-acpi_put_table-to-fix-memory-leak.patch diff --git a/next-5.10/tracing-hist-fix-issue-of-losting-command-info-in-er.patch b/queue-5.10/tracing-hist-fix-issue-of-losting-command-info-in-er.patch similarity index 100% rename from next-5.10/tracing-hist-fix-issue-of-losting-command-info-in-er.patch rename to queue-5.10/tracing-hist-fix-issue-of-losting-command-info-in-er.patch diff --git a/next-5.10/tty-serial-altera_uart_-r-t-x_chars-need-only-uart_p.patch b/queue-5.10/tty-serial-altera_uart_-r-t-x_chars-need-only-uart_p.patch similarity index 100% rename from next-5.10/tty-serial-altera_uart_-r-t-x_chars-need-only-uart_p.patch rename to queue-5.10/tty-serial-altera_uart_-r-t-x_chars-need-only-uart_p.patch diff --git a/next-5.10/tty-serial-clean-up-stop-tx-part-in-altera_uart_tx_c.patch b/queue-5.10/tty-serial-clean-up-stop-tx-part-in-altera_uart_tx_c.patch similarity index 100% rename from next-5.10/tty-serial-clean-up-stop-tx-part-in-altera_uart_tx_c.patch rename to queue-5.10/tty-serial-clean-up-stop-tx-part-in-altera_uart_tx_c.patch diff --git a/next-5.10/udf-avoid-double-brelse-in-udf_rename.patch b/queue-5.10/udf-avoid-double-brelse-in-udf_rename.patch similarity index 100% rename from next-5.10/udf-avoid-double-brelse-in-udf_rename.patch rename to queue-5.10/udf-avoid-double-brelse-in-udf_rename.patch diff --git a/next-5.10/uio-uio_dmem_genirq-fix-deadlock-between-irq-config-.patch b/queue-5.10/uio-uio_dmem_genirq-fix-deadlock-between-irq-config-.patch similarity index 100% rename from next-5.10/uio-uio_dmem_genirq-fix-deadlock-between-irq-config-.patch rename to queue-5.10/uio-uio_dmem_genirq-fix-deadlock-between-irq-config-.patch diff --git a/next-5.10/uio-uio_dmem_genirq-fix-missing-unlock-in-irq-config.patch b/queue-5.10/uio-uio_dmem_genirq-fix-missing-unlock-in-irq-config.patch similarity index 100% rename from next-5.10/uio-uio_dmem_genirq-fix-missing-unlock-in-irq-config.patch rename to queue-5.10/uio-uio_dmem_genirq-fix-missing-unlock-in-irq-config.patch diff --git a/next-5.10/uprobes-x86-allow-to-probe-a-nop-instruction-with-0x.patch b/queue-5.10/uprobes-x86-allow-to-probe-a-nop-instruction-with-0x.patch similarity index 100% rename from next-5.10/uprobes-x86-allow-to-probe-a-nop-instruction-with-0x.patch rename to queue-5.10/uprobes-x86-allow-to-probe-a-nop-instruction-with-0x.patch diff --git a/next-5.10/usb-dwc3-core-defer-probe-on-ulpi_read_id-timeout.patch b/queue-5.10/usb-dwc3-core-defer-probe-on-ulpi_read_id-timeout.patch similarity index 100% rename from next-5.10/usb-dwc3-core-defer-probe-on-ulpi_read_id-timeout.patch rename to queue-5.10/usb-dwc3-core-defer-probe-on-ulpi_read_id-timeout.patch diff --git a/next-5.10/usb-dwc3-fix-race-between-dwc3_set_mode-and-__dwc3_set_mode.patch b/queue-5.10/usb-dwc3-fix-race-between-dwc3_set_mode-and-__dwc3_set_mode.patch similarity index 100% rename from next-5.10/usb-dwc3-fix-race-between-dwc3_set_mode-and-__dwc3_set_mode.patch rename to queue-5.10/usb-dwc3-fix-race-between-dwc3_set_mode-and-__dwc3_set_mode.patch diff --git a/next-5.10/usb-dwc3-qcom-fix-memory-leak-in-dwc3_qcom_interconn.patch b/queue-5.10/usb-dwc3-qcom-fix-memory-leak-in-dwc3_qcom_interconn.patch similarity index 100% rename from next-5.10/usb-dwc3-qcom-fix-memory-leak-in-dwc3_qcom_interconn.patch rename to queue-5.10/usb-dwc3-qcom-fix-memory-leak-in-dwc3_qcom_interconn.patch diff --git a/next-5.10/usb-fotg210-udc-fix-ages-old-endianness-issues.patch b/queue-5.10/usb-fotg210-udc-fix-ages-old-endianness-issues.patch similarity index 100% rename from next-5.10/usb-fotg210-udc-fix-ages-old-endianness-issues.patch rename to queue-5.10/usb-fotg210-udc-fix-ages-old-endianness-issues.patch diff --git a/next-5.10/usb-gadget-f_hid-fix-f_hidg-lifetime-vs-cdev.patch b/queue-5.10/usb-gadget-f_hid-fix-f_hidg-lifetime-vs-cdev.patch similarity index 100% rename from next-5.10/usb-gadget-f_hid-fix-f_hidg-lifetime-vs-cdev.patch rename to queue-5.10/usb-gadget-f_hid-fix-f_hidg-lifetime-vs-cdev.patch diff --git a/next-5.10/usb-gadget-f_hid-fix-refcount-leak-on-error-path.patch b/queue-5.10/usb-gadget-f_hid-fix-refcount-leak-on-error-path.patch similarity index 100% rename from next-5.10/usb-gadget-f_hid-fix-refcount-leak-on-error-path.patch rename to queue-5.10/usb-gadget-f_hid-fix-refcount-leak-on-error-path.patch diff --git a/next-5.10/usb-gadget-f_hid-optional-setup-set_report-mode.patch b/queue-5.10/usb-gadget-f_hid-optional-setup-set_report-mode.patch similarity index 100% rename from next-5.10/usb-gadget-f_hid-optional-setup-set_report-mode.patch rename to queue-5.10/usb-gadget-f_hid-optional-setup-set_report-mode.patch diff --git a/next-5.10/usb-musb-remove-extra-check-in-musb_gadget_vbus_draw.patch b/queue-5.10/usb-musb-remove-extra-check-in-musb_gadget_vbus_draw.patch similarity index 100% rename from next-5.10/usb-musb-remove-extra-check-in-musb_gadget_vbus_draw.patch rename to queue-5.10/usb-musb-remove-extra-check-in-musb_gadget_vbus_draw.patch diff --git a/next-5.10/usb-roles-fix-of-node-refcount-leak-in-usb_role_swit.patch b/queue-5.10/usb-roles-fix-of-node-refcount-leak-in-usb_role_swit.patch similarity index 100% rename from next-5.10/usb-roles-fix-of-node-refcount-leak-in-usb_role_swit.patch rename to queue-5.10/usb-roles-fix-of-node-refcount-leak-in-usb_role_swit.patch diff --git a/next-5.10/usb-storage-add-check-for-kcalloc.patch b/queue-5.10/usb-storage-add-check-for-kcalloc.patch similarity index 100% rename from next-5.10/usb-storage-add-check-for-kcalloc.patch rename to queue-5.10/usb-storage-add-check-for-kcalloc.patch diff --git a/next-5.10/usb-typec-check-for-ops-exit-instead-of-ops-enter-in.patch b/queue-5.10/usb-typec-check-for-ops-exit-instead-of-ops-enter-in.patch similarity index 100% rename from next-5.10/usb-typec-check-for-ops-exit-instead-of-ops-enter-in.patch rename to queue-5.10/usb-typec-check-for-ops-exit-instead-of-ops-enter-in.patch diff --git a/next-5.10/usb-typec-tcpci-fix-of-node-refcount-leak-in-tcpci_r.patch b/queue-5.10/usb-typec-tcpci-fix-of-node-refcount-leak-in-tcpci_r.patch similarity index 100% rename from next-5.10/usb-typec-tcpci-fix-of-node-refcount-leak-in-tcpci_r.patch rename to queue-5.10/usb-typec-tcpci-fix-of-node-refcount-leak-in-tcpci_r.patch diff --git a/next-5.10/usb-typec-tipd-fix-spurious-fwnode_handle_put-in-err.patch b/queue-5.10/usb-typec-tipd-fix-spurious-fwnode_handle_put-in-err.patch similarity index 100% rename from next-5.10/usb-typec-tipd-fix-spurious-fwnode_handle_put-in-err.patch rename to queue-5.10/usb-typec-tipd-fix-spurious-fwnode_handle_put-in-err.patch diff --git a/next-5.10/venus-pm_helpers-fix-error-check-in-vcodec_domains_g.patch b/queue-5.10/venus-pm_helpers-fix-error-check-in-vcodec_domains_g.patch similarity index 100% rename from next-5.10/venus-pm_helpers-fix-error-check-in-vcodec_domains_g.patch rename to queue-5.10/venus-pm_helpers-fix-error-check-in-vcodec_domains_g.patch diff --git a/next-5.10/vfio-platform-do-not-pass-return-buffer-to-acpi-_rst.patch b/queue-5.10/vfio-platform-do-not-pass-return-buffer-to-acpi-_rst.patch similarity index 100% rename from next-5.10/vfio-platform-do-not-pass-return-buffer-to-acpi-_rst.patch rename to queue-5.10/vfio-platform-do-not-pass-return-buffer-to-acpi-_rst.patch diff --git a/next-5.10/video-hyperv_fb-avoid-taking-busy-spinlock-on-panic-.patch b/queue-5.10/video-hyperv_fb-avoid-taking-busy-spinlock-on-panic-.patch similarity index 100% rename from next-5.10/video-hyperv_fb-avoid-taking-busy-spinlock-on-panic-.patch rename to queue-5.10/video-hyperv_fb-avoid-taking-busy-spinlock-on-panic-.patch diff --git a/next-5.10/vme-fix-error-not-catched-in-fake_init.patch b/queue-5.10/vme-fix-error-not-catched-in-fake_init.patch similarity index 100% rename from next-5.10/vme-fix-error-not-catched-in-fake_init.patch rename to queue-5.10/vme-fix-error-not-catched-in-fake_init.patch diff --git a/next-5.10/wifi-ar5523-fix-use-after-free-on-ar5523_cmd-timed-o.patch b/queue-5.10/wifi-ar5523-fix-use-after-free-on-ar5523_cmd-timed-o.patch similarity index 100% rename from next-5.10/wifi-ar5523-fix-use-after-free-on-ar5523_cmd-timed-o.patch rename to queue-5.10/wifi-ar5523-fix-use-after-free-on-ar5523_cmd-timed-o.patch diff --git a/next-5.10/wifi-ath10k-fix-return-value-in-ath10k_pci_init.patch b/queue-5.10/wifi-ath10k-fix-return-value-in-ath10k_pci_init.patch similarity index 100% rename from next-5.10/wifi-ath10k-fix-return-value-in-ath10k_pci_init.patch rename to queue-5.10/wifi-ath10k-fix-return-value-in-ath10k_pci_init.patch diff --git a/next-5.10/wifi-ath9k-hif_usb-fix-memory-leak-of-urbs-in-ath9k_.patch b/queue-5.10/wifi-ath9k-hif_usb-fix-memory-leak-of-urbs-in-ath9k_.patch similarity index 100% rename from next-5.10/wifi-ath9k-hif_usb-fix-memory-leak-of-urbs-in-ath9k_.patch rename to queue-5.10/wifi-ath9k-hif_usb-fix-memory-leak-of-urbs-in-ath9k_.patch diff --git a/next-5.10/wifi-ath9k-hif_usb-fix-use-after-free-in-ath9k_hif_u.patch b/queue-5.10/wifi-ath9k-hif_usb-fix-use-after-free-in-ath9k_hif_u.patch similarity index 100% rename from next-5.10/wifi-ath9k-hif_usb-fix-use-after-free-in-ath9k_hif_u.patch rename to queue-5.10/wifi-ath9k-hif_usb-fix-use-after-free-in-ath9k_hif_u.patch diff --git a/next-5.10/wifi-ath9k-verify-the-expected-usb_endpoints-are-pre.patch b/queue-5.10/wifi-ath9k-verify-the-expected-usb_endpoints-are-pre.patch similarity index 100% rename from next-5.10/wifi-ath9k-verify-the-expected-usb_endpoints-are-pre.patch rename to queue-5.10/wifi-ath9k-verify-the-expected-usb_endpoints-are-pre.patch diff --git a/next-5.10/wifi-brcmfmac-fix-error-return-code-in-brcmf_sdio_do.patch b/queue-5.10/wifi-brcmfmac-fix-error-return-code-in-brcmf_sdio_do.patch similarity index 100% rename from next-5.10/wifi-brcmfmac-fix-error-return-code-in-brcmf_sdio_do.patch rename to queue-5.10/wifi-brcmfmac-fix-error-return-code-in-brcmf_sdio_do.patch diff --git a/next-5.10/wifi-brcmfmac-fix-potential-shift-out-of-bounds-in-b.patch b/queue-5.10/wifi-brcmfmac-fix-potential-shift-out-of-bounds-in-b.patch similarity index 100% rename from next-5.10/wifi-brcmfmac-fix-potential-shift-out-of-bounds-in-b.patch rename to queue-5.10/wifi-brcmfmac-fix-potential-shift-out-of-bounds-in-b.patch diff --git a/next-5.10/wifi-cfg80211-fix-not-unregister-reg_pdev-when-load_.patch b/queue-5.10/wifi-cfg80211-fix-not-unregister-reg_pdev-when-load_.patch similarity index 100% rename from next-5.10/wifi-cfg80211-fix-not-unregister-reg_pdev-when-load_.patch rename to queue-5.10/wifi-cfg80211-fix-not-unregister-reg_pdev-when-load_.patch diff --git a/next-5.10/wifi-iwlwifi-mvm-fix-double-free-on-tx-path.patch b/queue-5.10/wifi-iwlwifi-mvm-fix-double-free-on-tx-path.patch similarity index 100% rename from next-5.10/wifi-iwlwifi-mvm-fix-double-free-on-tx-path.patch rename to queue-5.10/wifi-iwlwifi-mvm-fix-double-free-on-tx-path.patch diff --git a/next-5.10/wifi-mac80211-fix-memory-leak-in-ieee80211_if_add.patch b/queue-5.10/wifi-mac80211-fix-memory-leak-in-ieee80211_if_add.patch similarity index 100% rename from next-5.10/wifi-mac80211-fix-memory-leak-in-ieee80211_if_add.patch rename to queue-5.10/wifi-mac80211-fix-memory-leak-in-ieee80211_if_add.patch diff --git a/next-5.10/wifi-mt76-fix-coverity-overrun-call-in-mt76_get_txpo.patch b/queue-5.10/wifi-mt76-fix-coverity-overrun-call-in-mt76_get_txpo.patch similarity index 100% rename from next-5.10/wifi-mt76-fix-coverity-overrun-call-in-mt76_get_txpo.patch rename to queue-5.10/wifi-mt76-fix-coverity-overrun-call-in-mt76_get_txpo.patch diff --git a/next-5.10/wifi-rsi-fix-handling-of-802.3-eapol-frames-sent-via.patch b/queue-5.10/wifi-rsi-fix-handling-of-802.3-eapol-frames-sent-via.patch similarity index 100% rename from next-5.10/wifi-rsi-fix-handling-of-802.3-eapol-frames-sent-via.patch rename to queue-5.10/wifi-rsi-fix-handling-of-802.3-eapol-frames-sent-via.patch diff --git a/next-5.10/wifi-rtl8xxxu-add-__packed-to-struct-rtl8723bu_c2h.patch b/queue-5.10/wifi-rtl8xxxu-add-__packed-to-struct-rtl8723bu_c2h.patch similarity index 100% rename from next-5.10/wifi-rtl8xxxu-add-__packed-to-struct-rtl8723bu_c2h.patch rename to queue-5.10/wifi-rtl8xxxu-add-__packed-to-struct-rtl8723bu_c2h.patch diff --git a/next-5.10/wifi-rtl8xxxu-fix-reading-the-vendor-of-combo-chips.patch b/queue-5.10/wifi-rtl8xxxu-fix-reading-the-vendor-of-combo-chips.patch similarity index 100% rename from next-5.10/wifi-rtl8xxxu-fix-reading-the-vendor-of-combo-chips.patch rename to queue-5.10/wifi-rtl8xxxu-fix-reading-the-vendor-of-combo-chips.patch diff --git a/next-5.10/wifi-rtl8xxxu-fix-the-channel-width-reporting.patch b/queue-5.10/wifi-rtl8xxxu-fix-the-channel-width-reporting.patch similarity index 100% rename from next-5.10/wifi-rtl8xxxu-fix-the-channel-width-reporting.patch rename to queue-5.10/wifi-rtl8xxxu-fix-the-channel-width-reporting.patch diff --git a/next-5.10/x86-hyperv-remove-unregister-syscore-call-from-hyper.patch b/queue-5.10/x86-hyperv-remove-unregister-syscore-call-from-hyper.patch similarity index 100% rename from next-5.10/x86-hyperv-remove-unregister-syscore-call-from-hyper.patch rename to queue-5.10/x86-hyperv-remove-unregister-syscore-call-from-hyper.patch diff --git a/next-5.10/x86-xen-fix-memory-leak-in-xen_init_lock_cpu.patch b/queue-5.10/x86-xen-fix-memory-leak-in-xen_init_lock_cpu.patch similarity index 100% rename from next-5.10/x86-xen-fix-memory-leak-in-xen_init_lock_cpu.patch rename to queue-5.10/x86-xen-fix-memory-leak-in-xen_init_lock_cpu.patch diff --git a/next-5.10/x86-xen-fix-memory-leak-in-xen_smp_intr_init-_pv.patch b/queue-5.10/x86-xen-fix-memory-leak-in-xen_smp_intr_init-_pv.patch similarity index 100% rename from next-5.10/x86-xen-fix-memory-leak-in-xen_smp_intr_init-_pv.patch rename to queue-5.10/x86-xen-fix-memory-leak-in-xen_smp_intr_init-_pv.patch diff --git a/next-5.10/xen-privcmd-fix-a-possible-warning-in-privcmd_ioctl_.patch b/queue-5.10/xen-privcmd-fix-a-possible-warning-in-privcmd_ioctl_.patch similarity index 100% rename from next-5.10/xen-privcmd-fix-a-possible-warning-in-privcmd_ioctl_.patch rename to queue-5.10/xen-privcmd-fix-a-possible-warning-in-privcmd_ioctl_.patch diff --git a/next-5.10/xprtrdma-fix-regbuf-data-not-freed-in-rpcrdma_req_cr.patch b/queue-5.10/xprtrdma-fix-regbuf-data-not-freed-in-rpcrdma_req_cr.patch similarity index 100% rename from next-5.10/xprtrdma-fix-regbuf-data-not-freed-in-rpcrdma_req_cr.patch rename to queue-5.10/xprtrdma-fix-regbuf-data-not-freed-in-rpcrdma_req_cr.patch