From: Greg Kroah-Hartman Date: Tue, 3 Jan 2023 08:12:02 +0000 (+0100) Subject: move 5.10 stuff out of the way X-Git-Tag: v6.0.17~1 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=cf6e3750e4cd26fb125bd25ce30ae5dea46e3f18;p=thirdparty%2Fkernel%2Fstable-queue.git move 5.10 stuff out of the way --- diff --git a/queue-5.10/acct-fix-potential-integer-overflow-in-encode_comp_t.patch b/next-5.10/acct-fix-potential-integer-overflow-in-encode_comp_t.patch similarity index 100% rename from queue-5.10/acct-fix-potential-integer-overflow-in-encode_comp_t.patch rename to next-5.10/acct-fix-potential-integer-overflow-in-encode_comp_t.patch diff --git a/queue-5.10/acpica-fix-error-code-path-in-acpi_ds_call_control_m.patch b/next-5.10/acpica-fix-error-code-path-in-acpi_ds_call_control_m.patch similarity index 100% rename from queue-5.10/acpica-fix-error-code-path-in-acpi_ds_call_control_m.patch rename to next-5.10/acpica-fix-error-code-path-in-acpi_ds_call_control_m.patch diff --git a/queue-5.10/acpica-fix-use-after-free-in-acpi_ut_copy_ipackage_t.patch b/next-5.10/acpica-fix-use-after-free-in-acpi_ut_copy_ipackage_t.patch similarity index 100% rename from queue-5.10/acpica-fix-use-after-free-in-acpi_ut_copy_ipackage_t.patch rename to next-5.10/acpica-fix-use-after-free-in-acpi_ut_copy_ipackage_t.patch diff --git a/queue-5.10/afs-fix-lost-servers_outstanding-count.patch b/next-5.10/afs-fix-lost-servers_outstanding-count.patch similarity index 100% rename from queue-5.10/afs-fix-lost-servers_outstanding-count.patch rename to next-5.10/afs-fix-lost-servers_outstanding-count.patch diff --git a/next-5.10/alpha-add-support-for-tif_notify_signal.patch b/next-5.10/alpha-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..3de3f334327 --- /dev/null +++ b/next-5.10/alpha-add-support-for-tif_notify_signal.patch @@ -0,0 +1,60 @@ +From f85a5fbc85e3edbfa86d83f163307c9ef597eece Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Thu, 8 Oct 2020 09:11:42 -0600 +Subject: alpha: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 5a9a8897c253a075805401d38d987ec1ac1824b6 ] + +Wire up TIF_NOTIFY_SIGNAL handling for alpha. + +Cc: linux-alpha@vger.kernel.org +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/alpha/include/asm/thread_info.h | 2 ++ + arch/alpha/kernel/entry.S | 2 +- + arch/alpha/kernel/signal.c | 2 +- + 3 files changed, 4 insertions(+), 2 deletions(-) + +--- a/arch/alpha/include/asm/thread_info.h ++++ b/arch/alpha/include/asm/thread_info.h +@@ -62,6 +62,7 @@ register struct thread_info *__current_t + #define TIF_SIGPENDING 2 /* signal pending */ + #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ + #define TIF_SYSCALL_AUDIT 4 /* syscall audit active */ ++#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ + #define TIF_DIE_IF_KERNEL 9 /* dik recursion lock */ + #define TIF_MEMDIE 13 /* is terminating due to OOM killer */ + #define TIF_POLLING_NRFLAG 14 /* idle is polling for TIF_NEED_RESCHED */ +@@ -71,6 +72,7 @@ register struct thread_info *__current_t + #define _TIF_NEED_RESCHED (1< +Date: Sat, 18 Sep 2021 18:08:56 -0400 +Subject: alpha: fix TIF_NOTIFY_SIGNAL handling + +From: Al Viro + +[ Upstream commit e2c7554cc6d85f95e3c6635f270ec839ab9fe05e ] + +it needs to be added to _TIF_WORK_MASK, or we might not reach +do_work_pending() in the first place... + +Fixes: 5a9a8897c253a "alpha: add support for TIF_NOTIFY_SIGNAL" +Signed-off-by: Al Viro +Signed-off-by: Greg Kroah-Hartman +--- + arch/alpha/include/asm/thread_info.h | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +--- a/arch/alpha/include/asm/thread_info.h ++++ b/arch/alpha/include/asm/thread_info.h +@@ -77,7 +77,7 @@ register struct thread_info *__current_t + + /* Work to do on interrupt/exception return. */ + #define _TIF_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \ +- _TIF_NOTIFY_RESUME) ++ _TIF_NOTIFY_RESUME | _TIF_NOTIFY_SIGNAL) + + /* Work to do on any return to userspace. */ + #define _TIF_ALLWORK_MASK (_TIF_WORK_MASK \ diff --git a/queue-5.10/alsa-asihpi-fix-missing-pci_disable_device.patch b/next-5.10/alsa-asihpi-fix-missing-pci_disable_device.patch similarity index 100% rename from queue-5.10/alsa-asihpi-fix-missing-pci_disable_device.patch rename to next-5.10/alsa-asihpi-fix-missing-pci_disable_device.patch diff --git a/queue-5.10/alsa-asoc-hda-move-rename-snd_hdac_ext_stop_streams-.patch b/next-5.10/alsa-asoc-hda-move-rename-snd_hdac_ext_stop_streams-.patch similarity index 100% rename from queue-5.10/alsa-asoc-hda-move-rename-snd_hdac_ext_stop_streams-.patch rename to next-5.10/alsa-asoc-hda-move-rename-snd_hdac_ext_stop_streams-.patch diff --git a/queue-5.10/alsa-hda-add-snd_hdac_stop_streams-helper.patch b/next-5.10/alsa-hda-add-snd_hdac_stop_streams-helper.patch similarity index 100% rename from queue-5.10/alsa-hda-add-snd_hdac_stop_streams-helper.patch rename to next-5.10/alsa-hda-add-snd_hdac_stop_streams-helper.patch diff --git a/queue-5.10/alsa-hda-hdmi-add-hp-device-0x8711-to-force-connect-list.patch b/next-5.10/alsa-hda-hdmi-add-hp-device-0x8711-to-force-connect-list.patch similarity index 100% rename from queue-5.10/alsa-hda-hdmi-add-hp-device-0x8711-to-force-connect-list.patch rename to next-5.10/alsa-hda-hdmi-add-hp-device-0x8711-to-force-connect-list.patch diff --git a/queue-5.10/alsa-hda-realtek-add-quirk-for-lenovo-tianyi510pro-14iob.patch b/next-5.10/alsa-hda-realtek-add-quirk-for-lenovo-tianyi510pro-14iob.patch similarity index 100% rename from queue-5.10/alsa-hda-realtek-add-quirk-for-lenovo-tianyi510pro-14iob.patch rename to next-5.10/alsa-hda-realtek-add-quirk-for-lenovo-tianyi510pro-14iob.patch diff --git a/queue-5.10/alsa-line6-correct-midi-status-byte-when-receiving-data-from-podxt.patch b/next-5.10/alsa-line6-correct-midi-status-byte-when-receiving-data-from-podxt.patch similarity index 100% rename from queue-5.10/alsa-line6-correct-midi-status-byte-when-receiving-data-from-podxt.patch rename to next-5.10/alsa-line6-correct-midi-status-byte-when-receiving-data-from-podxt.patch diff --git a/queue-5.10/alsa-line6-fix-stack-overflow-in-line6_midi_transmit.patch b/next-5.10/alsa-line6-fix-stack-overflow-in-line6_midi_transmit.patch similarity index 100% rename from queue-5.10/alsa-line6-fix-stack-overflow-in-line6_midi_transmit.patch rename to next-5.10/alsa-line6-fix-stack-overflow-in-line6_midi_transmit.patch diff --git a/queue-5.10/alsa-mts64-fix-possible-null-ptr-defer-in-snd_mts64_.patch b/next-5.10/alsa-mts64-fix-possible-null-ptr-defer-in-snd_mts64_.patch similarity index 100% rename from queue-5.10/alsa-mts64-fix-possible-null-ptr-defer-in-snd_mts64_.patch rename to next-5.10/alsa-mts64-fix-possible-null-ptr-defer-in-snd_mts64_.patch diff --git a/queue-5.10/alsa-pcm-fix-undefined-behavior-in-bit-shift-for-snd.patch b/next-5.10/alsa-pcm-fix-undefined-behavior-in-bit-shift-for-snd.patch similarity index 100% rename from queue-5.10/alsa-pcm-fix-undefined-behavior-in-bit-shift-for-snd.patch rename to next-5.10/alsa-pcm-fix-undefined-behavior-in-bit-shift-for-snd.patch diff --git a/queue-5.10/alsa-pcm-set-missing-stop_operating-flag-at-undoing-.patch b/next-5.10/alsa-pcm-set-missing-stop_operating-flag-at-undoing-.patch similarity index 100% rename from queue-5.10/alsa-pcm-set-missing-stop_operating-flag-at-undoing-.patch rename to next-5.10/alsa-pcm-set-missing-stop_operating-flag-at-undoing-.patch diff --git a/queue-5.10/alsa-seq-fix-undefined-behavior-in-bit-shift-for-snd.patch b/next-5.10/alsa-seq-fix-undefined-behavior-in-bit-shift-for-snd.patch similarity index 100% rename from queue-5.10/alsa-seq-fix-undefined-behavior-in-bit-shift-for-snd.patch rename to next-5.10/alsa-seq-fix-undefined-behavior-in-bit-shift-for-snd.patch diff --git a/queue-5.10/alsa-usb-audio-add-the-quirk-for-kt0206-device.patch b/next-5.10/alsa-usb-audio-add-the-quirk-for-kt0206-device.patch similarity index 100% rename from queue-5.10/alsa-usb-audio-add-the-quirk-for-kt0206-device.patch rename to next-5.10/alsa-usb-audio-add-the-quirk-for-kt0206-device.patch diff --git a/queue-5.10/amdgpu-pm-prevent-array-underflow-in-vega20_odn_edit.patch b/next-5.10/amdgpu-pm-prevent-array-underflow-in-vega20_odn_edit.patch similarity index 100% rename from queue-5.10/amdgpu-pm-prevent-array-underflow-in-vega20_odn_edit.patch rename to next-5.10/amdgpu-pm-prevent-array-underflow-in-vega20_odn_edit.patch diff --git a/queue-5.10/apparmor-fix-a-memleak-in-multi_transaction_new.patch b/next-5.10/apparmor-fix-a-memleak-in-multi_transaction_new.patch similarity index 100% rename from queue-5.10/apparmor-fix-a-memleak-in-multi_transaction_new.patch rename to next-5.10/apparmor-fix-a-memleak-in-multi_transaction_new.patch diff --git a/queue-5.10/apparmor-fix-abi-check-to-include-v8-abi.patch b/next-5.10/apparmor-fix-abi-check-to-include-v8-abi.patch similarity index 100% rename from queue-5.10/apparmor-fix-abi-check-to-include-v8-abi.patch rename to next-5.10/apparmor-fix-abi-check-to-include-v8-abi.patch diff --git a/queue-5.10/apparmor-fix-lockdep-warning-when-removing-a-namespa.patch b/next-5.10/apparmor-fix-lockdep-warning-when-removing-a-namespa.patch similarity index 100% rename from queue-5.10/apparmor-fix-lockdep-warning-when-removing-a-namespa.patch rename to next-5.10/apparmor-fix-lockdep-warning-when-removing-a-namespa.patch diff --git a/queue-5.10/apparmor-fix-memleak-in-alloc_ns.patch b/next-5.10/apparmor-fix-memleak-in-alloc_ns.patch similarity index 100% rename from queue-5.10/apparmor-fix-memleak-in-alloc_ns.patch rename to next-5.10/apparmor-fix-memleak-in-alloc_ns.patch diff --git a/queue-5.10/apparmor-use-pointer-to-struct-aa_label-for-lbs_cred.patch b/next-5.10/apparmor-use-pointer-to-struct-aa_label-for-lbs_cred.patch similarity index 100% rename from queue-5.10/apparmor-use-pointer-to-struct-aa_label-for-lbs_cred.patch rename to next-5.10/apparmor-use-pointer-to-struct-aa_label-for-lbs_cred.patch diff --git a/next-5.10/arc-add-support-for-tif_notify_signal.patch b/next-5.10/arc-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..a1ad6fd2889 --- /dev/null +++ b/next-5.10/arc-add-support-for-tif_notify_signal.patch @@ -0,0 +1,68 @@ +From b71f35bd26a9b588d59894e542c48b4360be52fc Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Mon, 12 Oct 2020 07:15:37 -0600 +Subject: arc: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 53855e12588743ea128ee31f913d1c6e2f1d32c8 ] + +Wire up TIF_NOTIFY_SIGNAL handling for arc. + +Cc: linux-snps-arc@lists.infradead.org +Acked-by: Vineet Gupta +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/arc/include/asm/thread_info.h | 4 +++- + arch/arc/kernel/entry.S | 3 ++- + arch/arc/kernel/signal.c | 2 +- + 3 files changed, 6 insertions(+), 3 deletions(-) + +--- a/arch/arc/include/asm/thread_info.h ++++ b/arch/arc/include/asm/thread_info.h +@@ -79,6 +79,7 @@ static inline __attribute_const__ struct + #define TIF_SIGPENDING 2 /* signal pending */ + #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ + #define TIF_SYSCALL_AUDIT 4 /* syscall auditing active */ ++#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ + #define TIF_SYSCALL_TRACE 15 /* syscall trace active */ + + /* true if poll_idle() is polling TIF_NEED_RESCHED */ +@@ -89,11 +90,12 @@ static inline __attribute_const__ struct + #define _TIF_SIGPENDING (1< +Date: Wed, 6 Jan 2021 12:34:36 -0800 +Subject: ARC: unbork 5.11 bootup: fix snafu in _TIF_NOTIFY_SIGNAL handling + +From: Vineet Gupta + +[ Upstream commit bb12433bf56e76789c6b08b36c546f745a6aa6e1 ] + +Linux 5.11.rcX was failing to boot on ARC HSDK board. Turns out we have +a couple of issues, this being the first one, and I'm to blame as I +didn't pay attention during review. + +TIF_NOTIFY_SIGNAL support requires checking multiple TIF_* bits in +kernel return code path. Old code only needed to check a single bit so +BBIT0 worked. New code needs to check multiple bits so +AND instruction. So needs to use bit mask variant _TIF_SIGPENDING + +Cc: Jens Axboe +Fixes: 53855e12588743ea128 ("arc: add support for TIF_NOTIFY_SIGNAL") +Link: https://github.com/foss-for-synopsys-dwc-arc-processors/linux/issues/34 +Signed-off-by: Vineet Gupta +Signed-off-by: Greg Kroah-Hartman +--- + arch/arc/kernel/entry.S | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +--- a/arch/arc/kernel/entry.S ++++ b/arch/arc/kernel/entry.S +@@ -308,7 +308,7 @@ resume_user_mode_begin: + mov r0, sp ; pt_regs for arg to do_signal()/do_notify_resume() + + GET_CURR_THR_INFO_FLAGS r9 +- and.f 0, r9, TIF_SIGPENDING|TIF_NOTIFY_SIGNAL ++ and.f 0, r9, _TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL + bz .Lchk_notify_resume + + ; Normal Trap/IRQ entry only saves Scratch (caller-saved) regs diff --git a/next-5.10/arch-ensure-parisc-powerpc-handle-pf_io_worker-in-copy_thread.patch b/next-5.10/arch-ensure-parisc-powerpc-handle-pf_io_worker-in-copy_thread.patch new file mode 100644 index 00000000000..dabbd7698bf --- /dev/null +++ b/next-5.10/arch-ensure-parisc-powerpc-handle-pf_io_worker-in-copy_thread.patch @@ -0,0 +1,44 @@ +From b9634adab82019ff70e445c09a7c26a209f6d010 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Tue, 23 Feb 2021 11:57:20 -0700 +Subject: arch: ensure parisc/powerpc handle PF_IO_WORKER in copy_thread() + +From: Jens Axboe + +[ Upstream commit 0100e6bbdbb79404e56939313662b42737026574 ] + +In the arch addition of PF_IO_WORKER, I missed parisc and powerpc for +some reason. Fix that up, ensuring they handle PF_IO_WORKER like they do +PF_KTHREAD in copy_thread(). + +Reported-by: Bruno Goncalves +Fixes: 4727dc20e042 ("arch: setup PF_IO_WORKER threads like PF_KTHREAD") +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/parisc/kernel/process.c | 2 +- + arch/powerpc/kernel/process.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +--- a/arch/parisc/kernel/process.c ++++ b/arch/parisc/kernel/process.c +@@ -200,7 +200,7 @@ copy_thread(unsigned long clone_flags, u + extern void * const ret_from_kernel_thread; + extern void * const child_return; + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + /* kernel thread */ + memset(cregs, 0, sizeof(struct pt_regs)); + if (!usp) /* idle thread */ +--- a/arch/powerpc/kernel/process.c ++++ b/arch/powerpc/kernel/process.c +@@ -1684,7 +1684,7 @@ int copy_thread(unsigned long clone_flag + /* Copy registers */ + sp -= sizeof(struct pt_regs); + childregs = (struct pt_regs *) sp; +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + /* kernel thread */ + memset(childregs, 0, sizeof(struct pt_regs)); + childregs->gpr[1] = sp + sizeof(struct pt_regs); diff --git a/next-5.10/arch-setup-pf_io_worker-threads-like-pf_kthread.patch b/next-5.10/arch-setup-pf_io_worker-threads-like-pf_kthread.patch new file mode 100644 index 00000000000..ca53757b9ac --- /dev/null +++ b/next-5.10/arch-setup-pf_io_worker-threads-like-pf_kthread.patch @@ -0,0 +1,283 @@ +From 50b21feb5a0272a0a8cb620d27d4c5e698cc1d21 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Wed, 17 Feb 2021 08:48:00 -0700 +Subject: arch: setup PF_IO_WORKER threads like PF_KTHREAD + +From: Jens Axboe + +[ Upstream commit 4727dc20e0422211a0e0c72b1ace4ed6096df8a6 ] + +PF_IO_WORKER are kernel threads too, but they aren't PF_KTHREAD in the +sense that we don't assign ->set_child_tid with our own structure. Just +ensure that every arch sets up the PF_IO_WORKER threads like kthreads +in the arch implementation of copy_thread(). + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/alpha/kernel/process.c | 2 +- + arch/arc/kernel/process.c | 2 +- + arch/arm/kernel/process.c | 2 +- + arch/arm64/kernel/process.c | 2 +- + arch/csky/kernel/process.c | 2 +- + arch/h8300/kernel/process.c | 2 +- + arch/hexagon/kernel/process.c | 2 +- + arch/ia64/kernel/process.c | 2 +- + arch/m68k/kernel/process.c | 2 +- + arch/microblaze/kernel/process.c | 2 +- + arch/mips/kernel/process.c | 2 +- + arch/nds32/kernel/process.c | 2 +- + arch/nios2/kernel/process.c | 2 +- + arch/openrisc/kernel/process.c | 2 +- + arch/riscv/kernel/process.c | 2 +- + arch/s390/kernel/process.c | 2 +- + arch/sh/kernel/process_32.c | 2 +- + arch/sparc/kernel/process_32.c | 2 +- + arch/sparc/kernel/process_64.c | 2 +- + arch/um/kernel/process.c | 2 +- + arch/x86/kernel/process.c | 2 +- + arch/xtensa/kernel/process.c | 2 +- + 22 files changed, 22 insertions(+), 22 deletions(-) + +--- a/arch/alpha/kernel/process.c ++++ b/arch/alpha/kernel/process.c +@@ -249,7 +249,7 @@ int copy_thread(unsigned long clone_flag + childti->pcb.ksp = (unsigned long) childstack; + childti->pcb.flags = 1; /* set FEN, clear everything else */ + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + /* kernel thread */ + memset(childstack, 0, + sizeof(struct switch_stack) + sizeof(struct pt_regs)); +--- a/arch/arc/kernel/process.c ++++ b/arch/arc/kernel/process.c +@@ -191,7 +191,7 @@ int copy_thread(unsigned long clone_flag + childksp[0] = 0; /* fp */ + childksp[1] = (unsigned long)ret_from_fork; /* blink */ + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + memset(c_regs, 0, sizeof(struct pt_regs)); + + c_callee->r13 = kthread_arg; +--- a/arch/arm/kernel/process.c ++++ b/arch/arm/kernel/process.c +@@ -243,7 +243,7 @@ int copy_thread(unsigned long clone_flag + thread->cpu_domain = get_domain(); + #endif + +- if (likely(!(p->flags & PF_KTHREAD))) { ++ if (likely(!(p->flags & (PF_KTHREAD | PF_IO_WORKER)))) { + *childregs = *current_pt_regs(); + childregs->ARM_r0 = 0; + if (stack_start) +--- a/arch/arm64/kernel/process.c ++++ b/arch/arm64/kernel/process.c +@@ -398,7 +398,7 @@ int copy_thread(unsigned long clone_flag + + ptrauth_thread_init_kernel(p); + +- if (likely(!(p->flags & PF_KTHREAD))) { ++ if (likely(!(p->flags & (PF_KTHREAD | PF_IO_WORKER)))) { + *childregs = *current_pt_regs(); + childregs->regs[0] = 0; + +--- a/arch/csky/kernel/process.c ++++ b/arch/csky/kernel/process.c +@@ -49,7 +49,7 @@ int copy_thread(unsigned long clone_flag + /* setup thread.sp for switch_to !!! */ + p->thread.sp = (unsigned long)childstack; + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + memset(childregs, 0, sizeof(struct pt_regs)); + childstack->r15 = (unsigned long) ret_from_kernel_thread; + childstack->r10 = kthread_arg; +--- a/arch/h8300/kernel/process.c ++++ b/arch/h8300/kernel/process.c +@@ -112,7 +112,7 @@ int copy_thread(unsigned long clone_flag + + childregs = (struct pt_regs *) (THREAD_SIZE + task_stack_page(p)) - 1; + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + memset(childregs, 0, sizeof(struct pt_regs)); + childregs->retpc = (unsigned long) ret_from_kernel_thread; + childregs->er4 = topstk; /* arg */ +--- a/arch/hexagon/kernel/process.c ++++ b/arch/hexagon/kernel/process.c +@@ -73,7 +73,7 @@ int copy_thread(unsigned long clone_flag + sizeof(*ss)); + ss->lr = (unsigned long)ret_from_fork; + p->thread.switch_sp = ss; +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + memset(childregs, 0, sizeof(struct pt_regs)); + /* r24 <- fn, r25 <- arg */ + ss->r24 = usp; +--- a/arch/ia64/kernel/process.c ++++ b/arch/ia64/kernel/process.c +@@ -338,7 +338,7 @@ copy_thread(unsigned long clone_flags, u + + ia64_drop_fpu(p); /* don't pick up stale state from a CPU's fph */ + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + if (unlikely(!user_stack_base)) { + /* fork_idle() called us */ + return 0; +--- a/arch/m68k/kernel/process.c ++++ b/arch/m68k/kernel/process.c +@@ -157,7 +157,7 @@ int copy_thread(unsigned long clone_flag + */ + p->thread.fs = get_fs().seg; + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + /* kernel thread */ + memset(frame, 0, sizeof(struct fork_frame)); + frame->regs.sr = PS_S; +--- a/arch/microblaze/kernel/process.c ++++ b/arch/microblaze/kernel/process.c +@@ -59,7 +59,7 @@ int copy_thread(unsigned long clone_flag + struct pt_regs *childregs = task_pt_regs(p); + struct thread_info *ti = task_thread_info(p); + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + /* if we're creating a new kernel thread then just zeroing all + * the registers. That's OK for a brand new thread.*/ + memset(childregs, 0, sizeof(struct pt_regs)); +--- a/arch/mips/kernel/process.c ++++ b/arch/mips/kernel/process.c +@@ -135,7 +135,7 @@ int copy_thread(unsigned long clone_flag + /* Put the stack after the struct pt_regs. */ + childksp = (unsigned long) childregs; + p->thread.cp0_status = (read_c0_status() & ~(ST0_CU2|ST0_CU1)) | ST0_KERNEL_CUMASK; +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + /* kernel thread */ + unsigned long status = p->thread.cp0_status; + memset(childregs, 0, sizeof(struct pt_regs)); +--- a/arch/nds32/kernel/process.c ++++ b/arch/nds32/kernel/process.c +@@ -156,7 +156,7 @@ int copy_thread(unsigned long clone_flag + + memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + memset(childregs, 0, sizeof(struct pt_regs)); + /* kernel thread fn */ + p->thread.cpu_context.r6 = stack_start; +--- a/arch/nios2/kernel/process.c ++++ b/arch/nios2/kernel/process.c +@@ -109,7 +109,7 @@ int copy_thread(unsigned long clone_flag + struct switch_stack *childstack = + ((struct switch_stack *)childregs) - 1; + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + memset(childstack, 0, + sizeof(struct switch_stack) + sizeof(struct pt_regs)); + +--- a/arch/openrisc/kernel/process.c ++++ b/arch/openrisc/kernel/process.c +@@ -167,7 +167,7 @@ copy_thread(unsigned long clone_flags, u + sp -= sizeof(struct pt_regs); + kregs = (struct pt_regs *)sp; + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + memset(kregs, 0, sizeof(struct pt_regs)); + kregs->gpr[20] = usp; /* fn, kernel thread */ + kregs->gpr[22] = arg; +--- a/arch/riscv/kernel/process.c ++++ b/arch/riscv/kernel/process.c +@@ -114,7 +114,7 @@ int copy_thread(unsigned long clone_flag + memset(&p->thread.s, 0, sizeof(p->thread.s)); + + /* p->thread holds context to be restored by __switch_to() */ +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + /* Kernel thread */ + memset(childregs, 0, sizeof(struct pt_regs)); + childregs->gp = gp_in_global; +--- a/arch/s390/kernel/process.c ++++ b/arch/s390/kernel/process.c +@@ -127,7 +127,7 @@ int copy_thread(unsigned long clone_flag + frame->sf.gprs[9] = (unsigned long) frame; + + /* Store access registers to kernel stack of new process. */ +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + /* kernel thread */ + memset(&frame->childregs, 0, sizeof(struct pt_regs)); + frame->childregs.psw.mask = PSW_KERNEL_BITS | PSW_MASK_DAT | +--- a/arch/sh/kernel/process_32.c ++++ b/arch/sh/kernel/process_32.c +@@ -114,7 +114,7 @@ int copy_thread(unsigned long clone_flag + + childregs = task_pt_regs(p); + p->thread.sp = (unsigned long) childregs; +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + memset(childregs, 0, sizeof(struct pt_regs)); + p->thread.pc = (unsigned long) ret_from_kernel_thread; + childregs->regs[4] = arg; +--- a/arch/sparc/kernel/process_32.c ++++ b/arch/sparc/kernel/process_32.c +@@ -309,7 +309,7 @@ int copy_thread(unsigned long clone_flag + ti->ksp = (unsigned long) new_stack; + p->thread.kregs = childregs; + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + extern int nwindows; + unsigned long psr; + memset(new_stack, 0, STACKFRAME_SZ + TRACEREG_SZ); +--- a/arch/sparc/kernel/process_64.c ++++ b/arch/sparc/kernel/process_64.c +@@ -597,7 +597,7 @@ int copy_thread(unsigned long clone_flag + sizeof(struct sparc_stackf)); + t->fpsaved[0] = 0; + +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + memset(child_trap_frame, 0, child_stack_sz); + __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] = + (current_pt_regs()->tstate + 1) & TSTATE_CWP; +--- a/arch/um/kernel/process.c ++++ b/arch/um/kernel/process.c +@@ -157,7 +157,7 @@ int copy_thread(unsigned long clone_flag + unsigned long arg, struct task_struct * p, unsigned long tls) + { + void (*handler)(void); +- int kthread = current->flags & PF_KTHREAD; ++ int kthread = current->flags & (PF_KTHREAD | PF_IO_WORKER); + int ret = 0; + + p->thread = (struct thread_struct) INIT_THREAD; +--- a/arch/x86/kernel/process.c ++++ b/arch/x86/kernel/process.c +@@ -162,7 +162,7 @@ int copy_thread(unsigned long clone_flag + #endif + + /* Kernel thread ? */ +- if (unlikely(p->flags & PF_KTHREAD)) { ++ if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + memset(childregs, 0, sizeof(struct pt_regs)); + kthread_frame_init(frame, sp, arg); + return 0; +--- a/arch/xtensa/kernel/process.c ++++ b/arch/xtensa/kernel/process.c +@@ -217,7 +217,7 @@ int copy_thread(unsigned long clone_flag + + p->thread.sp = (unsigned long)childregs; + +- if (!(p->flags & PF_KTHREAD)) { ++ if (!(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { + struct pt_regs *regs = current_pt_regs(); + unsigned long usp = usp_thread_fn ? + usp_thread_fn : regs->areg[1]; diff --git a/next-5.10/arm-add-support-for-tif_notify_signal.patch b/next-5.10/arm-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..c6cadc79fad --- /dev/null +++ b/next-5.10/arm-add-support-for-tif_notify_signal.patch @@ -0,0 +1,110 @@ +From 4548cba5ff2a7b81495b7c109b48972655715142 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 16:00:49 -0600 +Subject: arm: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 32d59773da38cd83e497a70eb9754d4bbae3aeae ] + +Wire up TIF_NOTIFY_SIGNAL handling for arm. + +Cc: linux-arm-kernel@lists.infradead.org +Acked-by: Russell King +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm/include/asm/thread_info.h | 7 ++++++- + arch/arm/kernel/entry-common.S | 6 +++--- + arch/arm/kernel/entry-v7m.S | 2 +- + arch/arm/kernel/signal.c | 2 +- + 4 files changed, 11 insertions(+), 6 deletions(-) + +--- a/arch/arm/include/asm/thread_info.h ++++ b/arch/arm/include/asm/thread_info.h +@@ -126,6 +126,8 @@ extern int vfp_restore_user_hwstate(stru + * thread information flags: + * TIF_USEDFPU - FPU was used by this task this quantum (SMP) + * TIF_POLLING_NRFLAG - true if poll_idle() is polling TIF_NEED_RESCHED ++ * ++ * Any bit in the range of 0..15 will cause do_work_pending() to be invoked. + */ + #define TIF_SIGPENDING 0 /* signal pending */ + #define TIF_NEED_RESCHED 1 /* rescheduling necessary */ +@@ -135,6 +137,7 @@ extern int vfp_restore_user_hwstate(stru + #define TIF_SYSCALL_AUDIT 5 /* syscall auditing active */ + #define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */ + #define TIF_SECCOMP 7 /* seccomp syscall filtering active */ ++#define TIF_NOTIFY_SIGNAL 8 /* signal notifications exist */ + + #define TIF_USING_IWMMXT 17 + #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ +@@ -148,6 +151,7 @@ extern int vfp_restore_user_hwstate(stru + #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) + #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) + #define _TIF_SECCOMP (1 << TIF_SECCOMP) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + #define _TIF_USING_IWMMXT (1 << TIF_USING_IWMMXT) + + /* Checks for any syscall work in entry-common.S */ +@@ -158,7 +162,8 @@ extern int vfp_restore_user_hwstate(stru + * Change these and you break ASM code in entry-common.S + */ + #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ +- _TIF_NOTIFY_RESUME | _TIF_UPROBE) ++ _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ ++ _TIF_NOTIFY_SIGNAL) + + #endif /* __KERNEL__ */ + #endif /* __ASM_ARM_THREAD_INFO_H */ +--- a/arch/arm/kernel/entry-common.S ++++ b/arch/arm/kernel/entry-common.S +@@ -53,7 +53,7 @@ __ret_fast_syscall: + cmp r2, #TASK_SIZE + blne addr_limit_check_failed + ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing +- tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK ++ movs r1, r1, lsl #16 + bne fast_work_pending + + +@@ -90,7 +90,7 @@ __ret_fast_syscall: + cmp r2, #TASK_SIZE + blne addr_limit_check_failed + ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing +- tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK ++ movs r1, r1, lsl #16 + beq no_work_pending + UNWIND(.fnend ) + ENDPROC(ret_fast_syscall) +@@ -131,7 +131,7 @@ ENTRY(ret_to_user_from_irq) + cmp r2, #TASK_SIZE + blne addr_limit_check_failed + ldr r1, [tsk, #TI_FLAGS] +- tst r1, #_TIF_WORK_MASK ++ movs r1, r1, lsl #16 + bne slow_work_pending + no_work_pending: + asm_trace_hardirqs_on save = 0 +--- a/arch/arm/kernel/entry-v7m.S ++++ b/arch/arm/kernel/entry-v7m.S +@@ -59,7 +59,7 @@ __irq_entry: + + get_thread_info tsk + ldr r2, [tsk, #TI_FLAGS] +- tst r2, #_TIF_WORK_MASK ++ movs r2, r2, lsl #16 + beq 2f @ no work pending + mov r0, #V7M_SCB_ICSR_PENDSVSET + str r0, [r1, V7M_SCB_ICSR] @ raise PendSV +--- a/arch/arm/kernel/signal.c ++++ b/arch/arm/kernel/signal.c +@@ -655,7 +655,7 @@ do_work_pending(struct pt_regs *regs, un + if (unlikely(!user_mode(regs))) + return 0; + local_irq_enable(); +- if (thread_flags & _TIF_SIGPENDING) { ++ if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) { + int restart = do_signal(regs, syscall); + if (unlikely(restart)) { + /* diff --git a/queue-5.10/arm-dts-armada-370-fix-assigned-addresses-for-every-.patch b/next-5.10/arm-dts-armada-370-fix-assigned-addresses-for-every-.patch similarity index 100% rename from queue-5.10/arm-dts-armada-370-fix-assigned-addresses-for-every-.patch rename to next-5.10/arm-dts-armada-370-fix-assigned-addresses-for-every-.patch diff --git a/queue-5.10/arm-dts-armada-375-fix-assigned-addresses-for-every-.patch b/next-5.10/arm-dts-armada-375-fix-assigned-addresses-for-every-.patch similarity index 100% rename from queue-5.10/arm-dts-armada-375-fix-assigned-addresses-for-every-.patch rename to next-5.10/arm-dts-armada-375-fix-assigned-addresses-for-every-.patch diff --git a/queue-5.10/arm-dts-armada-38x-fix-assigned-addresses-for-every-.patch b/next-5.10/arm-dts-armada-38x-fix-assigned-addresses-for-every-.patch similarity index 100% rename from queue-5.10/arm-dts-armada-38x-fix-assigned-addresses-for-every-.patch rename to next-5.10/arm-dts-armada-38x-fix-assigned-addresses-for-every-.patch diff --git a/queue-5.10/arm-dts-armada-39x-fix-assigned-addresses-for-every-.patch b/next-5.10/arm-dts-armada-39x-fix-assigned-addresses-for-every-.patch similarity index 100% rename from queue-5.10/arm-dts-armada-39x-fix-assigned-addresses-for-every-.patch rename to next-5.10/arm-dts-armada-39x-fix-assigned-addresses-for-every-.patch diff --git a/queue-5.10/arm-dts-armada-xp-fix-assigned-addresses-for-every-p.patch b/next-5.10/arm-dts-armada-xp-fix-assigned-addresses-for-every-p.patch similarity index 100% rename from queue-5.10/arm-dts-armada-xp-fix-assigned-addresses-for-every-p.patch rename to next-5.10/arm-dts-armada-xp-fix-assigned-addresses-for-every-p.patch diff --git a/queue-5.10/arm-dts-dove-fix-assigned-addresses-for-every-pcie-r.patch b/next-5.10/arm-dts-dove-fix-assigned-addresses-for-every-pcie-r.patch similarity index 100% rename from queue-5.10/arm-dts-dove-fix-assigned-addresses-for-every-pcie-r.patch rename to next-5.10/arm-dts-dove-fix-assigned-addresses-for-every-pcie-r.patch diff --git a/queue-5.10/arm-dts-qcom-apq8064-fix-coresight-compatible.patch b/next-5.10/arm-dts-qcom-apq8064-fix-coresight-compatible.patch similarity index 100% rename from queue-5.10/arm-dts-qcom-apq8064-fix-coresight-compatible.patch rename to next-5.10/arm-dts-qcom-apq8064-fix-coresight-compatible.patch diff --git a/queue-5.10/arm-dts-spear600-fix-clcd-interrupt.patch b/next-5.10/arm-dts-spear600-fix-clcd-interrupt.patch similarity index 100% rename from queue-5.10/arm-dts-spear600-fix-clcd-interrupt.patch rename to next-5.10/arm-dts-spear600-fix-clcd-interrupt.patch diff --git a/queue-5.10/arm-dts-stm32-drop-stm32mp15xc.dtsi-from-avenger96.patch b/next-5.10/arm-dts-stm32-drop-stm32mp15xc.dtsi-from-avenger96.patch similarity index 100% rename from queue-5.10/arm-dts-stm32-drop-stm32mp15xc.dtsi-from-avenger96.patch rename to next-5.10/arm-dts-stm32-drop-stm32mp15xc.dtsi-from-avenger96.patch diff --git a/queue-5.10/arm-dts-stm32-fix-av96-wlan-regulator-gpio-property.patch b/next-5.10/arm-dts-stm32-fix-av96-wlan-regulator-gpio-property.patch similarity index 100% rename from queue-5.10/arm-dts-stm32-fix-av96-wlan-regulator-gpio-property.patch rename to next-5.10/arm-dts-stm32-fix-av96-wlan-regulator-gpio-property.patch diff --git a/queue-5.10/arm-dts-turris-omnia-add-ethernet-aliases.patch b/next-5.10/arm-dts-turris-omnia-add-ethernet-aliases.patch similarity index 100% rename from queue-5.10/arm-dts-turris-omnia-add-ethernet-aliases.patch rename to next-5.10/arm-dts-turris-omnia-add-ethernet-aliases.patch diff --git a/queue-5.10/arm-dts-turris-omnia-add-switch-port-6-node.patch b/next-5.10/arm-dts-turris-omnia-add-switch-port-6-node.patch similarity index 100% rename from queue-5.10/arm-dts-turris-omnia-add-switch-port-6-node.patch rename to next-5.10/arm-dts-turris-omnia-add-switch-port-6-node.patch diff --git a/queue-5.10/arm-mmp-fix-timer_read-delay.patch b/next-5.10/arm-mmp-fix-timer_read-delay.patch similarity index 100% rename from queue-5.10/arm-mmp-fix-timer_read-delay.patch rename to next-5.10/arm-mmp-fix-timer_read-delay.patch diff --git a/next-5.10/arm64-add-support-for-tif_notify_signal.patch b/next-5.10/arm64-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..f90f46d1908 --- /dev/null +++ b/next-5.10/arm64-add-support-for-tif_notify_signal.patch @@ -0,0 +1,56 @@ +From eaa379f10805fdf0f096bfd32d3697f7af0dc185 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Thu, 22 Oct 2020 20:09:23 -0600 +Subject: arm64: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 192caabd4dd92c98d23ed4334d7596af05af2fb4 ] + +Wire up TIF_NOTIFY_SIGNAL handling for arm64. + +Cc: linux-arm-kernel@lists.infradead.org +Acked-by: Will Deacon +Acked-by: Catalin Marinas +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/thread_info.h | 5 ++++- + arch/arm64/kernel/signal.c | 2 +- + 2 files changed, 5 insertions(+), 2 deletions(-) + +--- a/arch/arm64/include/asm/thread_info.h ++++ b/arch/arm64/include/asm/thread_info.h +@@ -68,6 +68,7 @@ void arch_release_task_struct(struct tas + #define TIF_UPROBE 4 /* uprobe breakpoint or singlestep */ + #define TIF_FSCHECK 5 /* Check FS is USER_DS on return */ + #define TIF_MTE_ASYNC_FAULT 6 /* MTE Asynchronous Tag Check Fault */ ++#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ + #define TIF_SYSCALL_TRACE 8 /* syscall trace active */ + #define TIF_SYSCALL_AUDIT 9 /* syscall auditing */ + #define TIF_SYSCALL_TRACEPOINT 10 /* syscall tracepoint for ftrace */ +@@ -98,10 +99,12 @@ void arch_release_task_struct(struct tas + #define _TIF_32BIT (1 << TIF_32BIT) + #define _TIF_SVE (1 << TIF_SVE) + #define _TIF_MTE_ASYNC_FAULT (1 << TIF_MTE_ASYNC_FAULT) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + + #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ + _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \ +- _TIF_UPROBE | _TIF_FSCHECK | _TIF_MTE_ASYNC_FAULT) ++ _TIF_UPROBE | _TIF_FSCHECK | _TIF_MTE_ASYNC_FAULT | \ ++ _TIF_NOTIFY_SIGNAL) + + #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ + _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \ +--- a/arch/arm64/kernel/signal.c ++++ b/arch/arm64/kernel/signal.c +@@ -938,7 +938,7 @@ asmlinkage void do_notify_resume(struct + (void __user *)NULL, current); + } + +- if (thread_flags & _TIF_SIGPENDING) ++ if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs); + + if (thread_flags & _TIF_NOTIFY_RESUME) { diff --git a/queue-5.10/arm64-dts-armada-3720-turris-mox-add-missing-interru.patch b/next-5.10/arm64-dts-armada-3720-turris-mox-add-missing-interru.patch similarity index 100% rename from queue-5.10/arm64-dts-armada-3720-turris-mox-add-missing-interru.patch rename to next-5.10/arm64-dts-armada-3720-turris-mox-add-missing-interru.patch diff --git a/queue-5.10/arm64-dts-mediatek-mt6797-fix-26m-oscillator-unit-na.patch b/next-5.10/arm64-dts-mediatek-mt6797-fix-26m-oscillator-unit-na.patch similarity index 100% rename from queue-5.10/arm64-dts-mediatek-mt6797-fix-26m-oscillator-unit-na.patch rename to next-5.10/arm64-dts-mediatek-mt6797-fix-26m-oscillator-unit-na.patch diff --git a/queue-5.10/arm64-dts-mediatek-pumpkin-common-fix-devicetree-war.patch b/next-5.10/arm64-dts-mediatek-pumpkin-common-fix-devicetree-war.patch similarity index 100% rename from queue-5.10/arm64-dts-mediatek-pumpkin-common-fix-devicetree-war.patch rename to next-5.10/arm64-dts-mediatek-pumpkin-common-fix-devicetree-war.patch diff --git a/queue-5.10/arm64-dts-mt2712-evb-fix-usb-vbus-regulators-unit-na.patch b/next-5.10/arm64-dts-mt2712-evb-fix-usb-vbus-regulators-unit-na.patch similarity index 100% rename from queue-5.10/arm64-dts-mt2712-evb-fix-usb-vbus-regulators-unit-na.patch rename to next-5.10/arm64-dts-mt2712-evb-fix-usb-vbus-regulators-unit-na.patch diff --git a/queue-5.10/arm64-dts-mt2712-evb-fix-vproc-fixed-regulators-unit.patch b/next-5.10/arm64-dts-mt2712-evb-fix-vproc-fixed-regulators-unit.patch similarity index 100% rename from queue-5.10/arm64-dts-mt2712-evb-fix-vproc-fixed-regulators-unit.patch rename to next-5.10/arm64-dts-mt2712-evb-fix-vproc-fixed-regulators-unit.patch diff --git a/queue-5.10/arm64-dts-mt2712e-fix-unit-address-for-pinctrl-node.patch b/next-5.10/arm64-dts-mt2712e-fix-unit-address-for-pinctrl-node.patch similarity index 100% rename from queue-5.10/arm64-dts-mt2712e-fix-unit-address-for-pinctrl-node.patch rename to next-5.10/arm64-dts-mt2712e-fix-unit-address-for-pinctrl-node.patch diff --git a/queue-5.10/arm64-dts-mt2712e-fix-unit_address_vs_reg-warning-fo.patch b/next-5.10/arm64-dts-mt2712e-fix-unit_address_vs_reg-warning-fo.patch similarity index 100% rename from queue-5.10/arm64-dts-mt2712e-fix-unit_address_vs_reg-warning-fo.patch rename to next-5.10/arm64-dts-mt2712e-fix-unit_address_vs_reg-warning-fo.patch diff --git a/queue-5.10/arm64-dts-qcom-ipq6018-cp01-c1-use-blspi1-pins.patch b/next-5.10/arm64-dts-qcom-ipq6018-cp01-c1-use-blspi1-pins.patch similarity index 100% rename from queue-5.10/arm64-dts-qcom-ipq6018-cp01-c1-use-blspi1-pins.patch rename to next-5.10/arm64-dts-qcom-ipq6018-cp01-c1-use-blspi1-pins.patch diff --git a/queue-5.10/arm64-dts-qcom-msm8916-drop-mss-fallback-compatible.patch b/next-5.10/arm64-dts-qcom-msm8916-drop-mss-fallback-compatible.patch similarity index 100% rename from queue-5.10/arm64-dts-qcom-msm8916-drop-mss-fallback-compatible.patch rename to next-5.10/arm64-dts-qcom-msm8916-drop-mss-fallback-compatible.patch diff --git a/queue-5.10/arm64-dts-qcom-msm8996-fix-gpu-opp-table.patch b/next-5.10/arm64-dts-qcom-msm8996-fix-gpu-opp-table.patch similarity index 100% rename from queue-5.10/arm64-dts-qcom-msm8996-fix-gpu-opp-table.patch rename to next-5.10/arm64-dts-qcom-msm8996-fix-gpu-opp-table.patch diff --git a/queue-5.10/arm64-dts-qcom-sdm630-fix-uart1-pin-bias.patch b/next-5.10/arm64-dts-qcom-sdm630-fix-uart1-pin-bias.patch similarity index 100% rename from queue-5.10/arm64-dts-qcom-sdm630-fix-uart1-pin-bias.patch rename to next-5.10/arm64-dts-qcom-sdm630-fix-uart1-pin-bias.patch diff --git a/queue-5.10/arm64-dts-qcom-sdm845-cheza-fix-ap-suspend-pin-bias.patch b/next-5.10/arm64-dts-qcom-sdm845-cheza-fix-ap-suspend-pin-bias.patch similarity index 100% rename from queue-5.10/arm64-dts-qcom-sdm845-cheza-fix-ap-suspend-pin-bias.patch rename to next-5.10/arm64-dts-qcom-sdm845-cheza-fix-ap-suspend-pin-bias.patch diff --git a/queue-5.10/arm64-dts-ti-k3-am65-main-drop-dma-coherent-in-crypt.patch b/next-5.10/arm64-dts-ti-k3-am65-main-drop-dma-coherent-in-crypt.patch similarity index 100% rename from queue-5.10/arm64-dts-ti-k3-am65-main-drop-dma-coherent-in-crypt.patch rename to next-5.10/arm64-dts-ti-k3-am65-main-drop-dma-coherent-in-crypt.patch diff --git a/queue-5.10/arm64-dts-ti-k3-j721e-main-drop-dma-coherent-in-cryp.patch b/next-5.10/arm64-dts-ti-k3-j721e-main-drop-dma-coherent-in-cryp.patch similarity index 100% rename from queue-5.10/arm64-dts-ti-k3-j721e-main-drop-dma-coherent-in-cryp.patch rename to next-5.10/arm64-dts-ti-k3-j721e-main-drop-dma-coherent-in-cryp.patch diff --git a/queue-5.10/arm64-make-is_ttbrx_addr-noinstr-safe.patch b/next-5.10/arm64-make-is_ttbrx_addr-noinstr-safe.patch similarity index 100% rename from queue-5.10/arm64-make-is_ttbrx_addr-noinstr-safe.patch rename to next-5.10/arm64-make-is_ttbrx_addr-noinstr-safe.patch diff --git a/queue-5.10/asoc-audio-graph-card-fix-refcount-leak-of-cpu_ep-in.patch b/next-5.10/asoc-audio-graph-card-fix-refcount-leak-of-cpu_ep-in.patch similarity index 100% rename from queue-5.10/asoc-audio-graph-card-fix-refcount-leak-of-cpu_ep-in.patch rename to next-5.10/asoc-audio-graph-card-fix-refcount-leak-of-cpu_ep-in.patch diff --git a/queue-5.10/asoc-codecs-rt298-add-quirk-for-kbl-r-rvp-platform.patch b/next-5.10/asoc-codecs-rt298-add-quirk-for-kbl-r-rvp-platform.patch similarity index 100% rename from queue-5.10/asoc-codecs-rt298-add-quirk-for-kbl-r-rvp-platform.patch rename to next-5.10/asoc-codecs-rt298-add-quirk-for-kbl-r-rvp-platform.patch diff --git a/queue-5.10/asoc-dt-bindings-wcd9335-fix-reset-line-polarity-in-.patch b/next-5.10/asoc-dt-bindings-wcd9335-fix-reset-line-polarity-in-.patch similarity index 100% rename from queue-5.10/asoc-dt-bindings-wcd9335-fix-reset-line-polarity-in-.patch rename to next-5.10/asoc-dt-bindings-wcd9335-fix-reset-line-polarity-in-.patch diff --git a/queue-5.10/asoc-intel-skylake-fix-driver-hang-during-shutdown.patch b/next-5.10/asoc-intel-skylake-fix-driver-hang-during-shutdown.patch similarity index 100% rename from queue-5.10/asoc-intel-skylake-fix-driver-hang-during-shutdown.patch rename to next-5.10/asoc-intel-skylake-fix-driver-hang-during-shutdown.patch diff --git a/queue-5.10/asoc-mediatek-mt8173-enable-irq-when-pdata-is-ready.patch b/next-5.10/asoc-mediatek-mt8173-enable-irq-when-pdata-is-ready.patch similarity index 100% rename from queue-5.10/asoc-mediatek-mt8173-enable-irq-when-pdata-is-ready.patch rename to next-5.10/asoc-mediatek-mt8173-enable-irq-when-pdata-is-ready.patch diff --git a/queue-5.10/asoc-mediatek-mt8173-fix-debugfs-registration-for-co.patch b/next-5.10/asoc-mediatek-mt8173-fix-debugfs-registration-for-co.patch similarity index 100% rename from queue-5.10/asoc-mediatek-mt8173-fix-debugfs-registration-for-co.patch rename to next-5.10/asoc-mediatek-mt8173-fix-debugfs-registration-for-co.patch diff --git a/queue-5.10/asoc-mediatek-mt8173-rt5650-rt5514-fix-refcount-leak.patch b/next-5.10/asoc-mediatek-mt8173-rt5650-rt5514-fix-refcount-leak.patch similarity index 100% rename from queue-5.10/asoc-mediatek-mt8173-rt5650-rt5514-fix-refcount-leak.patch rename to next-5.10/asoc-mediatek-mt8173-rt5650-rt5514-fix-refcount-leak.patch diff --git a/queue-5.10/asoc-mediatek-mtk-btcvsd-add-checks-for-write-and-re.patch b/next-5.10/asoc-mediatek-mtk-btcvsd-add-checks-for-write-and-re.patch similarity index 100% rename from queue-5.10/asoc-mediatek-mtk-btcvsd-add-checks-for-write-and-re.patch rename to next-5.10/asoc-mediatek-mtk-btcvsd-add-checks-for-write-and-re.patch diff --git a/queue-5.10/asoc-pcm512x-fix-pm-disable-depth-imbalance-in-pcm51.patch b/next-5.10/asoc-pcm512x-fix-pm-disable-depth-imbalance-in-pcm51.patch similarity index 100% rename from queue-5.10/asoc-pcm512x-fix-pm-disable-depth-imbalance-in-pcm51.patch rename to next-5.10/asoc-pcm512x-fix-pm-disable-depth-imbalance-in-pcm51.patch diff --git a/queue-5.10/asoc-pxa-fix-null-pointer-dereference-in-filter.patch b/next-5.10/asoc-pxa-fix-null-pointer-dereference-in-filter.patch similarity index 100% rename from queue-5.10/asoc-pxa-fix-null-pointer-dereference-in-filter.patch rename to next-5.10/asoc-pxa-fix-null-pointer-dereference-in-filter.patch diff --git a/queue-5.10/asoc-qcom-add-checks-for-devm_kcalloc.patch b/next-5.10/asoc-qcom-add-checks-for-devm_kcalloc.patch similarity index 100% rename from queue-5.10/asoc-qcom-add-checks-for-devm_kcalloc.patch rename to next-5.10/asoc-qcom-add-checks-for-devm_kcalloc.patch diff --git a/queue-5.10/asoc-rockchip-pdm-add-missing-clk_disable_unprepare-.patch b/next-5.10/asoc-rockchip-pdm-add-missing-clk_disable_unprepare-.patch similarity index 100% rename from queue-5.10/asoc-rockchip-pdm-add-missing-clk_disable_unprepare-.patch rename to next-5.10/asoc-rockchip-pdm-add-missing-clk_disable_unprepare-.patch diff --git a/queue-5.10/asoc-rockchip-spdif-add-missing-clk_disable_unprepar.patch b/next-5.10/asoc-rockchip-spdif-add-missing-clk_disable_unprepar.patch similarity index 100% rename from queue-5.10/asoc-rockchip-spdif-add-missing-clk_disable_unprepar.patch rename to next-5.10/asoc-rockchip-spdif-add-missing-clk_disable_unprepar.patch diff --git a/queue-5.10/asoc-rt5670-remove-unbalanced-pm_runtime_put.patch b/next-5.10/asoc-rt5670-remove-unbalanced-pm_runtime_put.patch similarity index 100% rename from queue-5.10/asoc-rt5670-remove-unbalanced-pm_runtime_put.patch rename to next-5.10/asoc-rt5670-remove-unbalanced-pm_runtime_put.patch diff --git a/queue-5.10/asoc-wm8994-fix-potential-deadlock.patch b/next-5.10/asoc-wm8994-fix-potential-deadlock.patch similarity index 100% rename from queue-5.10/asoc-wm8994-fix-potential-deadlock.patch rename to next-5.10/asoc-wm8994-fix-potential-deadlock.patch diff --git a/queue-5.10/ata-ahci-fix-pcs-quirk-application-for-suspend.patch b/next-5.10/ata-ahci-fix-pcs-quirk-application-for-suspend.patch similarity index 100% rename from queue-5.10/ata-ahci-fix-pcs-quirk-application-for-suspend.patch rename to next-5.10/ata-ahci-fix-pcs-quirk-application-for-suspend.patch diff --git a/queue-5.10/binfmt-fix-error-return-code-in-load_elf_fdpic_binary.patch b/next-5.10/binfmt-fix-error-return-code-in-load_elf_fdpic_binary.patch similarity index 100% rename from queue-5.10/binfmt-fix-error-return-code-in-load_elf_fdpic_binary.patch rename to next-5.10/binfmt-fix-error-return-code-in-load_elf_fdpic_binary.patch diff --git a/queue-5.10/binfmt_misc-fix-shift-out-of-bounds-in-check_special.patch b/next-5.10/binfmt_misc-fix-shift-out-of-bounds-in-check_special.patch similarity index 100% rename from queue-5.10/binfmt_misc-fix-shift-out-of-bounds-in-check_special.patch rename to next-5.10/binfmt_misc-fix-shift-out-of-bounds-in-check_special.patch diff --git a/queue-5.10/blk-mq-fix-possible-memleak-when-register-hctx-faile.patch b/next-5.10/blk-mq-fix-possible-memleak-when-register-hctx-faile.patch similarity index 100% rename from queue-5.10/blk-mq-fix-possible-memleak-when-register-hctx-faile.patch rename to next-5.10/blk-mq-fix-possible-memleak-when-register-hctx-faile.patch diff --git a/queue-5.10/blktrace-fix-output-non-blktrace-event-when-blk_clas.patch b/next-5.10/blktrace-fix-output-non-blktrace-event-when-blk_clas.patch similarity index 100% rename from queue-5.10/blktrace-fix-output-non-blktrace-event-when-blk_clas.patch rename to next-5.10/blktrace-fix-output-non-blktrace-event-when-blk_clas.patch diff --git a/queue-5.10/bluetooth-btusb-don-t-call-kfree_skb-under-spin_lock.patch b/next-5.10/bluetooth-btusb-don-t-call-kfree_skb-under-spin_lock.patch similarity index 100% rename from queue-5.10/bluetooth-btusb-don-t-call-kfree_skb-under-spin_lock.patch rename to next-5.10/bluetooth-btusb-don-t-call-kfree_skb-under-spin_lock.patch diff --git a/queue-5.10/bluetooth-hci_bcsp-don-t-call-kfree_skb-under-spin_l.patch b/next-5.10/bluetooth-hci_bcsp-don-t-call-kfree_skb-under-spin_l.patch similarity index 100% rename from queue-5.10/bluetooth-hci_bcsp-don-t-call-kfree_skb-under-spin_l.patch rename to next-5.10/bluetooth-hci_bcsp-don-t-call-kfree_skb-under-spin_l.patch diff --git a/queue-5.10/bluetooth-hci_core-don-t-call-kfree_skb-under-spin_l.patch b/next-5.10/bluetooth-hci_core-don-t-call-kfree_skb-under-spin_l.patch similarity index 100% rename from queue-5.10/bluetooth-hci_core-don-t-call-kfree_skb-under-spin_l.patch rename to next-5.10/bluetooth-hci_core-don-t-call-kfree_skb-under-spin_l.patch diff --git a/queue-5.10/bluetooth-hci_h5-don-t-call-kfree_skb-under-spin_loc.patch b/next-5.10/bluetooth-hci_h5-don-t-call-kfree_skb-under-spin_loc.patch similarity index 100% rename from queue-5.10/bluetooth-hci_h5-don-t-call-kfree_skb-under-spin_loc.patch rename to next-5.10/bluetooth-hci_h5-don-t-call-kfree_skb-under-spin_loc.patch diff --git a/queue-5.10/bluetooth-hci_ll-don-t-call-kfree_skb-under-spin_loc.patch b/next-5.10/bluetooth-hci_ll-don-t-call-kfree_skb-under-spin_loc.patch similarity index 100% rename from queue-5.10/bluetooth-hci_ll-don-t-call-kfree_skb-under-spin_loc.patch rename to next-5.10/bluetooth-hci_ll-don-t-call-kfree_skb-under-spin_loc.patch diff --git a/queue-5.10/bluetooth-hci_qca-don-t-call-kfree_skb-under-spin_lo.patch b/next-5.10/bluetooth-hci_qca-don-t-call-kfree_skb-under-spin_lo.patch similarity index 100% rename from queue-5.10/bluetooth-hci_qca-don-t-call-kfree_skb-under-spin_lo.patch rename to next-5.10/bluetooth-hci_qca-don-t-call-kfree_skb-under-spin_lo.patch diff --git a/queue-5.10/bluetooth-rfcomm-don-t-call-kfree_skb-under-spin_loc.patch b/next-5.10/bluetooth-rfcomm-don-t-call-kfree_skb-under-spin_loc.patch similarity index 100% rename from queue-5.10/bluetooth-rfcomm-don-t-call-kfree_skb-under-spin_loc.patch rename to next-5.10/bluetooth-rfcomm-don-t-call-kfree_skb-under-spin_loc.patch diff --git a/queue-5.10/bonding-fix-link-recovery-in-mode-2-when-updelay-is-.patch b/next-5.10/bonding-fix-link-recovery-in-mode-2-when-updelay-is-.patch similarity index 100% rename from queue-5.10/bonding-fix-link-recovery-in-mode-2-when-updelay-is-.patch rename to next-5.10/bonding-fix-link-recovery-in-mode-2-when-updelay-is-.patch diff --git a/queue-5.10/bonding-uninitialized-variable-in-bond_miimon_inspec.patch b/next-5.10/bonding-uninitialized-variable-in-bond_miimon_inspec.patch similarity index 100% rename from queue-5.10/bonding-uninitialized-variable-in-bond_miimon_inspec.patch rename to next-5.10/bonding-uninitialized-variable-in-bond_miimon_inspec.patch diff --git a/queue-5.10/bpf-check-the-other-end-of-slot_type-for-stack_spill.patch b/next-5.10/bpf-check-the-other-end-of-slot_type-for-stack_spill.patch similarity index 100% rename from queue-5.10/bpf-check-the-other-end-of-slot_type-for-stack_spill.patch rename to next-5.10/bpf-check-the-other-end-of-slot_type-for-stack_spill.patch diff --git a/queue-5.10/bpf-fix-slot-type-check-in-check_stack_write_var_off.patch b/next-5.10/bpf-fix-slot-type-check-in-check_stack_write_var_off.patch similarity index 100% rename from queue-5.10/bpf-fix-slot-type-check-in-check_stack_write_var_off.patch rename to next-5.10/bpf-fix-slot-type-check-in-check_stack_write_var_off.patch diff --git a/queue-5.10/bpf-make-sure-skb-len-0-when-redirecting-to-a-tunnel.patch b/next-5.10/bpf-make-sure-skb-len-0-when-redirecting-to-a-tunnel.patch similarity index 100% rename from queue-5.10/bpf-make-sure-skb-len-0-when-redirecting-to-a-tunnel.patch rename to next-5.10/bpf-make-sure-skb-len-0-when-redirecting-to-a-tunnel.patch diff --git a/queue-5.10/bpf-move-skb-len-0-checks-into-__bpf_redirect.patch b/next-5.10/bpf-move-skb-len-0-checks-into-__bpf_redirect.patch similarity index 100% rename from queue-5.10/bpf-move-skb-len-0-checks-into-__bpf_redirect.patch rename to next-5.10/bpf-move-skb-len-0-checks-into-__bpf_redirect.patch diff --git a/queue-5.10/bpf-prevent-decl_tag-from-being-referenced-in-func_p.patch b/next-5.10/bpf-prevent-decl_tag-from-being-referenced-in-func_p.patch similarity index 100% rename from queue-5.10/bpf-prevent-decl_tag-from-being-referenced-in-func_p.patch rename to next-5.10/bpf-prevent-decl_tag-from-being-referenced-in-func_p.patch diff --git a/queue-5.10/bpf-propagate-precision-across-all-frames-not-just-t.patch b/next-5.10/bpf-propagate-precision-across-all-frames-not-just-t.patch similarity index 100% rename from queue-5.10/bpf-propagate-precision-across-all-frames-not-just-t.patch rename to next-5.10/bpf-propagate-precision-across-all-frames-not-just-t.patch diff --git a/queue-5.10/bpf-propagate-precision-in-alu-alu64-operations.patch b/next-5.10/bpf-propagate-precision-in-alu-alu64-operations.patch similarity index 100% rename from queue-5.10/bpf-propagate-precision-in-alu-alu64-operations.patch rename to next-5.10/bpf-propagate-precision-in-alu-alu64-operations.patch diff --git a/queue-5.10/bpf-sockmap-fix-data-loss-caused-by-using-apply_byte.patch b/next-5.10/bpf-sockmap-fix-data-loss-caused-by-using-apply_byte.patch similarity index 100% rename from queue-5.10/bpf-sockmap-fix-data-loss-caused-by-using-apply_byte.patch rename to next-5.10/bpf-sockmap-fix-data-loss-caused-by-using-apply_byte.patch diff --git a/queue-5.10/bpf-sockmap-fix-race-in-sock_map_free.patch b/next-5.10/bpf-sockmap-fix-race-in-sock_map_free.patch similarity index 100% rename from queue-5.10/bpf-sockmap-fix-race-in-sock_map_free.patch rename to next-5.10/bpf-sockmap-fix-race-in-sock_map_free.patch diff --git a/queue-5.10/bpf-sockmap-fix-repeated-calls-to-sock_put-when-msg-.patch b/next-5.10/bpf-sockmap-fix-repeated-calls-to-sock_put-when-msg-.patch similarity index 100% rename from queue-5.10/bpf-sockmap-fix-repeated-calls-to-sock_put-when-msg-.patch rename to next-5.10/bpf-sockmap-fix-repeated-calls-to-sock_put-when-msg-.patch diff --git a/queue-5.10/brcmfmac-return-error-when-getting-invalid-max_flowr.patch b/next-5.10/brcmfmac-return-error-when-getting-invalid-max_flowr.patch similarity index 100% rename from queue-5.10/brcmfmac-return-error-when-getting-invalid-max_flowr.patch rename to next-5.10/brcmfmac-return-error-when-getting-invalid-max_flowr.patch diff --git a/next-5.10/c6x-add-support-for-tif_notify_signal.patch b/next-5.10/c6x-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..a43c5259484 --- /dev/null +++ b/next-5.10/c6x-add-support-for-tif_notify_signal.patch @@ -0,0 +1,59 @@ +From bc746e497c6c8bcda6bb30d58a2502db4eaaf600 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 14:35:34 -0600 +Subject: c6x: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 6d665a4d8b4264def0fbb72da3a500d9904ffe3e ] + +Wire up TIF_NOTIFY_SIGNAL handling for c6x. + +Cc: linux-c6x-dev@linux-c6x.org +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/c6x/include/asm/thread_info.h | 1 + + arch/c6x/kernel/asm-offsets.c | 1 + + arch/c6x/kernel/signal.c | 3 ++- + 3 files changed, 4 insertions(+), 1 deletion(-) + +--- a/arch/c6x/include/asm/thread_info.h ++++ b/arch/c6x/include/asm/thread_info.h +@@ -82,6 +82,7 @@ struct thread_info *current_thread_info( + #define TIF_SIGPENDING 2 /* signal pending */ + #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ + #define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */ ++#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ + + #define TIF_MEMDIE 17 /* OOM killer killed process */ + +--- a/arch/c6x/kernel/asm-offsets.c ++++ b/arch/c6x/kernel/asm-offsets.c +@@ -116,6 +116,7 @@ void foo(void) + DEFINE(_TIF_NOTIFY_RESUME, (1< + #include + ++#include + #include + #include + +@@ -313,7 +314,7 @@ asmlinkage void do_notify_resume(struct + int syscall) + { + /* deal with pending signal delivery */ +- if (thread_info_flags & (1 << TIF_SIGPENDING)) ++ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs, syscall); + + if (thread_info_flags & (1 << TIF_NOTIFY_RESUME)) diff --git a/queue-5.10/can-kvaser_usb-add-struct-kvaser_usb_busparams.patch b/next-5.10/can-kvaser_usb-add-struct-kvaser_usb_busparams.patch similarity index 100% rename from queue-5.10/can-kvaser_usb-add-struct-kvaser_usb_busparams.patch rename to next-5.10/can-kvaser_usb-add-struct-kvaser_usb_busparams.patch diff --git a/queue-5.10/can-kvaser_usb-compare-requested-bittiming-parameter.patch b/next-5.10/can-kvaser_usb-compare-requested-bittiming-parameter.patch similarity index 100% rename from queue-5.10/can-kvaser_usb-compare-requested-bittiming-parameter.patch rename to next-5.10/can-kvaser_usb-compare-requested-bittiming-parameter.patch diff --git a/queue-5.10/can-kvaser_usb-do-not-increase-tx-statistics-when-se.patch b/next-5.10/can-kvaser_usb-do-not-increase-tx-statistics-when-se.patch similarity index 100% rename from queue-5.10/can-kvaser_usb-do-not-increase-tx-statistics-when-se.patch rename to next-5.10/can-kvaser_usb-do-not-increase-tx-statistics-when-se.patch diff --git a/queue-5.10/can-kvaser_usb-kvaser_usb_leaf-get-capabilities-from.patch b/next-5.10/can-kvaser_usb-kvaser_usb_leaf-get-capabilities-from.patch similarity index 100% rename from queue-5.10/can-kvaser_usb-kvaser_usb_leaf-get-capabilities-from.patch rename to next-5.10/can-kvaser_usb-kvaser_usb_leaf-get-capabilities-from.patch diff --git a/queue-5.10/can-kvaser_usb-kvaser_usb_leaf-handle-cmd_error_even.patch b/next-5.10/can-kvaser_usb-kvaser_usb_leaf-handle-cmd_error_even.patch similarity index 100% rename from queue-5.10/can-kvaser_usb-kvaser_usb_leaf-handle-cmd_error_even.patch rename to next-5.10/can-kvaser_usb-kvaser_usb_leaf-handle-cmd_error_even.patch diff --git a/queue-5.10/can-kvaser_usb-kvaser_usb_leaf-rename-leaf-usbcan-_c.patch b/next-5.10/can-kvaser_usb-kvaser_usb_leaf-rename-leaf-usbcan-_c.patch similarity index 100% rename from queue-5.10/can-kvaser_usb-kvaser_usb_leaf-rename-leaf-usbcan-_c.patch rename to next-5.10/can-kvaser_usb-kvaser_usb_leaf-rename-leaf-usbcan-_c.patch diff --git a/queue-5.10/can-kvaser_usb_leaf-fix-bogus-restart-events.patch b/next-5.10/can-kvaser_usb_leaf-fix-bogus-restart-events.patch similarity index 100% rename from queue-5.10/can-kvaser_usb_leaf-fix-bogus-restart-events.patch rename to next-5.10/can-kvaser_usb_leaf-fix-bogus-restart-events.patch diff --git a/queue-5.10/can-kvaser_usb_leaf-fix-improved-state-not-being-rep.patch b/next-5.10/can-kvaser_usb_leaf-fix-improved-state-not-being-rep.patch similarity index 100% rename from queue-5.10/can-kvaser_usb_leaf-fix-improved-state-not-being-rep.patch rename to next-5.10/can-kvaser_usb_leaf-fix-improved-state-not-being-rep.patch diff --git a/queue-5.10/can-kvaser_usb_leaf-fix-wrong-can-state-after-stoppi.patch b/next-5.10/can-kvaser_usb_leaf-fix-wrong-can-state-after-stoppi.patch similarity index 100% rename from queue-5.10/can-kvaser_usb_leaf-fix-wrong-can-state-after-stoppi.patch rename to next-5.10/can-kvaser_usb_leaf-fix-wrong-can-state-after-stoppi.patch diff --git a/queue-5.10/can-kvaser_usb_leaf-set-warning-state-even-without-b.patch b/next-5.10/can-kvaser_usb_leaf-set-warning-state-even-without-b.patch similarity index 100% rename from queue-5.10/can-kvaser_usb_leaf-set-warning-state-even-without-b.patch rename to next-5.10/can-kvaser_usb_leaf-set-warning-state-even-without-b.patch diff --git a/queue-5.10/can-tcan4x5x-remove-invalid-write-in-clear_interrupt.patch b/next-5.10/can-tcan4x5x-remove-invalid-write-in-clear_interrupt.patch similarity index 100% rename from queue-5.10/can-tcan4x5x-remove-invalid-write-in-clear_interrupt.patch rename to next-5.10/can-tcan4x5x-remove-invalid-write-in-clear_interrupt.patch diff --git a/queue-5.10/chardev-fix-error-handling-in-cdev_device_add.patch b/next-5.10/chardev-fix-error-handling-in-cdev_device_add.patch similarity index 100% rename from queue-5.10/chardev-fix-error-handling-in-cdev_device_add.patch rename to next-5.10/chardev-fix-error-handling-in-cdev_device_add.patch diff --git a/queue-5.10/cifs-fix-oops-during-encryption.patch b/next-5.10/cifs-fix-oops-during-encryption.patch similarity index 100% rename from queue-5.10/cifs-fix-oops-during-encryption.patch rename to next-5.10/cifs-fix-oops-during-encryption.patch diff --git a/queue-5.10/class-fix-possible-memory-leak-in-__class_register.patch b/next-5.10/class-fix-possible-memory-leak-in-__class_register.patch similarity index 100% rename from queue-5.10/class-fix-possible-memory-leak-in-__class_register.patch rename to next-5.10/class-fix-possible-memory-leak-in-__class_register.patch diff --git a/queue-5.10/clk-imx-replace-osc_hdmi-with-dummy.patch b/next-5.10/clk-imx-replace-osc_hdmi-with-dummy.patch similarity index 100% rename from queue-5.10/clk-imx-replace-osc_hdmi-with-dummy.patch rename to next-5.10/clk-imx-replace-osc_hdmi-with-dummy.patch diff --git a/queue-5.10/clk-qcom-clk-krait-fix-wrong-div2-functions.patch b/next-5.10/clk-qcom-clk-krait-fix-wrong-div2-functions.patch similarity index 100% rename from queue-5.10/clk-qcom-clk-krait-fix-wrong-div2-functions.patch rename to next-5.10/clk-qcom-clk-krait-fix-wrong-div2-functions.patch diff --git a/queue-5.10/clk-qcom-gcc-sm8250-use-retention-mode-for-usb-gdscs.patch b/next-5.10/clk-qcom-gcc-sm8250-use-retention-mode-for-usb-gdscs.patch similarity index 100% rename from queue-5.10/clk-qcom-gcc-sm8250-use-retention-mode-for-usb-gdscs.patch rename to next-5.10/clk-qcom-gcc-sm8250-use-retention-mode-for-usb-gdscs.patch diff --git a/queue-5.10/clk-renesas-r9a06g032-repair-grave-increment-error.patch b/next-5.10/clk-renesas-r9a06g032-repair-grave-increment-error.patch similarity index 100% rename from queue-5.10/clk-renesas-r9a06g032-repair-grave-increment-error.patch rename to next-5.10/clk-renesas-r9a06g032-repair-grave-increment-error.patch diff --git a/queue-5.10/clk-rockchip-fix-memory-leak-in-rockchip_clk_registe.patch b/next-5.10/clk-rockchip-fix-memory-leak-in-rockchip_clk_registe.patch similarity index 100% rename from queue-5.10/clk-rockchip-fix-memory-leak-in-rockchip_clk_registe.patch rename to next-5.10/clk-rockchip-fix-memory-leak-in-rockchip_clk_registe.patch diff --git a/queue-5.10/clk-samsung-fix-memory-leak-in-_samsung_clk_register.patch b/next-5.10/clk-samsung-fix-memory-leak-in-_samsung_clk_register.patch similarity index 100% rename from queue-5.10/clk-samsung-fix-memory-leak-in-_samsung_clk_register.patch rename to next-5.10/clk-samsung-fix-memory-leak-in-_samsung_clk_register.patch diff --git a/queue-5.10/clk-socfpga-clk-pll-remove-unused-variable-rc.patch b/next-5.10/clk-socfpga-clk-pll-remove-unused-variable-rc.patch similarity index 100% rename from queue-5.10/clk-socfpga-clk-pll-remove-unused-variable-rc.patch rename to next-5.10/clk-socfpga-clk-pll-remove-unused-variable-rc.patch diff --git a/queue-5.10/clk-socfpga-fix-memory-leak-in-socfpga_gate_init.patch b/next-5.10/clk-socfpga-fix-memory-leak-in-socfpga_gate_init.patch similarity index 100% rename from queue-5.10/clk-socfpga-fix-memory-leak-in-socfpga_gate_init.patch rename to next-5.10/clk-socfpga-fix-memory-leak-in-socfpga_gate_init.patch diff --git a/queue-5.10/clk-socfpga-use-clk_hw_register-for-a5-c5.patch b/next-5.10/clk-socfpga-use-clk_hw_register-for-a5-c5.patch similarity index 100% rename from queue-5.10/clk-socfpga-use-clk_hw_register-for-a5-c5.patch rename to next-5.10/clk-socfpga-use-clk_hw_register-for-a5-c5.patch diff --git a/queue-5.10/clk-st-fix-memory-leak-in-st_of_quadfs_setup.patch b/next-5.10/clk-st-fix-memory-leak-in-st_of_quadfs_setup.patch similarity index 100% rename from queue-5.10/clk-st-fix-memory-leak-in-st_of_quadfs_setup.patch rename to next-5.10/clk-st-fix-memory-leak-in-st_of_quadfs_setup.patch diff --git a/queue-5.10/clocksource-drivers-sh_cmt-access-registers-accordin.patch b/next-5.10/clocksource-drivers-sh_cmt-access-registers-accordin.patch similarity index 100% rename from queue-5.10/clocksource-drivers-sh_cmt-access-registers-accordin.patch rename to next-5.10/clocksource-drivers-sh_cmt-access-registers-accordin.patch diff --git a/queue-5.10/clocksource-drivers-sh_cmt-make-sure-channel-clock-s.patch b/next-5.10/clocksource-drivers-sh_cmt-make-sure-channel-clock-s.patch similarity index 100% rename from queue-5.10/clocksource-drivers-sh_cmt-make-sure-channel-clock-s.patch rename to next-5.10/clocksource-drivers-sh_cmt-make-sure-channel-clock-s.patch diff --git a/queue-5.10/clocksource-drivers-timer-ti-dm-fix-missing-clk_disa.patch b/next-5.10/clocksource-drivers-timer-ti-dm-fix-missing-clk_disa.patch similarity index 100% rename from queue-5.10/clocksource-drivers-timer-ti-dm-fix-missing-clk_disa.patch rename to next-5.10/clocksource-drivers-timer-ti-dm-fix-missing-clk_disa.patch diff --git a/queue-5.10/configfs-fix-possible-memory-leak-in-configfs_create.patch b/next-5.10/configfs-fix-possible-memory-leak-in-configfs_create.patch similarity index 100% rename from queue-5.10/configfs-fix-possible-memory-leak-in-configfs_create.patch rename to next-5.10/configfs-fix-possible-memory-leak-in-configfs_create.patch diff --git a/next-5.10/coredump-limit-what-can-interrupt-coredumps.patch b/next-5.10/coredump-limit-what-can-interrupt-coredumps.patch new file mode 100644 index 00000000000..0d91f001463 --- /dev/null +++ b/next-5.10/coredump-limit-what-can-interrupt-coredumps.patch @@ -0,0 +1,61 @@ +From 428760d101b98af1e7675882337dcad7c6786d87 Mon Sep 17 00:00:00 2001 +From: "Eric W. Biederman" +Date: Thu, 10 Jun 2021 15:11:11 -0500 +Subject: coredump: Limit what can interrupt coredumps + +From: "Eric W. Biederman" + +[ Upstream commit 06af8679449d4ed282df13191fc52d5ba28ec536 ] + +Olivier Langlois has been struggling with coredumps being incompletely written in +processes using io_uring. + +Olivier Langlois writes: +> io_uring is a big user of task_work and any event that io_uring made a +> task waiting for that occurs during the core dump generation will +> generate a TIF_NOTIFY_SIGNAL. +> +> Here are the detailed steps of the problem: +> 1. io_uring calls vfs_poll() to install a task to a file wait queue +> with io_async_wake() as the wakeup function cb from io_arm_poll_handler() +> 2. wakeup function ends up calling task_work_add() with TWA_SIGNAL +> 3. task_work_add() sets the TIF_NOTIFY_SIGNAL bit by calling +> set_notify_signal() + +The coredump code deliberately supports being interrupted by SIGKILL, +and depends upon prepare_signal to filter out all other signals. Now +that signal_pending includes wake ups for TIF_NOTIFY_SIGNAL this hack +in dump_emitted by the coredump code no longer works. + +Make the coredump code more robust by explicitly testing for all of +the wakeup conditions the coredump code supports. This prevents +new wakeup conditions from breaking the coredump code, as well +as fixing the current issue. + +The filesystem code that the coredump code uses already limits +itself to only aborting on fatal_signal_pending. So it should +not develop surprising wake-up reasons either. + +v2: Don't remove the now unnecessary code in prepare_signal. + +Cc: stable@vger.kernel.org +Fixes: 12db8b690010 ("entry: Add support for TIF_NOTIFY_SIGNAL") +Reported-by: Olivier Langlois +Signed-off-by: "Eric W. Biederman" +Signed-off-by: Linus Torvalds +Signed-off-by: Greg Kroah-Hartman +--- + fs/coredump.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +--- a/fs/coredump.c ++++ b/fs/coredump.c +@@ -523,7 +523,7 @@ static bool dump_interrupted(void) + * but then we need to teach dump_write() to restart and clear + * TIF_SIGPENDING. + */ +- return signal_pending(current); ++ return fatal_signal_pending(current) || freezing(current); + } + + static void wait_for_dump_helpers(struct file *file) diff --git a/queue-5.10/counter-stm32-lptimer-cnt-fix-the-check-on-arr-and-c.patch b/next-5.10/counter-stm32-lptimer-cnt-fix-the-check-on-arr-and-c.patch similarity index 100% rename from queue-5.10/counter-stm32-lptimer-cnt-fix-the-check-on-arr-and-c.patch rename to next-5.10/counter-stm32-lptimer-cnt-fix-the-check-on-arr-and-c.patch diff --git a/queue-5.10/cpu-hotplug-make-target_store-a-nop-when-target-stat.patch b/next-5.10/cpu-hotplug-make-target_store-a-nop-when-target-stat.patch similarity index 100% rename from queue-5.10/cpu-hotplug-make-target_store-a-nop-when-target-stat.patch rename to next-5.10/cpu-hotplug-make-target_store-a-nop-when-target-stat.patch diff --git a/queue-5.10/cpufreq-amd_freq_sensitivity-add-missing-pci_dev_put.patch b/next-5.10/cpufreq-amd_freq_sensitivity-add-missing-pci_dev_put.patch similarity index 100% rename from queue-5.10/cpufreq-amd_freq_sensitivity-add-missing-pci_dev_put.patch rename to next-5.10/cpufreq-amd_freq_sensitivity-add-missing-pci_dev_put.patch diff --git a/queue-5.10/cpufreq-qcom-hw-fix-memory-leak-in-qcom_cpufreq_hw_r.patch b/next-5.10/cpufreq-qcom-hw-fix-memory-leak-in-qcom_cpufreq_hw_r.patch similarity index 100% rename from queue-5.10/cpufreq-qcom-hw-fix-memory-leak-in-qcom_cpufreq_hw_r.patch rename to next-5.10/cpufreq-qcom-hw-fix-memory-leak-in-qcom_cpufreq_hw_r.patch diff --git a/queue-5.10/cpuidle-dt-return-the-correct-numbers-of-parsed-idle.patch b/next-5.10/cpuidle-dt-return-the-correct-numbers-of-parsed-idle.patch similarity index 100% rename from queue-5.10/cpuidle-dt-return-the-correct-numbers-of-parsed-idle.patch rename to next-5.10/cpuidle-dt-return-the-correct-numbers-of-parsed-idle.patch diff --git a/queue-5.10/crypto-amlogic-remove-kcalloc-without-check.patch b/next-5.10/crypto-amlogic-remove-kcalloc-without-check.patch similarity index 100% rename from queue-5.10/crypto-amlogic-remove-kcalloc-without-check.patch rename to next-5.10/crypto-amlogic-remove-kcalloc-without-check.patch diff --git a/queue-5.10/crypto-ccree-make-cc_debugfs_global_fini-available-f.patch b/next-5.10/crypto-ccree-make-cc_debugfs_global_fini-available-f.patch similarity index 100% rename from queue-5.10/crypto-ccree-make-cc_debugfs_global_fini-available-f.patch rename to next-5.10/crypto-ccree-make-cc_debugfs_global_fini-available-f.patch diff --git a/queue-5.10/crypto-ccree-remove-debugfs-when-platform_driver_reg.patch b/next-5.10/crypto-ccree-remove-debugfs-when-platform_driver_reg.patch similarity index 100% rename from queue-5.10/crypto-ccree-remove-debugfs-when-platform_driver_reg.patch rename to next-5.10/crypto-ccree-remove-debugfs-when-platform_driver_reg.patch diff --git a/queue-5.10/crypto-cryptd-use-request-context-instead-of-stack-f.patch b/next-5.10/crypto-cryptd-use-request-context-instead-of-stack-f.patch similarity index 100% rename from queue-5.10/crypto-cryptd-use-request-context-instead-of-stack-f.patch rename to next-5.10/crypto-cryptd-use-request-context-instead-of-stack-f.patch diff --git a/queue-5.10/crypto-hisilicon-qm-add-missing-pci_dev_put-in-q_num.patch b/next-5.10/crypto-hisilicon-qm-add-missing-pci_dev_put-in-q_num.patch similarity index 100% rename from queue-5.10/crypto-hisilicon-qm-add-missing-pci_dev_put-in-q_num.patch rename to next-5.10/crypto-hisilicon-qm-add-missing-pci_dev_put-in-q_num.patch diff --git a/queue-5.10/crypto-img-hash-fix-variable-dereferenced-before-che.patch b/next-5.10/crypto-img-hash-fix-variable-dereferenced-before-che.patch similarity index 100% rename from queue-5.10/crypto-img-hash-fix-variable-dereferenced-before-che.patch rename to next-5.10/crypto-img-hash-fix-variable-dereferenced-before-che.patch diff --git a/queue-5.10/crypto-nitrox-avoid-double-free-on-error-path-in-nit.patch b/next-5.10/crypto-nitrox-avoid-double-free-on-error-path-in-nit.patch similarity index 100% rename from queue-5.10/crypto-nitrox-avoid-double-free-on-error-path-in-nit.patch rename to next-5.10/crypto-nitrox-avoid-double-free-on-error-path-in-nit.patch diff --git a/queue-5.10/crypto-omap-sham-use-pm_runtime_resume_and_get-in-om.patch b/next-5.10/crypto-omap-sham-use-pm_runtime_resume_and_get-in-om.patch similarity index 100% rename from queue-5.10/crypto-omap-sham-use-pm_runtime_resume_and_get-in-om.patch rename to next-5.10/crypto-omap-sham-use-pm_runtime_resume_and_get-in-om.patch diff --git a/queue-5.10/crypto-rockchip-add-fallback-for-ahash.patch b/next-5.10/crypto-rockchip-add-fallback-for-ahash.patch similarity index 100% rename from queue-5.10/crypto-rockchip-add-fallback-for-ahash.patch rename to next-5.10/crypto-rockchip-add-fallback-for-ahash.patch diff --git a/queue-5.10/crypto-rockchip-add-fallback-for-cipher.patch b/next-5.10/crypto-rockchip-add-fallback-for-cipher.patch similarity index 100% rename from queue-5.10/crypto-rockchip-add-fallback-for-cipher.patch rename to next-5.10/crypto-rockchip-add-fallback-for-cipher.patch diff --git a/queue-5.10/crypto-rockchip-better-handle-cipher-key.patch b/next-5.10/crypto-rockchip-better-handle-cipher-key.patch similarity index 100% rename from queue-5.10/crypto-rockchip-better-handle-cipher-key.patch rename to next-5.10/crypto-rockchip-better-handle-cipher-key.patch diff --git a/queue-5.10/crypto-rockchip-delete-unneeded-variable-initializat.patch b/next-5.10/crypto-rockchip-delete-unneeded-variable-initializat.patch similarity index 100% rename from queue-5.10/crypto-rockchip-delete-unneeded-variable-initializat.patch rename to next-5.10/crypto-rockchip-delete-unneeded-variable-initializat.patch diff --git a/queue-5.10/crypto-rockchip-do-not-do-custom-power-management.patch b/next-5.10/crypto-rockchip-do-not-do-custom-power-management.patch similarity index 100% rename from queue-5.10/crypto-rockchip-do-not-do-custom-power-management.patch rename to next-5.10/crypto-rockchip-do-not-do-custom-power-management.patch diff --git a/queue-5.10/crypto-rockchip-do-not-store-mode-globally.patch b/next-5.10/crypto-rockchip-do-not-store-mode-globally.patch similarity index 100% rename from queue-5.10/crypto-rockchip-do-not-store-mode-globally.patch rename to next-5.10/crypto-rockchip-do-not-store-mode-globally.patch diff --git a/queue-5.10/crypto-rockchip-remove-non-aligned-handling.patch b/next-5.10/crypto-rockchip-remove-non-aligned-handling.patch similarity index 100% rename from queue-5.10/crypto-rockchip-remove-non-aligned-handling.patch rename to next-5.10/crypto-rockchip-remove-non-aligned-handling.patch diff --git a/queue-5.10/crypto-rockchip-rework-by-using-crypto_engine.patch b/next-5.10/crypto-rockchip-rework-by-using-crypto_engine.patch similarity index 100% rename from queue-5.10/crypto-rockchip-rework-by-using-crypto_engine.patch rename to next-5.10/crypto-rockchip-rework-by-using-crypto_engine.patch diff --git a/queue-5.10/crypto-sun8i-ss-use-dma_addr-instead-u32.patch b/next-5.10/crypto-sun8i-ss-use-dma_addr-instead-u32.patch similarity index 100% rename from queue-5.10/crypto-sun8i-ss-use-dma_addr-instead-u32.patch rename to next-5.10/crypto-sun8i-ss-use-dma_addr-instead-u32.patch diff --git a/queue-5.10/crypto-tcrypt-fix-multibuffer-skcipher-speed-test-me.patch b/next-5.10/crypto-tcrypt-fix-multibuffer-skcipher-speed-test-me.patch similarity index 100% rename from queue-5.10/crypto-tcrypt-fix-multibuffer-skcipher-speed-test-me.patch rename to next-5.10/crypto-tcrypt-fix-multibuffer-skcipher-speed-test-me.patch diff --git a/next-5.10/csky-add-support-for-tif_notify_signal.patch b/next-5.10/csky-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..8d158001c28 --- /dev/null +++ b/next-5.10/csky-add-support-for-tif_notify_signal.patch @@ -0,0 +1,59 @@ +From 9f7678180bf6ff85fbd32bd83ff287a7eb4a6602 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 14:39:00 -0600 +Subject: csky: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit f3924d763c8af4c5d441b666c01f4de03ac9449e ] + +Wire up TIF_NOTIFY_SIGNAL handling for csky. + +Cc: linux-csky@vger.kernel.org +Acked-by: Guo Ren +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/csky/include/asm/thread_info.h | 5 ++++- + arch/csky/kernel/signal.c | 2 +- + 2 files changed, 5 insertions(+), 2 deletions(-) + +--- a/arch/csky/include/asm/thread_info.h ++++ b/arch/csky/include/asm/thread_info.h +@@ -64,6 +64,7 @@ static inline struct thread_info *curren + #define TIF_SYSCALL_TRACE 4 /* syscall trace active */ + #define TIF_SYSCALL_TRACEPOINT 5 /* syscall tracepoint instrumentation */ + #define TIF_SYSCALL_AUDIT 6 /* syscall auditing */ ++#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ + #define TIF_POLLING_NRFLAG 16 /* poll_idle() is TIF_NEED_RESCHED */ + #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ + #define TIF_RESTORE_SIGMASK 20 /* restore signal mask in do_signal() */ +@@ -75,6 +76,7 @@ static inline struct thread_info *curren + #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) + #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) + #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + #define _TIF_UPROBE (1 << TIF_UPROBE) + #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) + #define _TIF_MEMDIE (1 << TIF_MEMDIE) +@@ -82,7 +84,8 @@ static inline struct thread_info *curren + #define _TIF_SECCOMP (1 << TIF_SECCOMP) + + #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ +- _TIF_NOTIFY_RESUME | _TIF_UPROBE) ++ _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ ++ _TIF_NOTIFY_SIGNAL) + + #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ + _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP) +--- a/arch/csky/kernel/signal.c ++++ b/arch/csky/kernel/signal.c +@@ -261,7 +261,7 @@ asmlinkage void do_notify_resume(struct + uprobe_notify_resume(regs); + + /* Handle pending signal delivery */ +- if (thread_info_flags & _TIF_SIGPENDING) ++ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs); + + if (thread_info_flags & _TIF_NOTIFY_RESUME) { diff --git a/queue-5.10/cxl-fix-possible-null-ptr-deref-in-cxl_guest_init_af.patch b/next-5.10/cxl-fix-possible-null-ptr-deref-in-cxl_guest_init_af.patch similarity index 100% rename from queue-5.10/cxl-fix-possible-null-ptr-deref-in-cxl_guest_init_af.patch rename to next-5.10/cxl-fix-possible-null-ptr-deref-in-cxl_guest_init_af.patch diff --git a/queue-5.10/cxl-fix-possible-null-ptr-deref-in-cxl_pci_init_afu-.patch b/next-5.10/cxl-fix-possible-null-ptr-deref-in-cxl_pci_init_afu-.patch similarity index 100% rename from queue-5.10/cxl-fix-possible-null-ptr-deref-in-cxl_pci_init_afu-.patch rename to next-5.10/cxl-fix-possible-null-ptr-deref-in-cxl_pci_init_afu-.patch diff --git a/queue-5.10/cxl-fix-refcount-leak-in-cxl_calc_capp_routing.patch b/next-5.10/cxl-fix-refcount-leak-in-cxl_calc_capp_routing.patch similarity index 100% rename from queue-5.10/cxl-fix-refcount-leak-in-cxl_calc_capp_routing.patch rename to next-5.10/cxl-fix-refcount-leak-in-cxl_calc_capp_routing.patch diff --git a/queue-5.10/debugfs-fix-error-when-writing-negative-value-to-ato.patch b/next-5.10/debugfs-fix-error-when-writing-negative-value-to-ato.patch similarity index 100% rename from queue-5.10/debugfs-fix-error-when-writing-negative-value-to-ato.patch rename to next-5.10/debugfs-fix-error-when-writing-negative-value-to-ato.patch diff --git a/queue-5.10/dmaengine-idxd-fix-crc_val-field-for-completion-reco.patch b/next-5.10/dmaengine-idxd-fix-crc_val-field-for-completion-reco.patch similarity index 100% rename from queue-5.10/dmaengine-idxd-fix-crc_val-field-for-completion-reco.patch rename to next-5.10/dmaengine-idxd-fix-crc_val-field-for-completion-reco.patch diff --git a/queue-5.10/docs-fault-injection-fix-non-working-usage-of-negati.patch b/next-5.10/docs-fault-injection-fix-non-working-usage-of-negati.patch similarity index 100% rename from queue-5.10/docs-fault-injection-fix-non-working-usage-of-negati.patch rename to next-5.10/docs-fault-injection-fix-non-working-usage-of-negati.patch diff --git a/queue-5.10/drbd-fix-an-invalid-memory-access-caused-by-incorrec.patch b/next-5.10/drbd-fix-an-invalid-memory-access-caused-by-incorrec.patch similarity index 100% rename from queue-5.10/drbd-fix-an-invalid-memory-access-caused-by-incorrec.patch rename to next-5.10/drbd-fix-an-invalid-memory-access-caused-by-incorrec.patch diff --git a/queue-5.10/drivers-dio-fix-possible-memory-leak-in-dio_init.patch b/next-5.10/drivers-dio-fix-possible-memory-leak-in-dio_init.patch similarity index 100% rename from queue-5.10/drivers-dio-fix-possible-memory-leak-in-dio_init.patch rename to next-5.10/drivers-dio-fix-possible-memory-leak-in-dio_init.patch diff --git a/queue-5.10/drivers-mcb-fix-resource-leak-in-mcb_probe.patch b/next-5.10/drivers-mcb-fix-resource-leak-in-mcb_probe.patch similarity index 100% rename from queue-5.10/drivers-mcb-fix-resource-leak-in-mcb_probe.patch rename to next-5.10/drivers-mcb-fix-resource-leak-in-mcb_probe.patch diff --git a/queue-5.10/drivers-md-md-bitmap-check-the-return-value-of-md_bi.patch b/next-5.10/drivers-md-md-bitmap-check-the-return-value-of-md_bi.patch similarity index 100% rename from queue-5.10/drivers-md-md-bitmap-check-the-return-value-of-md_bi.patch rename to next-5.10/drivers-md-md-bitmap-check-the-return-value-of-md_bi.patch diff --git a/queue-5.10/drivers-net-qlcnic-fix-potential-memory-leak-in-qlcn.patch b/next-5.10/drivers-net-qlcnic-fix-potential-memory-leak-in-qlcn.patch similarity index 100% rename from queue-5.10/drivers-net-qlcnic-fix-potential-memory-leak-in-qlcn.patch rename to next-5.10/drivers-net-qlcnic-fix-potential-memory-leak-in-qlcn.patch diff --git a/queue-5.10/drivers-soc-ti-knav_qmss_queue-mark-knav_acc_firmwar.patch b/next-5.10/drivers-soc-ti-knav_qmss_queue-mark-knav_acc_firmwar.patch similarity index 100% rename from queue-5.10/drivers-soc-ti-knav_qmss_queue-mark-knav_acc_firmwar.patch rename to next-5.10/drivers-soc-ti-knav_qmss_queue-mark-knav_acc_firmwar.patch diff --git a/queue-5.10/drm-amd-display-fix-array-index-out-of-bound-error-i.patch b/next-5.10/drm-amd-display-fix-array-index-out-of-bound-error-i.patch similarity index 100% rename from queue-5.10/drm-amd-display-fix-array-index-out-of-bound-error-i.patch rename to next-5.10/drm-amd-display-fix-array-index-out-of-bound-error-i.patch diff --git a/queue-5.10/drm-amd-display-prevent-memory-leak.patch b/next-5.10/drm-amd-display-prevent-memory-leak.patch similarity index 100% rename from queue-5.10/drm-amd-display-prevent-memory-leak.patch rename to next-5.10/drm-amd-display-prevent-memory-leak.patch diff --git a/queue-5.10/drm-amd-pm-smu11-baco-is-supported-when-it-s-in-baco.patch b/next-5.10/drm-amd-pm-smu11-baco-is-supported-when-it-s-in-baco.patch similarity index 100% rename from queue-5.10/drm-amd-pm-smu11-baco-is-supported-when-it-s-in-baco.patch rename to next-5.10/drm-amd-pm-smu11-baco-is-supported-when-it-s-in-baco.patch diff --git a/queue-5.10/drm-amdgpu-fix-pci-device-refcount-leak-in-amdgpu_at.patch b/next-5.10/drm-amdgpu-fix-pci-device-refcount-leak-in-amdgpu_at.patch similarity index 100% rename from queue-5.10/drm-amdgpu-fix-pci-device-refcount-leak-in-amdgpu_at.patch rename to next-5.10/drm-amdgpu-fix-pci-device-refcount-leak-in-amdgpu_at.patch diff --git a/queue-5.10/drm-amdgpu-fix-pci-device-refcount-leak.patch b/next-5.10/drm-amdgpu-fix-pci-device-refcount-leak.patch similarity index 100% rename from queue-5.10/drm-amdgpu-fix-pci-device-refcount-leak.patch rename to next-5.10/drm-amdgpu-fix-pci-device-refcount-leak.patch diff --git a/queue-5.10/drm-amdgpu-fix-type-of-second-parameter-in-odn_edit_.patch b/next-5.10/drm-amdgpu-fix-type-of-second-parameter-in-odn_edit_.patch similarity index 100% rename from queue-5.10/drm-amdgpu-fix-type-of-second-parameter-in-odn_edit_.patch rename to next-5.10/drm-amdgpu-fix-type-of-second-parameter-in-odn_edit_.patch diff --git a/queue-5.10/drm-amdgpu-fix-type-of-second-parameter-in-trans_msg.patch b/next-5.10/drm-amdgpu-fix-type-of-second-parameter-in-trans_msg.patch similarity index 100% rename from queue-5.10/drm-amdgpu-fix-type-of-second-parameter-in-trans_msg.patch rename to next-5.10/drm-amdgpu-fix-type-of-second-parameter-in-trans_msg.patch diff --git a/queue-5.10/drm-bridge-adv7533-remove-dynamic-lane-switching-fro.patch b/next-5.10/drm-bridge-adv7533-remove-dynamic-lane-switching-fro.patch similarity index 100% rename from queue-5.10/drm-bridge-adv7533-remove-dynamic-lane-switching-fro.patch rename to next-5.10/drm-bridge-adv7533-remove-dynamic-lane-switching-fro.patch diff --git a/queue-5.10/drm-etnaviv-add-missing-quirks-for-gc300.patch b/next-5.10/drm-etnaviv-add-missing-quirks-for-gc300.patch similarity index 100% rename from queue-5.10/drm-etnaviv-add-missing-quirks-for-gc300.patch rename to next-5.10/drm-etnaviv-add-missing-quirks-for-gc300.patch diff --git a/queue-5.10/drm-fourcc-add-packed-10bit-yuv-4-2-0-format.patch b/next-5.10/drm-fourcc-add-packed-10bit-yuv-4-2-0-format.patch similarity index 100% rename from queue-5.10/drm-fourcc-add-packed-10bit-yuv-4-2-0-format.patch rename to next-5.10/drm-fourcc-add-packed-10bit-yuv-4-2-0-format.patch diff --git a/queue-5.10/drm-fourcc-fix-vsub-hsub-for-q410-and-q401.patch b/next-5.10/drm-fourcc-fix-vsub-hsub-for-q410-and-q401.patch similarity index 100% rename from queue-5.10/drm-fourcc-fix-vsub-hsub-for-q410-and-q401.patch rename to next-5.10/drm-fourcc-fix-vsub-hsub-for-q410-and-q401.patch diff --git a/queue-5.10/drm-fsl-dcu-fix-return-type-of-fsl_dcu_drm_connector.patch b/next-5.10/drm-fsl-dcu-fix-return-type-of-fsl_dcu_drm_connector.patch similarity index 100% rename from queue-5.10/drm-fsl-dcu-fix-return-type-of-fsl_dcu_drm_connector.patch rename to next-5.10/drm-fsl-dcu-fix-return-type-of-fsl_dcu_drm_connector.patch diff --git a/queue-5.10/drm-mediatek-modify-dpi-power-on-off-sequence.patch b/next-5.10/drm-mediatek-modify-dpi-power-on-off-sequence.patch similarity index 100% rename from queue-5.10/drm-mediatek-modify-dpi-power-on-off-sequence.patch rename to next-5.10/drm-mediatek-modify-dpi-power-on-off-sequence.patch diff --git a/queue-5.10/drm-msm-hdmi-drop-unused-gpio-support.patch b/next-5.10/drm-msm-hdmi-drop-unused-gpio-support.patch similarity index 100% rename from queue-5.10/drm-msm-hdmi-drop-unused-gpio-support.patch rename to next-5.10/drm-msm-hdmi-drop-unused-gpio-support.patch diff --git a/queue-5.10/drm-msm-hdmi-switch-to-drm_bridge_connector.patch b/next-5.10/drm-msm-hdmi-switch-to-drm_bridge_connector.patch similarity index 100% rename from queue-5.10/drm-msm-hdmi-switch-to-drm_bridge_connector.patch rename to next-5.10/drm-msm-hdmi-switch-to-drm_bridge_connector.patch diff --git a/queue-5.10/drm-msm-use-drm_mode_copy.patch b/next-5.10/drm-msm-use-drm_mode_copy.patch similarity index 100% rename from queue-5.10/drm-msm-use-drm_mode_copy.patch rename to next-5.10/drm-msm-use-drm_mode_copy.patch diff --git a/queue-5.10/drm-panel-panel-sitronix-st7701-remove-panel-on-dsi-.patch b/next-5.10/drm-panel-panel-sitronix-st7701-remove-panel-on-dsi-.patch similarity index 100% rename from queue-5.10/drm-panel-panel-sitronix-st7701-remove-panel-on-dsi-.patch rename to next-5.10/drm-panel-panel-sitronix-st7701-remove-panel-on-dsi-.patch diff --git a/queue-5.10/drm-radeon-add-the-missed-acpi_put_table-to-fix-memo.patch b/next-5.10/drm-radeon-add-the-missed-acpi_put_table-to-fix-memo.patch similarity index 100% rename from queue-5.10/drm-radeon-add-the-missed-acpi_put_table-to-fix-memo.patch rename to next-5.10/drm-radeon-add-the-missed-acpi_put_table-to-fix-memo.patch diff --git a/queue-5.10/drm-radeon-fix-pci-device-refcount-leak-in-radeon_at.patch b/next-5.10/drm-radeon-fix-pci-device-refcount-leak-in-radeon_at.patch similarity index 100% rename from queue-5.10/drm-radeon-fix-pci-device-refcount-leak-in-radeon_at.patch rename to next-5.10/drm-radeon-fix-pci-device-refcount-leak-in-radeon_at.patch diff --git a/queue-5.10/drm-rockchip-lvds-fix-pm-usage-counter-unbalance-in-.patch b/next-5.10/drm-rockchip-lvds-fix-pm-usage-counter-unbalance-in-.patch similarity index 100% rename from queue-5.10/drm-rockchip-lvds-fix-pm-usage-counter-unbalance-in-.patch rename to next-5.10/drm-rockchip-lvds-fix-pm-usage-counter-unbalance-in-.patch diff --git a/queue-5.10/drm-rockchip-use-drm_mode_copy.patch b/next-5.10/drm-rockchip-use-drm_mode_copy.patch similarity index 100% rename from queue-5.10/drm-rockchip-use-drm_mode_copy.patch rename to next-5.10/drm-rockchip-use-drm_mode_copy.patch diff --git a/queue-5.10/drm-sti-fix-return-type-of-sti_-dvo-hda-hdmi-_connec.patch b/next-5.10/drm-sti-fix-return-type-of-sti_-dvo-hda-hdmi-_connec.patch similarity index 100% rename from queue-5.10/drm-sti-fix-return-type-of-sti_-dvo-hda-hdmi-_connec.patch rename to next-5.10/drm-sti-fix-return-type-of-sti_-dvo-hda-hdmi-_connec.patch diff --git a/queue-5.10/drm-sti-use-drm_mode_copy.patch b/next-5.10/drm-sti-use-drm_mode_copy.patch similarity index 100% rename from queue-5.10/drm-sti-use-drm_mode_copy.patch rename to next-5.10/drm-sti-use-drm_mode_copy.patch diff --git a/queue-5.10/drm-tegra-add-missing-clk_disable_unprepare-in-tegra.patch b/next-5.10/drm-tegra-add-missing-clk_disable_unprepare-in-tegra.patch similarity index 100% rename from queue-5.10/drm-tegra-add-missing-clk_disable_unprepare-in-tegra.patch rename to next-5.10/drm-tegra-add-missing-clk_disable_unprepare-in-tegra.patch diff --git a/queue-5.10/edac-i10nm-fix-refcount-leak-in-pci_get_dev_wrapper.patch b/next-5.10/edac-i10nm-fix-refcount-leak-in-pci_get_dev_wrapper.patch similarity index 100% rename from queue-5.10/edac-i10nm-fix-refcount-leak-in-pci_get_dev_wrapper.patch rename to next-5.10/edac-i10nm-fix-refcount-leak-in-pci_get_dev_wrapper.patch diff --git a/next-5.10/entry-add-support-for-tif_notify_signal.patch b/next-5.10/entry-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..f234acc3e72 --- /dev/null +++ b/next-5.10/entry-add-support-for-tif_notify_signal.patch @@ -0,0 +1,234 @@ +From d02e3fa856542e39531f7f0910389b334cd9b1f3 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Mon, 26 Oct 2020 14:32:28 -0600 +Subject: entry: Add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 12db8b690010ccfadf9d0b49a1e1798e47dbbe1a ] + +Add TIF_NOTIFY_SIGNAL handling in the generic entry code, which if set, +will return true if signal_pending() is used in a wait loop. That causes an +exit of the loop so that notify_signal tracehooks can be run. If the wait +loop is currently inside a system call, the system call is restarted once +task_work has been processed. + +In preparation for only having arch_do_signal() handle syscall restarts if +_TIF_SIGPENDING isn't set, rename it to arch_do_signal_or_restart(). Pass +in a boolean that tells the architecture specific signal handler if it +should attempt to get a signal, or just process a potential syscall +restart. + +For !CONFIG_GENERIC_ENTRY archs, add the TIF_NOTIFY_SIGNAL handling to +get_signal(). This is done to minimize the needed architecture changes to +support this feature. + +Signed-off-by: Jens Axboe +Signed-off-by: Thomas Gleixner +Reviewed-by: Oleg Nesterov +Link: https://lore.kernel.org/r/20201026203230.386348-3-axboe@kernel.dk +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/kernel/signal.c | 4 ++-- + include/linux/entry-common.h | 11 ++++++++--- + include/linux/entry-kvm.h | 4 ++-- + include/linux/sched/signal.h | 11 ++++++++++- + include/linux/tracehook.h | 27 +++++++++++++++++++++++++++ + kernel/entry/common.c | 14 +++++++++++--- + kernel/entry/kvm.c | 3 +++ + kernel/signal.c | 14 ++++++++++++++ + 8 files changed, 77 insertions(+), 11 deletions(-) + +--- a/arch/x86/kernel/signal.c ++++ b/arch/x86/kernel/signal.c +@@ -798,11 +798,11 @@ static inline unsigned long get_nr_resta + * want to handle. Thus you cannot kill init even with a SIGKILL even by + * mistake. + */ +-void arch_do_signal(struct pt_regs *regs) ++void arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal) + { + struct ksignal ksig; + +- if (get_signal(&ksig)) { ++ if (has_signal && get_signal(&ksig)) { + /* Whee! Actually deliver the signal. */ + handle_signal(&ksig, regs); + return; +--- a/include/linux/entry-common.h ++++ b/include/linux/entry-common.h +@@ -37,6 +37,10 @@ + # define _TIF_UPROBE (0) + #endif + ++#ifndef _TIF_NOTIFY_SIGNAL ++# define _TIF_NOTIFY_SIGNAL (0) ++#endif ++ + /* + * TIF flags handled in syscall_enter_from_user_mode() + */ +@@ -69,7 +73,7 @@ + + #define EXIT_TO_USER_MODE_WORK \ + (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ +- _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | \ ++ _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | \ + ARCH_EXIT_TO_USER_MODE_WORK) + + /** +@@ -259,12 +263,13 @@ static __always_inline void arch_exit_to + #endif + + /** +- * arch_do_signal - Architecture specific signal delivery function ++ * arch_do_signal_or_restart - Architecture specific signal delivery function + * @regs: Pointer to currents pt_regs ++ * @has_signal: actual signal to handle + * + * Invoked from exit_to_user_mode_loop(). + */ +-void arch_do_signal(struct pt_regs *regs); ++void arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal); + + /** + * arch_syscall_exit_tracehook - Wrapper around tracehook_report_syscall_exit() +--- a/include/linux/entry-kvm.h ++++ b/include/linux/entry-kvm.h +@@ -11,8 +11,8 @@ + # define ARCH_XFER_TO_GUEST_MODE_WORK (0) + #endif + +-#define XFER_TO_GUEST_MODE_WORK \ +- (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ ++#define XFER_TO_GUEST_MODE_WORK \ ++ (_TIF_NEED_RESCHED | _TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL | \ + _TIF_NOTIFY_RESUME | ARCH_XFER_TO_GUEST_MODE_WORK) + + struct kvm_vcpu; +--- a/include/linux/sched/signal.h ++++ b/include/linux/sched/signal.h +@@ -361,6 +361,15 @@ static inline int task_sigpending(struct + + static inline int signal_pending(struct task_struct *p) + { ++#if defined(TIF_NOTIFY_SIGNAL) ++ /* ++ * TIF_NOTIFY_SIGNAL isn't really a signal, but it requires the same ++ * behavior in terms of ensuring that we break out of wait loops ++ * so that notify signal callbacks can be processed. ++ */ ++ if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL))) ++ return 1; ++#endif + return task_sigpending(p); + } + +@@ -508,7 +517,7 @@ extern int set_user_sigmask(const sigset + static inline void restore_saved_sigmask_unless(bool interrupted) + { + if (interrupted) +- WARN_ON(!test_thread_flag(TIF_SIGPENDING)); ++ WARN_ON(!signal_pending(current)); + else + restore_saved_sigmask(); + } +--- a/include/linux/tracehook.h ++++ b/include/linux/tracehook.h +@@ -198,4 +198,31 @@ static inline void tracehook_notify_resu + blkcg_maybe_throttle_current(); + } + ++/* ++ * called by exit_to_user_mode_loop() if ti_work & _TIF_NOTIFY_SIGNAL. This ++ * is currently used by TWA_SIGNAL based task_work, which requires breaking ++ * wait loops to ensure that task_work is noticed and run. ++ */ ++static inline void tracehook_notify_signal(void) ++{ ++#if defined(TIF_NOTIFY_SIGNAL) ++ clear_thread_flag(TIF_NOTIFY_SIGNAL); ++ smp_mb__after_atomic(); ++ if (current->task_works) ++ task_work_run(); ++#endif ++} ++ ++/* ++ * Called when we have work to process from exit_to_user_mode_loop() ++ */ ++static inline void set_notify_signal(struct task_struct *task) ++{ ++#if defined(TIF_NOTIFY_SIGNAL) ++ if (!test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) && ++ !wake_up_state(task, TASK_INTERRUPTIBLE)) ++ kick_process(task); ++#endif ++} ++ + #endif /* */ +--- a/kernel/entry/common.c ++++ b/kernel/entry/common.c +@@ -135,7 +135,15 @@ static __always_inline void exit_to_user + } + + /* Workaround to allow gradual conversion of architecture code */ +-void __weak arch_do_signal(struct pt_regs *regs) { } ++void __weak arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal) { } ++ ++static void handle_signal_work(struct pt_regs *regs, unsigned long ti_work) ++{ ++ if (ti_work & _TIF_NOTIFY_SIGNAL) ++ tracehook_notify_signal(); ++ ++ arch_do_signal_or_restart(regs, ti_work & _TIF_SIGPENDING); ++} + + static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, + unsigned long ti_work) +@@ -157,8 +165,8 @@ static unsigned long exit_to_user_mode_l + if (ti_work & _TIF_PATCH_PENDING) + klp_update_patch_state(current); + +- if (ti_work & _TIF_SIGPENDING) +- arch_do_signal(regs); ++ if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) ++ handle_signal_work(regs, ti_work); + + if (ti_work & _TIF_NOTIFY_RESUME) { + tracehook_notify_resume(regs); +--- a/kernel/entry/kvm.c ++++ b/kernel/entry/kvm.c +@@ -8,6 +8,9 @@ static int xfer_to_guest_mode_work(struc + do { + int ret; + ++ if (ti_work & _TIF_NOTIFY_SIGNAL) ++ tracehook_notify_signal(); ++ + if (ti_work & _TIF_SIGPENDING) { + kvm_handle_signal_exit(vcpu); + return -EINTR; +--- a/kernel/signal.c ++++ b/kernel/signal.c +@@ -2520,6 +2520,20 @@ bool get_signal(struct ksignal *ksig) + struct signal_struct *signal = current->signal; + int signr; + ++ /* ++ * For non-generic architectures, check for TIF_NOTIFY_SIGNAL so ++ * that the arch handlers don't all have to do it. If we get here ++ * without TIF_SIGPENDING, just exit after running signal work. ++ */ ++#ifdef TIF_NOTIFY_SIGNAL ++ if (!IS_ENABLED(CONFIG_GENERIC_ENTRY)) { ++ if (test_thread_flag(TIF_NOTIFY_SIGNAL)) ++ tracehook_notify_signal(); ++ if (!task_sigpending(current)) ++ return false; ++ } ++#endif ++ + if (unlikely(uprobe_deny_signal())) + return false; + diff --git a/next-5.10/entry-kvm-exit-to-user-mode-when-tif_notify_signal-is-set.patch b/next-5.10/entry-kvm-exit-to-user-mode-when-tif_notify_signal-is-set.patch new file mode 100644 index 00000000000..7550ea6a631 --- /dev/null +++ b/next-5.10/entry-kvm-exit-to-user-mode-when-tif_notify_signal-is-set.patch @@ -0,0 +1,51 @@ +From 963feaebdd2ec09424d8d7dda014c4859cce19c5 Mon Sep 17 00:00:00 2001 +From: Seth Forshee +Date: Wed, 4 May 2022 13:08:40 -0500 +Subject: entry/kvm: Exit to user mode when TIF_NOTIFY_SIGNAL is set + +From: Seth Forshee + +[ Upstream commit 3e684903a8574ffc9475fdf13c4780a7adb506ad ] + +A livepatch transition may stall indefinitely when a kvm vCPU is heavily +loaded. To the host, the vCPU task is a user thread which is spending a +very long time in the ioctl(KVM_RUN) syscall. During livepatch +transition, set_notify_signal() will be called on such tasks to +interrupt the syscall so that the task can be transitioned. This +interrupts guest execution, but when xfer_to_guest_mode_work() sees that +TIF_NOTIFY_SIGNAL is set but not TIF_SIGPENDING it concludes that an +exit to user mode is unnecessary, and guest execution is resumed without +transitioning the task for the livepatch. + +This handling of TIF_NOTIFY_SIGNAL is incorrect, as set_notify_signal() +is expected to break tasks out of interruptible kernel loops and cause +them to return to userspace. Change xfer_to_guest_mode_work() to handle +TIF_NOTIFY_SIGNAL the same as TIF_SIGPENDING, signaling to the vCPU run +loop that an exit to userpsace is needed. Any pending task_work will be +run when get_signal() is called from exit_to_user_mode_loop(), so there +is no longer any need to run task work from xfer_to_guest_mode_work(). + +Suggested-by: "Eric W. Biederman" +Cc: Petr Mladek +Signed-off-by: Seth Forshee +Message-Id: <20220504180840.2907296-1-sforshee@digitalocean.com> +Signed-off-by: Paolo Bonzini +Signed-off-by: Greg Kroah-Hartman +--- + kernel/entry/kvm.c | 5 +---- + 1 file changed, 1 insertion(+), 4 deletions(-) + +--- a/kernel/entry/kvm.c ++++ b/kernel/entry/kvm.c +@@ -8,10 +8,7 @@ static int xfer_to_guest_mode_work(struc + do { + int ret; + +- if (ti_work & _TIF_NOTIFY_SIGNAL) +- tracehook_notify_signal(); +- +- if (ti_work & _TIF_SIGPENDING) { ++ if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) { + kvm_handle_signal_exit(vcpu); + return -EINTR; + } diff --git a/queue-5.10/ethernet-s2io-don-t-call-dev_kfree_skb-under-spin_lo.patch b/next-5.10/ethernet-s2io-don-t-call-dev_kfree_skb-under-spin_lo.patch similarity index 100% rename from queue-5.10/ethernet-s2io-don-t-call-dev_kfree_skb-under-spin_lo.patch rename to next-5.10/ethernet-s2io-don-t-call-dev_kfree_skb-under-spin_lo.patch diff --git a/queue-5.10/ethtool-avoiding-integer-overflow-in-ethtool_phys_id.patch b/next-5.10/ethtool-avoiding-integer-overflow-in-ethtool_phys_id.patch similarity index 100% rename from queue-5.10/ethtool-avoiding-integer-overflow-in-ethtool_phys_id.patch rename to next-5.10/ethtool-avoiding-integer-overflow-in-ethtool_phys_id.patch diff --git a/queue-5.10/eventfd-change-int-to-__u64-in-eventfd_signal-ifndef.patch b/next-5.10/eventfd-change-int-to-__u64-in-eventfd_signal-ifndef.patch similarity index 100% rename from queue-5.10/eventfd-change-int-to-__u64-in-eventfd_signal-ifndef.patch rename to next-5.10/eventfd-change-int-to-__u64-in-eventfd_signal-ifndef.patch diff --git a/next-5.10/eventfd-provide-a-eventfd_signal_mask-helper.patch b/next-5.10/eventfd-provide-a-eventfd_signal_mask-helper.patch new file mode 100644 index 00000000000..10b94de1ffe --- /dev/null +++ b/next-5.10/eventfd-provide-a-eventfd_signal_mask-helper.patch @@ -0,0 +1,113 @@ +From 53270319f184c35499cf2ffeac8a75078b94934a Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Sun, 20 Nov 2022 10:13:44 -0700 +Subject: eventfd: provide a eventfd_signal_mask() helper + +From: Jens Axboe + +[ Upstream commit 03e02acda8e267a8183e1e0ed289ff1ef9cd7ed8 ] + +This is identical to eventfd_signal(), but it allows the caller to pass +in a mask to be used for the poll wakeup key. The use case is avoiding +repeated multishot triggers if we have a dependency between eventfd and +io_uring. + +If we setup an eventfd context and register that as the io_uring eventfd, +and at the same time queue a multishot poll request for the eventfd +context, then any CQE posted will repeatedly trigger the multishot request +until it terminates when the CQ ring overflows. + +In preparation for io_uring detecting this circular dependency, add the +mentioned helper so that io_uring can pass in EPOLL_URING as part of the +poll wakeup key. + +Cc: stable@vger.kernel.org # 6.0 +[axboe: fold in !CONFIG_EVENTFD fix from Zhang Qilong] +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + fs/eventfd.c | 37 +++++++++++++++++++++---------------- + include/linux/eventfd.h | 7 +++++++ + 2 files changed, 28 insertions(+), 16 deletions(-) + +--- a/fs/eventfd.c ++++ b/fs/eventfd.c +@@ -45,21 +45,7 @@ struct eventfd_ctx { + int id; + }; + +-/** +- * eventfd_signal - Adds @n to the eventfd counter. +- * @ctx: [in] Pointer to the eventfd context. +- * @n: [in] Value of the counter to be added to the eventfd internal counter. +- * The value cannot be negative. +- * +- * This function is supposed to be called by the kernel in paths that do not +- * allow sleeping. In this function we allow the counter to reach the ULLONG_MAX +- * value, and we signal this as overflow condition by returning a EPOLLERR +- * to poll(2). +- * +- * Returns the amount by which the counter was incremented. This will be less +- * than @n if the counter has overflowed. +- */ +-__u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n) ++__u64 eventfd_signal_mask(struct eventfd_ctx *ctx, __u64 n, unsigned mask) + { + unsigned long flags; + +@@ -80,12 +66,31 @@ __u64 eventfd_signal(struct eventfd_ctx + n = ULLONG_MAX - ctx->count; + ctx->count += n; + if (waitqueue_active(&ctx->wqh)) +- wake_up_locked_poll(&ctx->wqh, EPOLLIN); ++ wake_up_locked_poll(&ctx->wqh, EPOLLIN | mask); + this_cpu_dec(eventfd_wake_count); + spin_unlock_irqrestore(&ctx->wqh.lock, flags); + + return n; + } ++ ++/** ++ * eventfd_signal - Adds @n to the eventfd counter. ++ * @ctx: [in] Pointer to the eventfd context. ++ * @n: [in] Value of the counter to be added to the eventfd internal counter. ++ * The value cannot be negative. ++ * ++ * This function is supposed to be called by the kernel in paths that do not ++ * allow sleeping. In this function we allow the counter to reach the ULLONG_MAX ++ * value, and we signal this as overflow condition by returning a EPOLLERR ++ * to poll(2). ++ * ++ * Returns the amount by which the counter was incremented. This will be less ++ * than @n if the counter has overflowed. ++ */ ++__u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n) ++{ ++ return eventfd_signal_mask(ctx, n, 0); ++} + EXPORT_SYMBOL_GPL(eventfd_signal); + + static void eventfd_free_ctx(struct eventfd_ctx *ctx) +--- a/include/linux/eventfd.h ++++ b/include/linux/eventfd.h +@@ -39,6 +39,7 @@ struct file *eventfd_fget(int fd); + struct eventfd_ctx *eventfd_ctx_fdget(int fd); + struct eventfd_ctx *eventfd_ctx_fileget(struct file *file); + __u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n); ++__u64 eventfd_signal_mask(struct eventfd_ctx *ctx, __u64 n, unsigned mask); + int eventfd_ctx_remove_wait_queue(struct eventfd_ctx *ctx, wait_queue_entry_t *wait, + __u64 *cnt); + +@@ -65,6 +66,12 @@ static inline int eventfd_signal(struct + { + return -ENOSYS; + } ++ ++static inline int eventfd_signal_mask(struct eventfd_ctx *ctx, __u64 n, ++ unsigned mask) ++{ ++ return -ENOSYS; ++} + + static inline void eventfd_ctx_put(struct eventfd_ctx *ctx) + { diff --git a/next-5.10/eventpoll-add-epoll_uring_wake-poll-wakeup-flag.patch b/next-5.10/eventpoll-add-epoll_uring_wake-poll-wakeup-flag.patch new file mode 100644 index 00000000000..a0b3e63e48d --- /dev/null +++ b/next-5.10/eventpoll-add-epoll_uring_wake-poll-wakeup-flag.patch @@ -0,0 +1,112 @@ +From 5d0d6c3d70a751ae214e471989fc729956c6fccb Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Sun, 20 Nov 2022 10:10:53 -0700 +Subject: eventpoll: add EPOLL_URING_WAKE poll wakeup flag + +From: Jens Axboe + +[ Upstream commit caf1aeaffc3b09649a56769e559333ae2c4f1802 ] + +We can have dependencies between epoll and io_uring. Consider an epoll +context, identified by the epfd file descriptor, and an io_uring file +descriptor identified by iofd. If we add iofd to the epfd context, and +arm a multishot poll request for epfd with iofd, then the multishot +poll request will repeatedly trigger and generate events until terminated +by CQ ring overflow. This isn't a desired behavior. + +Add EPOLL_URING so that io_uring can pass it in as part of the poll wakeup +key, and io_uring can check for that to detect a potential recursive +invocation. + +Cc: stable@vger.kernel.org # 6.0 +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + fs/eventpoll.c | 18 ++++++++++-------- + include/uapi/linux/eventpoll.h | 6 ++++++ + 2 files changed, 16 insertions(+), 8 deletions(-) + +--- a/fs/eventpoll.c ++++ b/fs/eventpoll.c +@@ -548,7 +548,8 @@ out_unlock: + */ + #ifdef CONFIG_DEBUG_LOCK_ALLOC + +-static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi) ++static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi, ++ unsigned pollflags) + { + struct eventpoll *ep_src; + unsigned long flags; +@@ -579,16 +580,17 @@ static void ep_poll_safewake(struct even + } + spin_lock_irqsave_nested(&ep->poll_wait.lock, flags, nests); + ep->nests = nests + 1; +- wake_up_locked_poll(&ep->poll_wait, EPOLLIN); ++ wake_up_locked_poll(&ep->poll_wait, EPOLLIN | pollflags); + ep->nests = 0; + spin_unlock_irqrestore(&ep->poll_wait.lock, flags); + } + + #else + +-static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi) ++static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi, ++ unsigned pollflags) + { +- wake_up_poll(&ep->poll_wait, EPOLLIN); ++ wake_up_poll(&ep->poll_wait, EPOLLIN | pollflags); + } + + #endif +@@ -815,7 +817,7 @@ static void ep_free(struct eventpoll *ep + + /* We need to release all tasks waiting for these file */ + if (waitqueue_active(&ep->poll_wait)) +- ep_poll_safewake(ep, NULL); ++ ep_poll_safewake(ep, NULL, 0); + + /* + * We need to lock this because we could be hit by +@@ -1284,7 +1286,7 @@ out_unlock: + + /* We have to call this outside the lock */ + if (pwake) +- ep_poll_safewake(ep, epi); ++ ep_poll_safewake(ep, epi, pollflags & EPOLL_URING_WAKE); + + if (!(epi->event.events & EPOLLEXCLUSIVE)) + ewake = 1; +@@ -1589,7 +1591,7 @@ static int ep_insert(struct eventpoll *e + + /* We have to call this outside the lock */ + if (pwake) +- ep_poll_safewake(ep, NULL); ++ ep_poll_safewake(ep, NULL, 0); + + return 0; + +@@ -1692,7 +1694,7 @@ static int ep_modify(struct eventpoll *e + + /* We have to call this outside the lock */ + if (pwake) +- ep_poll_safewake(ep, NULL); ++ ep_poll_safewake(ep, NULL, 0); + + return 0; + } +--- a/include/uapi/linux/eventpoll.h ++++ b/include/uapi/linux/eventpoll.h +@@ -41,6 +41,12 @@ + #define EPOLLMSG (__force __poll_t)0x00000400 + #define EPOLLRDHUP (__force __poll_t)0x00002000 + ++/* ++ * Internal flag - wakeup generated by io_uring, used to detect recursion back ++ * into the io_uring poll handler. ++ */ ++#define EPOLL_URING_WAKE ((__force __poll_t)(1U << 27)) ++ + /* Set exclusive wakeup mode for the target file descriptor */ + #define EPOLLEXCLUSIVE ((__force __poll_t)(1U << 28)) + diff --git a/queue-5.10/f2fs-avoid-victim-selection-from-previous-victim-sec.patch b/next-5.10/f2fs-avoid-victim-selection-from-previous-victim-sec.patch similarity index 100% rename from queue-5.10/f2fs-avoid-victim-selection-from-previous-victim-sec.patch rename to next-5.10/f2fs-avoid-victim-selection-from-previous-victim-sec.patch diff --git a/queue-5.10/f2fs-fix-normal-discard-process.patch b/next-5.10/f2fs-fix-normal-discard-process.patch similarity index 100% rename from queue-5.10/f2fs-fix-normal-discard-process.patch rename to next-5.10/f2fs-fix-normal-discard-process.patch diff --git a/queue-5.10/f2fs-fix-the-race-condition-of-resize-flag-between-r.patch b/next-5.10/f2fs-fix-the-race-condition-of-resize-flag-between-r.patch similarity index 100% rename from queue-5.10/f2fs-fix-the-race-condition-of-resize-flag-between-r.patch rename to next-5.10/f2fs-fix-the-race-condition-of-resize-flag-between-r.patch diff --git a/queue-5.10/f2fs-should-put-a-page-when-checking-the-summary-info.patch b/next-5.10/f2fs-should-put-a-page-when-checking-the-summary-info.patch similarity index 100% rename from queue-5.10/f2fs-should-put-a-page-when-checking-the-summary-info.patch rename to next-5.10/f2fs-should-put-a-page-when-checking-the-summary-info.patch diff --git a/queue-5.10/fbdev-pm2fb-fix-missing-pci_disable_device.patch b/next-5.10/fbdev-pm2fb-fix-missing-pci_disable_device.patch similarity index 100% rename from queue-5.10/fbdev-pm2fb-fix-missing-pci_disable_device.patch rename to next-5.10/fbdev-pm2fb-fix-missing-pci_disable_device.patch diff --git a/queue-5.10/fbdev-ssd1307fb-drop-optional-dependency.patch b/next-5.10/fbdev-ssd1307fb-drop-optional-dependency.patch similarity index 100% rename from queue-5.10/fbdev-ssd1307fb-drop-optional-dependency.patch rename to next-5.10/fbdev-ssd1307fb-drop-optional-dependency.patch diff --git a/queue-5.10/fbdev-uvesafb-fixes-an-error-handling-path-in-uvesaf.patch b/next-5.10/fbdev-uvesafb-fixes-an-error-handling-path-in-uvesaf.patch similarity index 100% rename from queue-5.10/fbdev-uvesafb-fixes-an-error-handling-path-in-uvesaf.patch rename to next-5.10/fbdev-uvesafb-fixes-an-error-handling-path-in-uvesaf.patch diff --git a/queue-5.10/fbdev-vermilion-decrease-reference-count-in-error-pa.patch b/next-5.10/fbdev-vermilion-decrease-reference-count-in-error-pa.patch similarity index 100% rename from queue-5.10/fbdev-vermilion-decrease-reference-count-in-error-pa.patch rename to next-5.10/fbdev-vermilion-decrease-reference-count-in-error-pa.patch diff --git a/queue-5.10/fbdev-via-fix-error-in-via_core_init.patch b/next-5.10/fbdev-via-fix-error-in-via_core_init.patch similarity index 100% rename from queue-5.10/fbdev-via-fix-error-in-via_core_init.patch rename to next-5.10/fbdev-via-fix-error-in-via_core_init.patch diff --git a/next-5.10/file-rename-__close_fd_get_file-close_fd_get_file.patch b/next-5.10/file-rename-__close_fd_get_file-close_fd_get_file.patch new file mode 100644 index 00000000000..6ab08626cf4 --- /dev/null +++ b/next-5.10/file-rename-__close_fd_get_file-close_fd_get_file.patch @@ -0,0 +1,80 @@ +From ba51a050af49c1df47fe79687d5f14256620d508 Mon Sep 17 00:00:00 2001 +From: "Eric W. Biederman" +Date: Fri, 20 Nov 2020 17:14:40 -0600 +Subject: file: Rename __close_fd_get_file close_fd_get_file + +From: "Eric W. Biederman" + +[ Upstream commit 9fe83c43e71cdb8e5b9520bcb98706a2b3c680c8 ] + +The function close_fd_get_file is explicitly a variant of +__close_fd[1]. Now that __close_fd has been renamed close_fd, rename +close_fd_get_file to be consistent with close_fd. + +When __alloc_fd, __close_fd and __fd_install were introduced the +double underscore indicated that the function took a struct +files_struct parameter. The function __close_fd_get_file never has so +the naming has always been inconsistent. This just cleans things up +so there are not any lingering mentions or references __close_fd left +in the code. + +[1] 80cd795630d6 ("binder: fix use-after-free due to ksys_close() during fdget()") +Link: https://lkml.kernel.org/r/20201120231441.29911-23-ebiederm@xmission.com +Signed-off-by: Eric W. Biederman +Signed-off-by: Greg Kroah-Hartman +--- + drivers/android/binder.c | 2 +- + fs/file.c | 4 ++-- + fs/io_uring.c | 2 +- + include/linux/fdtable.h | 2 +- + 4 files changed, 5 insertions(+), 5 deletions(-) + +--- a/drivers/android/binder.c ++++ b/drivers/android/binder.c +@@ -2255,7 +2255,7 @@ static void binder_deferred_fd_close(int + if (!twcb) + return; + init_task_work(&twcb->twork, binder_do_fd_close); +- __close_fd_get_file(fd, &twcb->file); ++ close_fd_get_file(fd, &twcb->file); + if (twcb->file) { + filp_close(twcb->file, current->files); + task_work_add(current, &twcb->twork, TWA_RESUME); +--- a/fs/file.c ++++ b/fs/file.c +@@ -780,11 +780,11 @@ int __close_range(unsigned fd, unsigned + } + + /* +- * variant of __close_fd that gets a ref on the file for later fput. ++ * variant of close_fd that gets a ref on the file for later fput. + * The caller must ensure that filp_close() called on the file, and then + * an fput(). + */ +-int __close_fd_get_file(unsigned int fd, struct file **res) ++int close_fd_get_file(unsigned int fd, struct file **res) + { + struct files_struct *files = current->files; + struct file *file; +--- a/fs/io_uring.c ++++ b/fs/io_uring.c +@@ -4270,7 +4270,7 @@ static int io_close(struct io_kiocb *req + + /* might be already done during nonblock submission */ + if (!close->put_file) { +- ret = __close_fd_get_file(close->fd, &close->put_file); ++ ret = close_fd_get_file(close->fd, &close->put_file); + if (ret < 0) + return (ret == -ENOENT) ? -EBADF : ret; + } +--- a/include/linux/fdtable.h ++++ b/include/linux/fdtable.h +@@ -123,7 +123,7 @@ extern void __fd_install(struct files_st + extern int __close_fd(struct files_struct *files, + unsigned int fd); + extern int __close_range(unsigned int fd, unsigned int max_fd, unsigned int flags); +-extern int __close_fd_get_file(unsigned int fd, struct file **res); ++extern int close_fd_get_file(unsigned int fd, struct file **res); + extern int unshare_fd(unsigned long unshare_flags, unsigned int max_fds, + struct files_struct **new_fdp); + diff --git a/queue-5.10/firmware-raspberrypi-fix-possible-memory-leak-in-rpi.patch b/next-5.10/firmware-raspberrypi-fix-possible-memory-leak-in-rpi.patch similarity index 100% rename from queue-5.10/firmware-raspberrypi-fix-possible-memory-leak-in-rpi.patch rename to next-5.10/firmware-raspberrypi-fix-possible-memory-leak-in-rpi.patch diff --git a/next-5.10/fix-handling-of-nd-depth-on-lookup_cached-failures-in-try_to_unlazy.patch b/next-5.10/fix-handling-of-nd-depth-on-lookup_cached-failures-in-try_to_unlazy.patch new file mode 100644 index 00000000000..f576a7a414c --- /dev/null +++ b/next-5.10/fix-handling-of-nd-depth-on-lookup_cached-failures-in-try_to_unlazy.patch @@ -0,0 +1,64 @@ +From b93a6e389bdc72d423e666cbb5475ca7c7c4deb9 Mon Sep 17 00:00:00 2001 +From: Al Viro +Date: Mon, 15 Feb 2021 12:03:23 -0500 +Subject: fix handling of nd->depth on LOOKUP_CACHED failures in try_to_unlazy* + +From: Al Viro + +[ Upstream commit eacd9aa8cedeb412842c7b339adbaa0477fdd5ad ] + +After switching to non-RCU mode, we want nd->depth to match the number +of entries in nd->stack[] that need eventual path_put(). +legitimize_links() takes care of that on failures; unfortunately, +failure exits added for LOOKUP_CACHED do not. + +We could add the logics for that into those failure exits, both in +try_to_unlazy() and in try_to_unlazy_next(), but since both checks +are immediately followed by legitimize_links() and there's no calls +of legitimize_links() other than those two... It's easier to +move the check (and required handling of nd->depth on failure) into +legitimize_links() itself. + +[caught by Jens: ... and since we are zeroing ->depth here, we need +to do drop_links() first] + +Fixes: 6c6ec2b0a3e0 "fs: add support for LOOKUP_CACHED" +Tested-by: Jens Axboe +Signed-off-by: Al Viro +Signed-off-by: Greg Kroah-Hartman +--- + fs/namei.c | 9 +++++---- + 1 file changed, 5 insertions(+), 4 deletions(-) + +--- a/fs/namei.c ++++ b/fs/namei.c +@@ -630,6 +630,11 @@ static inline bool legitimize_path(struc + static bool legitimize_links(struct nameidata *nd) + { + int i; ++ if (unlikely(nd->flags & LOOKUP_CACHED)) { ++ drop_links(nd); ++ nd->depth = 0; ++ return false; ++ } + for (i = 0; i < nd->depth; i++) { + struct saved *last = nd->stack + i; + if (unlikely(!legitimize_path(nd, &last->link, last->seq))) { +@@ -686,8 +691,6 @@ static bool try_to_unlazy(struct nameida + BUG_ON(!(nd->flags & LOOKUP_RCU)); + + nd->flags &= ~LOOKUP_RCU; +- if (nd->flags & LOOKUP_CACHED) +- goto out1; + if (unlikely(!legitimize_links(nd))) + goto out1; + if (unlikely(!legitimize_path(nd, &nd->path, nd->seq))) +@@ -724,8 +727,6 @@ static bool try_to_unlazy_next(struct na + BUG_ON(!(nd->flags & LOOKUP_RCU)); + + nd->flags &= ~LOOKUP_RCU; +- if (nd->flags & LOOKUP_CACHED) +- goto out2; + if (unlikely(!legitimize_links(nd))) + goto out2; + if (unlikely(!legitimize_mnt(nd->path.mnt, nd->m_seq))) diff --git a/next-5.10/fs-add-support-for-lookup_cached.patch b/next-5.10/fs-add-support-for-lookup_cached.patch new file mode 100644 index 00000000000..1c68635dfeb --- /dev/null +++ b/next-5.10/fs-add-support-for-lookup_cached.patch @@ -0,0 +1,76 @@ +From 53e825b10aa8f037c19c51997f5f8402d046d883 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Thu, 17 Dec 2020 09:19:09 -0700 +Subject: fs: add support for LOOKUP_CACHED + +From: Jens Axboe + +[ Upstream commit 6c6ec2b0a3e0381d886d531bd1471dfdb1509237 ] + +io_uring always punts opens to async context, since there's no control +over whether the lookup blocks or not. Add LOOKUP_CACHED to support +just doing the fast RCU based lookups, which we know will not block. If +we can do a cached path resolution of the filename, then we don't have +to always punt lookups for a worker. + +During path resolution, we always do LOOKUP_RCU first. If that fails and +we terminate LOOKUP_RCU, then fail a LOOKUP_CACHED attempt as well. + +Cc: Al Viro +Signed-off-by: Jens Axboe +Signed-off-by: Al Viro +Signed-off-by: Greg Kroah-Hartman +--- + fs/namei.c | 9 +++++++++ + include/linux/namei.h | 1 + + 2 files changed, 10 insertions(+) + +--- a/fs/namei.c ++++ b/fs/namei.c +@@ -686,6 +686,8 @@ static bool try_to_unlazy(struct nameida + BUG_ON(!(nd->flags & LOOKUP_RCU)); + + nd->flags &= ~LOOKUP_RCU; ++ if (nd->flags & LOOKUP_CACHED) ++ goto out1; + if (unlikely(!legitimize_links(nd))) + goto out1; + if (unlikely(!legitimize_path(nd, &nd->path, nd->seq))) +@@ -722,6 +724,8 @@ static bool try_to_unlazy_next(struct na + BUG_ON(!(nd->flags & LOOKUP_RCU)); + + nd->flags &= ~LOOKUP_RCU; ++ if (nd->flags & LOOKUP_CACHED) ++ goto out2; + if (unlikely(!legitimize_links(nd))) + goto out2; + if (unlikely(!legitimize_mnt(nd->path.mnt, nd->m_seq))) +@@ -792,6 +796,7 @@ static int complete_walk(struct nameidat + */ + if (!(nd->flags & (LOOKUP_ROOT | LOOKUP_IS_SCOPED))) + nd->root.mnt = NULL; ++ nd->flags &= ~LOOKUP_CACHED; + if (!try_to_unlazy(nd)) + return -ECHILD; + } +@@ -2204,6 +2209,10 @@ static const char *path_init(struct name + int error; + const char *s = nd->name->name; + ++ /* LOOKUP_CACHED requires RCU, ask caller to retry */ ++ if ((flags & (LOOKUP_RCU | LOOKUP_CACHED)) == LOOKUP_CACHED) ++ return ERR_PTR(-EAGAIN); ++ + if (!*s) + flags &= ~LOOKUP_RCU; + if (flags & LOOKUP_RCU) +--- a/include/linux/namei.h ++++ b/include/linux/namei.h +@@ -46,6 +46,7 @@ enum {LAST_NORM, LAST_ROOT, LAST_DOT, LA + #define LOOKUP_NO_XDEV 0x040000 /* No mountpoint crossing. */ + #define LOOKUP_BENEATH 0x080000 /* No escaping from starting point. */ + #define LOOKUP_IN_ROOT 0x100000 /* Treat dirfd as fs root. */ ++#define LOOKUP_CACHED 0x200000 /* Only do cached lookup */ + /* LOOKUP_* flags which do scope-related checks based on the dirfd. */ + #define LOOKUP_IS_SCOPED (LOOKUP_BENEATH | LOOKUP_IN_ROOT) + diff --git a/queue-5.10/fs-don-t-audit-the-capability-check-in-simple_xattr_.patch b/next-5.10/fs-don-t-audit-the-capability-check-in-simple_xattr_.patch similarity index 100% rename from queue-5.10/fs-don-t-audit-the-capability-check-in-simple_xattr_.patch rename to next-5.10/fs-don-t-audit-the-capability-check-in-simple_xattr_.patch diff --git a/next-5.10/fs-expose-lookup_cached-through-openat2-resolve_cached.patch b/next-5.10/fs-expose-lookup_cached-through-openat2-resolve_cached.patch new file mode 100644 index 00000000000..e9297d32c35 --- /dev/null +++ b/next-5.10/fs-expose-lookup_cached-through-openat2-resolve_cached.patch @@ -0,0 +1,65 @@ +From cb9294e007feacca1a5849773460dde71e50cdde Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Thu, 17 Dec 2020 09:19:10 -0700 +Subject: fs: expose LOOKUP_CACHED through openat2() RESOLVE_CACHED + +From: Jens Axboe + +[ Upstream commit 99668f618062816ca7ba639b007eb145b9d3d41e ] + +Now that we support non-blocking path resolution internally, expose it +via openat2() in the struct open_how ->resolve flags. This allows +applications using openat2() to limit path resolution to the extent that +it is already cached. + +If the lookup cannot be satisfied in a non-blocking manner, openat2(2) +will return -1/-EAGAIN. + +Cc: Al Viro +Signed-off-by: Jens Axboe +Signed-off-by: Al Viro +Signed-off-by: Greg Kroah-Hartman +--- + fs/open.c | 6 ++++++ + include/linux/fcntl.h | 2 +- + include/uapi/linux/openat2.h | 4 ++++ + 3 files changed, 11 insertions(+), 1 deletion(-) + +--- a/fs/open.c ++++ b/fs/open.c +@@ -1099,6 +1099,12 @@ inline int build_open_flags(const struct + lookup_flags |= LOOKUP_BENEATH; + if (how->resolve & RESOLVE_IN_ROOT) + lookup_flags |= LOOKUP_IN_ROOT; ++ if (how->resolve & RESOLVE_CACHED) { ++ /* Don't bother even trying for create/truncate/tmpfile open */ ++ if (flags & (O_TRUNC | O_CREAT | O_TMPFILE)) ++ return -EAGAIN; ++ lookup_flags |= LOOKUP_CACHED; ++ } + + op->lookup_flags = lookup_flags; + return 0; +--- a/include/linux/fcntl.h ++++ b/include/linux/fcntl.h +@@ -19,7 +19,7 @@ + /* List of all valid flags for the how->resolve argument: */ + #define VALID_RESOLVE_FLAGS \ + (RESOLVE_NO_XDEV | RESOLVE_NO_MAGICLINKS | RESOLVE_NO_SYMLINKS | \ +- RESOLVE_BENEATH | RESOLVE_IN_ROOT) ++ RESOLVE_BENEATH | RESOLVE_IN_ROOT | RESOLVE_CACHED) + + /* List of all open_how "versions". */ + #define OPEN_HOW_SIZE_VER0 24 /* sizeof first published struct */ +--- a/include/uapi/linux/openat2.h ++++ b/include/uapi/linux/openat2.h +@@ -35,5 +35,9 @@ struct open_how { + #define RESOLVE_IN_ROOT 0x10 /* Make all jumps to "/" and ".." + be scoped inside the dirfd + (similar to chroot(2)). */ ++#define RESOLVE_CACHED 0x20 /* Only complete if resolution can be ++ completed through cached lookup. May ++ return -EAGAIN if that's not ++ possible. */ + + #endif /* _UAPI_LINUX_OPENAT2_H */ diff --git a/queue-5.10/fs-jfs-fix-shift-out-of-bounds-in-dballocag.patch b/next-5.10/fs-jfs-fix-shift-out-of-bounds-in-dballocag.patch similarity index 100% rename from queue-5.10/fs-jfs-fix-shift-out-of-bounds-in-dballocag.patch rename to next-5.10/fs-jfs-fix-shift-out-of-bounds-in-dballocag.patch diff --git a/queue-5.10/fs-jfs-fix-shift-out-of-bounds-in-dbdiscardag.patch b/next-5.10/fs-jfs-fix-shift-out-of-bounds-in-dbdiscardag.patch similarity index 100% rename from queue-5.10/fs-jfs-fix-shift-out-of-bounds-in-dbdiscardag.patch rename to next-5.10/fs-jfs-fix-shift-out-of-bounds-in-dbdiscardag.patch diff --git a/next-5.10/fs-make-do_renameat2-take-struct-filename.patch b/next-5.10/fs-make-do_renameat2-take-struct-filename.patch new file mode 100644 index 00000000000..e24e860c508 --- /dev/null +++ b/next-5.10/fs-make-do_renameat2-take-struct-filename.patch @@ -0,0 +1,134 @@ +From aa359e48cc999513b0167703009d3db363818540 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Sat, 26 Sep 2020 17:20:17 -0600 +Subject: fs: make do_renameat2() take struct filename + +From: Jens Axboe + +[ Upstream commit e886663cfd029b64a1d8da7efae7014526d884e9 ] + +Pass in the struct filename pointers instead of the user string, and +update the three callers to do the same. + +This behaves like do_unlinkat(), which also takes a filename struct and +puts it when it is done. Converting callers is then trivial. + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + fs/internal.h | 2 ++ + fs/namei.c | 40 ++++++++++++++++++++++------------------ + 2 files changed, 24 insertions(+), 18 deletions(-) + +--- a/fs/internal.h ++++ b/fs/internal.h +@@ -77,6 +77,8 @@ extern int vfs_path_lookup(struct dentry + long do_rmdir(int dfd, struct filename *name); + long do_unlinkat(int dfd, struct filename *name); + int may_linkat(struct path *link); ++int do_renameat2(int olddfd, struct filename *oldname, int newdfd, ++ struct filename *newname, unsigned int flags); + + /* + * namespace.c +--- a/fs/namei.c ++++ b/fs/namei.c +@@ -4353,8 +4353,8 @@ out: + } + EXPORT_SYMBOL(vfs_rename); + +-static int do_renameat2(int olddfd, const char __user *oldname, int newdfd, +- const char __user *newname, unsigned int flags) ++int do_renameat2(int olddfd, struct filename *from, int newdfd, ++ struct filename *to, unsigned int flags) + { + struct dentry *old_dentry, *new_dentry; + struct dentry *trap; +@@ -4362,32 +4362,30 @@ static int do_renameat2(int olddfd, cons + struct qstr old_last, new_last; + int old_type, new_type; + struct inode *delegated_inode = NULL; +- struct filename *from; +- struct filename *to; + unsigned int lookup_flags = 0, target_flags = LOOKUP_RENAME_TARGET; + bool should_retry = false; +- int error; ++ int error = -EINVAL; + + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) +- return -EINVAL; ++ goto put_both; + + if ((flags & (RENAME_NOREPLACE | RENAME_WHITEOUT)) && + (flags & RENAME_EXCHANGE)) +- return -EINVAL; ++ goto put_both; + + if (flags & RENAME_EXCHANGE) + target_flags = 0; + + retry: +- from = filename_parentat(olddfd, getname(oldname), lookup_flags, +- &old_path, &old_last, &old_type); ++ from = filename_parentat(olddfd, from, lookup_flags, &old_path, ++ &old_last, &old_type); + if (IS_ERR(from)) { + error = PTR_ERR(from); +- goto exit; ++ goto put_new; + } + +- to = filename_parentat(newdfd, getname(newname), lookup_flags, +- &new_path, &new_last, &new_type); ++ to = filename_parentat(newdfd, to, lookup_flags, &new_path, &new_last, ++ &new_type); + if (IS_ERR(to)) { + error = PTR_ERR(to); + goto exit1; +@@ -4480,34 +4478,40 @@ exit2: + if (retry_estale(error, lookup_flags)) + should_retry = true; + path_put(&new_path); +- putname(to); + exit1: + path_put(&old_path); +- putname(from); + if (should_retry) { + should_retry = false; + lookup_flags |= LOOKUP_REVAL; + goto retry; + } +-exit: ++put_both: ++ if (!IS_ERR(from)) ++ putname(from); ++put_new: ++ if (!IS_ERR(to)) ++ putname(to); + return error; + } + + SYSCALL_DEFINE5(renameat2, int, olddfd, const char __user *, oldname, + int, newdfd, const char __user *, newname, unsigned int, flags) + { +- return do_renameat2(olddfd, oldname, newdfd, newname, flags); ++ return do_renameat2(olddfd, getname(oldname), newdfd, getname(newname), ++ flags); + } + + SYSCALL_DEFINE4(renameat, int, olddfd, const char __user *, oldname, + int, newdfd, const char __user *, newname) + { +- return do_renameat2(olddfd, oldname, newdfd, newname, 0); ++ return do_renameat2(olddfd, getname(oldname), newdfd, getname(newname), ++ 0); + } + + SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newname) + { +- return do_renameat2(AT_FDCWD, oldname, AT_FDCWD, newname, 0); ++ return do_renameat2(AT_FDCWD, getname(oldname), AT_FDCWD, ++ getname(newname), 0); + } + + int readlink_copy(char __user *buffer, int buflen, const char *link) diff --git a/next-5.10/fs-provide-locked-helper-variant-of-close_fd_get_file.patch b/next-5.10/fs-provide-locked-helper-variant-of-close_fd_get_file.patch new file mode 100644 index 00000000000..5fdefdff4fe --- /dev/null +++ b/next-5.10/fs-provide-locked-helper-variant-of-close_fd_get_file.patch @@ -0,0 +1,100 @@ +From 369d83e272a9f1a316ce79b1236f1f3265e40efe Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Tue, 19 Jan 2021 15:41:52 -0700 +Subject: fs: provide locked helper variant of close_fd_get_file() + +From: Jens Axboe + +[ Upstream commit 53dec2ea74f2ef360e8455439be96a780baa6097 ] + +Assumes current->files->file_lock is already held on invocation. Helps +the caller check the file before removing the fd, if it needs to. + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + fs/file.c | 36 +++++++++++++++++++++++++----------- + fs/internal.h | 1 + + 2 files changed, 26 insertions(+), 11 deletions(-) + +--- a/fs/file.c ++++ b/fs/file.c +@@ -22,6 +22,8 @@ + #include + #include + ++#include "internal.h" ++ + unsigned int sysctl_nr_open __read_mostly = 1024*1024; + unsigned int sysctl_nr_open_min = BITS_PER_LONG; + /* our min() is unusable in constant expressions ;-/ */ +@@ -780,36 +782,48 @@ int __close_range(unsigned fd, unsigned + } + + /* +- * variant of close_fd that gets a ref on the file for later fput. +- * The caller must ensure that filp_close() called on the file, and then +- * an fput(). ++ * See close_fd_get_file() below, this variant assumes current->files->file_lock ++ * is held. + */ +-int close_fd_get_file(unsigned int fd, struct file **res) ++int __close_fd_get_file(unsigned int fd, struct file **res) + { + struct files_struct *files = current->files; + struct file *file; + struct fdtable *fdt; + +- spin_lock(&files->file_lock); + fdt = files_fdtable(files); + if (fd >= fdt->max_fds) +- goto out_unlock; ++ goto out_err; + file = fdt->fd[fd]; + if (!file) +- goto out_unlock; ++ goto out_err; + rcu_assign_pointer(fdt->fd[fd], NULL); + __put_unused_fd(files, fd); +- spin_unlock(&files->file_lock); + get_file(file); + *res = file; + return 0; +- +-out_unlock: +- spin_unlock(&files->file_lock); ++out_err: + *res = NULL; + return -ENOENT; + } + ++/* ++ * variant of close_fd that gets a ref on the file for later fput. ++ * The caller must ensure that filp_close() called on the file, and then ++ * an fput(). ++ */ ++int close_fd_get_file(unsigned int fd, struct file **res) ++{ ++ struct files_struct *files = current->files; ++ int ret; ++ ++ spin_lock(&files->file_lock); ++ ret = __close_fd_get_file(fd, res); ++ spin_unlock(&files->file_lock); ++ ++ return ret; ++} ++ + void do_close_on_exec(struct files_struct *files) + { + unsigned i; +--- a/fs/internal.h ++++ b/fs/internal.h +@@ -134,6 +134,7 @@ extern struct file *do_file_open_root(st + const char *, const struct open_flags *); + extern struct open_how build_open_how(int flags, umode_t mode); + extern int build_open_flags(const struct open_how *how, struct open_flags *op); ++extern int __close_fd_get_file(unsigned int fd, struct file **res); + + long do_sys_ftruncate(unsigned int fd, loff_t length, int small); + int chmod_common(const struct path *path, umode_t mode); diff --git a/queue-5.10/fs-sysv-fix-sysv_nblocks-returns-wrong-value.patch b/next-5.10/fs-sysv-fix-sysv_nblocks-returns-wrong-value.patch similarity index 100% rename from queue-5.10/fs-sysv-fix-sysv_nblocks-returns-wrong-value.patch rename to next-5.10/fs-sysv-fix-sysv_nblocks-returns-wrong-value.patch diff --git a/queue-5.10/futex-move-to-kernel-futex.patch b/next-5.10/futex-move-to-kernel-futex.patch similarity index 100% rename from queue-5.10/futex-move-to-kernel-futex.patch rename to next-5.10/futex-move-to-kernel-futex.patch diff --git a/queue-5.10/futex-resend-potentially-swallowed-owner-death-notif.patch b/next-5.10/futex-resend-potentially-swallowed-owner-death-notif.patch similarity index 100% rename from queue-5.10/futex-resend-potentially-swallowed-owner-death-notif.patch rename to next-5.10/futex-resend-potentially-swallowed-owner-death-notif.patch diff --git a/queue-5.10/gcov-add-support-for-checksum-field.patch b/next-5.10/gcov-add-support-for-checksum-field.patch similarity index 100% rename from queue-5.10/gcov-add-support-for-checksum-field.patch rename to next-5.10/gcov-add-support-for-checksum-field.patch diff --git a/queue-5.10/genirq-add-irqf_no_autoen-for-request_irq-nmi.patch b/next-5.10/genirq-add-irqf_no_autoen-for-request_irq-nmi.patch similarity index 100% rename from queue-5.10/genirq-add-irqf_no_autoen-for-request_irq-nmi.patch rename to next-5.10/genirq-add-irqf_no_autoen-for-request_irq-nmi.patch diff --git a/queue-5.10/genirq-irqdesc-don-t-try-to-remove-non-existing-sysf.patch b/next-5.10/genirq-irqdesc-don-t-try-to-remove-non-existing-sysf.patch similarity index 100% rename from queue-5.10/genirq-irqdesc-don-t-try-to-remove-non-existing-sysf.patch rename to next-5.10/genirq-irqdesc-don-t-try-to-remove-non-existing-sysf.patch diff --git a/queue-5.10/gpiolib-cdev-fix-null-pointer-dereferences.patch b/next-5.10/gpiolib-cdev-fix-null-pointer-dereferences.patch similarity index 100% rename from queue-5.10/gpiolib-cdev-fix-null-pointer-dereferences.patch rename to next-5.10/gpiolib-cdev-fix-null-pointer-dereferences.patch diff --git a/queue-5.10/gpiolib-get-rid-of-redundant-else.patch b/next-5.10/gpiolib-get-rid-of-redundant-else.patch similarity index 100% rename from queue-5.10/gpiolib-get-rid-of-redundant-else.patch rename to next-5.10/gpiolib-get-rid-of-redundant-else.patch diff --git a/next-5.10/h8300-add-support-for-tif_notify_signal.patch b/next-5.10/h8300-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..3c866d2dcb3 --- /dev/null +++ b/next-5.10/h8300-add-support-for-tif_notify_signal.patch @@ -0,0 +1,57 @@ +From b83ee681e7dd713540f41cadb6217784083b38ec Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 14:42:33 -0600 +Subject: h8300: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 2f9799ad0111ee742ccc02dd2ea2c87646746fc1 ] + +Wire up TIF_NOTIFY_SIGNAL handling for h8300. + +Cc: uclinux-h8-devel@lists.sourceforge.jp +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/h8300/include/asm/thread_info.h | 4 +++- + arch/h8300/kernel/signal.c | 2 +- + 2 files changed, 4 insertions(+), 2 deletions(-) + +--- a/arch/h8300/include/asm/thread_info.h ++++ b/arch/h8300/include/asm/thread_info.h +@@ -73,6 +73,7 @@ static inline struct thread_info *curren + #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ + #define TIF_SYSCALL_TRACEPOINT 8 /* for ftrace syscall instrumentation */ + #define TIF_POLLING_NRFLAG 9 /* true if poll_idle() is polling TIF_NEED_RESCHED */ ++#define TIF_NOTIFY_SIGNAL 10 /* signal notifications exist */ + + /* as above, but as bit values */ + #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) +@@ -83,6 +84,7 @@ static inline struct thread_info *curren + #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) + #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) + #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + + /* work to do in syscall trace */ + #define _TIF_WORK_SYSCALL_MASK (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP | \ +@@ -92,7 +94,7 @@ static inline struct thread_info *curren + #define _TIF_ALLWORK_MASK (_TIF_SYSCALL_TRACE | _TIF_SIGPENDING | \ + _TIF_NEED_RESCHED | _TIF_SYSCALL_AUDIT | \ + _TIF_SINGLESTEP | _TIF_NOTIFY_RESUME | \ +- _TIF_SYSCALL_TRACEPOINT) ++ _TIF_SYSCALL_TRACEPOINT | _TIF_NOTIFY_SIGNAL) + + /* work to do on interrupt/exception return */ + #define _TIF_WORK_MASK (_TIF_ALLWORK_MASK & ~(_TIF_SYSCALL_TRACE | \ +--- a/arch/h8300/kernel/signal.c ++++ b/arch/h8300/kernel/signal.c +@@ -279,7 +279,7 @@ static void do_signal(struct pt_regs *re + + asmlinkage void do_notify_resume(struct pt_regs *regs, u32 thread_info_flags) + { +- if (thread_info_flags & _TIF_SIGPENDING) ++ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs); + + if (thread_info_flags & _TIF_NOTIFY_RESUME) diff --git a/queue-5.10/hamradio-baycom_epp-fix-return-type-of-baycom_send_p.patch b/next-5.10/hamradio-baycom_epp-fix-return-type-of-baycom_send_p.patch similarity index 100% rename from queue-5.10/hamradio-baycom_epp-fix-return-type-of-baycom_send_p.patch rename to next-5.10/hamradio-baycom_epp-fix-return-type-of-baycom_send_p.patch diff --git a/queue-5.10/hamradio-don-t-call-dev_kfree_skb-under-spin_lock_ir.patch b/next-5.10/hamradio-don-t-call-dev_kfree_skb-under-spin_lock_ir.patch similarity index 100% rename from queue-5.10/hamradio-don-t-call-dev_kfree_skb-under-spin_lock_ir.patch rename to next-5.10/hamradio-don-t-call-dev_kfree_skb-under-spin_lock_ir.patch diff --git a/next-5.10/hexagon-add-support-for-tif_notify_signal.patch b/next-5.10/hexagon-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..8d1106b7038 --- /dev/null +++ b/next-5.10/hexagon-add-support-for-tif_notify_signal.patch @@ -0,0 +1,49 @@ +From ba4b24618290c7ee52180aa3706729e4fb2fd4e4 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 14:45:22 -0600 +Subject: hexagon: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit aeec8193578a71d0aee21218351849d38121ce90 ] + +Wire up TIF_NOTIFY_SIGNAL handling for hexagon. + +Cc: linux-hexagon@vger.kernel.org +Acked-by: Brian Cain +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/hexagon/include/asm/thread_info.h | 2 ++ + arch/hexagon/kernel/process.c | 2 +- + 2 files changed, 3 insertions(+), 1 deletion(-) + +--- a/arch/hexagon/include/asm/thread_info.h ++++ b/arch/hexagon/include/asm/thread_info.h +@@ -95,6 +95,7 @@ register struct thread_info *__current_t + #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ + #define TIF_SINGLESTEP 4 /* restore ss @ return to usr mode */ + #define TIF_RESTORE_SIGMASK 6 /* restore sig mask in do_signal() */ ++#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ + /* true if poll_idle() is polling TIF_NEED_RESCHED */ + #define TIF_MEMDIE 17 /* OOM killer killed process */ + +@@ -103,6 +104,7 @@ register struct thread_info *__current_t + #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) + #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) + #define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + + /* work to do on interrupt/exception return - All but TIF_SYSCALL_TRACE */ + #define _TIF_WORK_MASK (0x0000FFFF & ~_TIF_SYSCALL_TRACE) +--- a/arch/hexagon/kernel/process.c ++++ b/arch/hexagon/kernel/process.c +@@ -174,7 +174,7 @@ int do_work_pending(struct pt_regs *regs + return 1; + } + +- if (thread_info_flags & _TIF_SIGPENDING) { ++ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) { + do_signal(regs); + return 1; + } diff --git a/queue-5.10/hfs-fix-oob-read-in-__hfs_brec_find.patch b/next-5.10/hfs-fix-oob-read-in-__hfs_brec_find.patch similarity index 100% rename from queue-5.10/hfs-fix-oob-read-in-__hfs_brec_find.patch rename to next-5.10/hfs-fix-oob-read-in-__hfs_brec_find.patch diff --git a/queue-5.10/hfs-fix-oob-write-in-hfs_asc2mac.patch b/next-5.10/hfs-fix-oob-write-in-hfs_asc2mac.patch similarity index 100% rename from queue-5.10/hfs-fix-oob-write-in-hfs_asc2mac.patch rename to next-5.10/hfs-fix-oob-write-in-hfs_asc2mac.patch diff --git a/queue-5.10/hfsplus-fix-bug-causing-custom-uid-and-gid-being-unable-to-be-assigned-with-mount.patch b/next-5.10/hfsplus-fix-bug-causing-custom-uid-and-gid-being-unable-to-be-assigned-with-mount.patch similarity index 100% rename from queue-5.10/hfsplus-fix-bug-causing-custom-uid-and-gid-being-unable-to-be-assigned-with-mount.patch rename to next-5.10/hfsplus-fix-bug-causing-custom-uid-and-gid-being-unable-to-be-assigned-with-mount.patch diff --git a/queue-5.10/hid-hid-sensor-custom-set-fixed-size-for-custom-attr.patch b/next-5.10/hid-hid-sensor-custom-set-fixed-size-for-custom-attr.patch similarity index 100% rename from queue-5.10/hid-hid-sensor-custom-set-fixed-size-for-custom-attr.patch rename to next-5.10/hid-hid-sensor-custom-set-fixed-size-for-custom-attr.patch diff --git a/queue-5.10/hid-mcp2221-don-t-connect-hidraw.patch b/next-5.10/hid-mcp2221-don-t-connect-hidraw.patch similarity index 100% rename from queue-5.10/hid-mcp2221-don-t-connect-hidraw.patch rename to next-5.10/hid-mcp2221-don-t-connect-hidraw.patch diff --git a/queue-5.10/hid-multitouch-fix-asus-expertbook-p2-p2451fa-trackp.patch b/next-5.10/hid-multitouch-fix-asus-expertbook-p2-p2451fa-trackp.patch similarity index 100% rename from queue-5.10/hid-multitouch-fix-asus-expertbook-p2-p2451fa-trackp.patch rename to next-5.10/hid-multitouch-fix-asus-expertbook-p2-p2451fa-trackp.patch diff --git a/queue-5.10/hid-plantronics-additional-pids-for-double-volume-ke.patch b/next-5.10/hid-plantronics-additional-pids-for-double-volume-ke.patch similarity index 100% rename from queue-5.10/hid-plantronics-additional-pids-for-double-volume-ke.patch rename to next-5.10/hid-plantronics-additional-pids-for-double-volume-ke.patch diff --git a/queue-5.10/hid-wacom-ensure-bootloader-pid-is-usable-in-hidraw-mode.patch b/next-5.10/hid-wacom-ensure-bootloader-pid-is-usable-in-hidraw-mode.patch similarity index 100% rename from queue-5.10/hid-wacom-ensure-bootloader-pid-is-usable-in-hidraw-mode.patch rename to next-5.10/hid-wacom-ensure-bootloader-pid-is-usable-in-hidraw-mode.patch diff --git a/queue-5.10/hsi-omap_ssi_core-fix-error-handling-in-ssi_init.patch b/next-5.10/hsi-omap_ssi_core-fix-error-handling-in-ssi_init.patch similarity index 100% rename from queue-5.10/hsi-omap_ssi_core-fix-error-handling-in-ssi_init.patch rename to next-5.10/hsi-omap_ssi_core-fix-error-handling-in-ssi_init.patch diff --git a/queue-5.10/hsi-omap_ssi_core-fix-possible-memory-leak-in-ssi_pr.patch b/next-5.10/hsi-omap_ssi_core-fix-possible-memory-leak-in-ssi_pr.patch similarity index 100% rename from queue-5.10/hsi-omap_ssi_core-fix-possible-memory-leak-in-ssi_pr.patch rename to next-5.10/hsi-omap_ssi_core-fix-possible-memory-leak-in-ssi_pr.patch diff --git a/queue-5.10/hsi-omap_ssi_core-fix-unbalanced-pm_runtime_disable.patch b/next-5.10/hsi-omap_ssi_core-fix-unbalanced-pm_runtime_disable.patch similarity index 100% rename from queue-5.10/hsi-omap_ssi_core-fix-unbalanced-pm_runtime_disable.patch rename to next-5.10/hsi-omap_ssi_core-fix-unbalanced-pm_runtime_disable.patch diff --git a/queue-5.10/hsr-add-a-rcu-read-lock-to-hsr_forward_skb.patch b/next-5.10/hsr-add-a-rcu-read-lock-to-hsr_forward_skb.patch similarity index 100% rename from queue-5.10/hsr-add-a-rcu-read-lock-to-hsr_forward_skb.patch rename to next-5.10/hsr-add-a-rcu-read-lock-to-hsr_forward_skb.patch diff --git a/queue-5.10/hsr-disable-netpoll.patch b/next-5.10/hsr-disable-netpoll.patch similarity index 100% rename from queue-5.10/hsr-disable-netpoll.patch rename to next-5.10/hsr-disable-netpoll.patch diff --git a/queue-5.10/hsr-synchronize-sending-frames-to-have-always-increm.patch b/next-5.10/hsr-synchronize-sending-frames-to-have-always-increm.patch similarity index 100% rename from queue-5.10/hsr-synchronize-sending-frames-to-have-always-increm.patch rename to next-5.10/hsr-synchronize-sending-frames-to-have-always-increm.patch diff --git a/queue-5.10/hsr-synchronize-sequence-number-updates.patch b/next-5.10/hsr-synchronize-sequence-number-updates.patch similarity index 100% rename from queue-5.10/hsr-synchronize-sequence-number-updates.patch rename to next-5.10/hsr-synchronize-sequence-number-updates.patch diff --git a/queue-5.10/hugetlbfs-fix-null-ptr-deref-in-hugetlbfs_parse_para.patch b/next-5.10/hugetlbfs-fix-null-ptr-deref-in-hugetlbfs_parse_para.patch similarity index 100% rename from queue-5.10/hugetlbfs-fix-null-ptr-deref-in-hugetlbfs_parse_para.patch rename to next-5.10/hugetlbfs-fix-null-ptr-deref-in-hugetlbfs_parse_para.patch diff --git a/queue-5.10/hwmon-jc42-convert-register-access-and-caching-to-re.patch b/next-5.10/hwmon-jc42-convert-register-access-and-caching-to-re.patch similarity index 100% rename from queue-5.10/hwmon-jc42-convert-register-access-and-caching-to-re.patch rename to next-5.10/hwmon-jc42-convert-register-access-and-caching-to-re.patch diff --git a/queue-5.10/hwmon-jc42-fix-missing-unlock-on-error-in-jc42_write.patch b/next-5.10/hwmon-jc42-fix-missing-unlock-on-error-in-jc42_write.patch similarity index 100% rename from queue-5.10/hwmon-jc42-fix-missing-unlock-on-error-in-jc42_write.patch rename to next-5.10/hwmon-jc42-fix-missing-unlock-on-error-in-jc42_write.patch diff --git a/queue-5.10/hwmon-jc42-restore-the-min-max-critical-temperatures.patch b/next-5.10/hwmon-jc42-restore-the-min-max-critical-temperatures.patch similarity index 100% rename from queue-5.10/hwmon-jc42-restore-the-min-max-critical-temperatures.patch rename to next-5.10/hwmon-jc42-restore-the-min-max-critical-temperatures.patch diff --git a/queue-5.10/hwrng-amd-fix-pci-device-refcount-leak.patch b/next-5.10/hwrng-amd-fix-pci-device-refcount-leak.patch similarity index 100% rename from queue-5.10/hwrng-amd-fix-pci-device-refcount-leak.patch rename to next-5.10/hwrng-amd-fix-pci-device-refcount-leak.patch diff --git a/queue-5.10/hwrng-geode-fix-pci-device-refcount-leak.patch b/next-5.10/hwrng-geode-fix-pci-device-refcount-leak.patch similarity index 100% rename from queue-5.10/hwrng-geode-fix-pci-device-refcount-leak.patch rename to next-5.10/hwrng-geode-fix-pci-device-refcount-leak.patch diff --git a/queue-5.10/i2c-ismt-fix-an-out-of-bounds-bug-in-ismt_access.patch b/next-5.10/i2c-ismt-fix-an-out-of-bounds-bug-in-ismt_access.patch similarity index 100% rename from queue-5.10/i2c-ismt-fix-an-out-of-bounds-bug-in-ismt_access.patch rename to next-5.10/i2c-ismt-fix-an-out-of-bounds-bug-in-ismt_access.patch diff --git a/queue-5.10/i2c-mux-reg-check-return-value-after-calling-platfor.patch b/next-5.10/i2c-mux-reg-check-return-value-after-calling-platfor.patch similarity index 100% rename from queue-5.10/i2c-mux-reg-check-return-value-after-calling-platfor.patch rename to next-5.10/i2c-mux-reg-check-return-value-after-calling-platfor.patch diff --git a/queue-5.10/i2c-pxa-pci-fix-missing-pci_disable_device-on-error-.patch b/next-5.10/i2c-pxa-pci-fix-missing-pci_disable_device-on-error-.patch similarity index 100% rename from queue-5.10/i2c-pxa-pci-fix-missing-pci_disable_device-on-error-.patch rename to next-5.10/i2c-pxa-pci-fix-missing-pci_disable_device-on-error-.patch diff --git a/next-5.10/ia64-add-support-for-tif_notify_signal.patch b/next-5.10/ia64-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..a327b343b51 --- /dev/null +++ b/next-5.10/ia64-add-support-for-tif_notify_signal.patch @@ -0,0 +1,59 @@ +From 4732812234a886b6e508d15ebb8ba1f61cb82c87 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 14:49:43 -0600 +Subject: ia64: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit b269c229b0e89aedb7943c06673b56b6052cf5e5 ] + +Wire up TIF_NOTIFY_SIGNAL handling for ia64. + +Cc: linux-ia64@vger.kernel.org +[axboe: added fixes from Mike Rapoport ] +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/ia64/include/asm/thread_info.h | 4 +++- + arch/ia64/kernel/process.c | 3 ++- + 2 files changed, 5 insertions(+), 2 deletions(-) + +--- a/arch/ia64/include/asm/thread_info.h ++++ b/arch/ia64/include/asm/thread_info.h +@@ -103,6 +103,7 @@ struct thread_info { + #define TIF_SYSCALL_TRACE 2 /* syscall trace active */ + #define TIF_SYSCALL_AUDIT 3 /* syscall auditing active */ + #define TIF_SINGLESTEP 4 /* restore singlestep on return to user mode */ ++#define TIF_NOTIFY_SIGNAL 5 /* signal notification exist */ + #define TIF_NOTIFY_RESUME 6 /* resumption notification requested */ + #define TIF_MEMDIE 17 /* is terminating due to OOM killer */ + #define TIF_MCA_INIT 18 /* this task is processing MCA or INIT */ +@@ -115,6 +116,7 @@ struct thread_info { + #define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) + #define _TIF_SYSCALL_TRACEAUDIT (_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP) + #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) + #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) + #define _TIF_MCA_INIT (1 << TIF_MCA_INIT) +@@ -124,7 +126,7 @@ struct thread_info { + + /* "work to do on user-return" bits */ + #define TIF_ALLWORK_MASK (_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SYSCALL_AUDIT|\ +- _TIF_NEED_RESCHED|_TIF_SYSCALL_TRACE) ++ _TIF_NEED_RESCHED|_TIF_SYSCALL_TRACE|_TIF_NOTIFY_SIGNAL) + /* like TIF_ALLWORK_BITS but sans TIF_SYSCALL_TRACE or TIF_SYSCALL_AUDIT */ + #define TIF_WORK_MASK (TIF_ALLWORK_MASK&~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT)) + +--- a/arch/ia64/kernel/process.c ++++ b/arch/ia64/kernel/process.c +@@ -171,7 +171,8 @@ do_notify_resume_user(sigset_t *unused, + } + + /* deal with pending signal delivery */ +- if (test_thread_flag(TIF_SIGPENDING)) { ++ if (test_thread_flag(TIF_SIGPENDING) || ++ test_thread_flag(TIF_NOTIFY_SIGNAL)) { + local_irq_enable(); /* force interrupt enable */ + ia64_do_signal(scr, in_syscall); + } diff --git a/next-5.10/ia64-don-t-call-handle_signal-unless-there-s-actually-a-signal-queued.patch b/next-5.10/ia64-don-t-call-handle_signal-unless-there-s-actually-a-signal-queued.patch new file mode 100644 index 00000000000..c9431cb81d6 --- /dev/null +++ b/next-5.10/ia64-don-t-call-handle_signal-unless-there-s-actually-a-signal-queued.patch @@ -0,0 +1,40 @@ +From 621649af8b9db1666b299d6b8c1bd384f01c4e23 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Tue, 2 Mar 2021 17:22:11 -0700 +Subject: ia64: don't call handle_signal() unless there's actually a signal queued + +From: Jens Axboe + +[ Upstream commit f5f4fc4649ae542b1a25670b17aaf3cbb6187acc ] + +Sergei and John both reported that ia64 failed to boot in 5.11, and it +was related to signals. Turns out the ia64 signal handling is a bit odd, +it doesn't check the return value of get_signal() for whether there's a +signal to deliver or not. With the introduction of TIF_NOTIFY_SIGNAL, +then task_work could trigger it. + +Fix it by only calling handle_signal() if we actually have a real signal +to deliver. This brings it in line with all other archs, too. + +Fixes: b269c229b0e8 ("ia64: add support for TIF_NOTIFY_SIGNAL") +Reported-by: Sergei Trofimovich +Reported-by: John Paul Adrian Glaubitz +Tested-by: Sergei Trofimovich +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/ia64/kernel/signal.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +--- a/arch/ia64/kernel/signal.c ++++ b/arch/ia64/kernel/signal.c +@@ -341,7 +341,8 @@ ia64_do_signal (struct sigscratch *scr, + * need to push through a forced SIGSEGV. + */ + while (1) { +- get_signal(&ksig); ++ if (!get_signal(&ksig)) ++ break; + + /* + * get_signal() may have run a debugger (via notify_parent()) diff --git a/queue-5.10/ib-ipoib-fix-queue-count-inconsistency-for-pkey-chil.patch b/next-5.10/ib-ipoib-fix-queue-count-inconsistency-for-pkey-chil.patch similarity index 100% rename from queue-5.10/ib-ipoib-fix-queue-count-inconsistency-for-pkey-chil.patch rename to next-5.10/ib-ipoib-fix-queue-count-inconsistency-for-pkey-chil.patch diff --git a/queue-5.10/igb-do-not-free-q_vector-unless-new-one-was-allocate.patch b/next-5.10/igb-do-not-free-q_vector-unless-new-one-was-allocate.patch similarity index 100% rename from queue-5.10/igb-do-not-free-q_vector-unless-new-one-was-allocate.patch rename to next-5.10/igb-do-not-free-q_vector-unless-new-one-was-allocate.patch diff --git a/queue-5.10/igc-add-checking-for-basetime-less-than-zero.patch b/next-5.10/igc-add-checking-for-basetime-less-than-zero.patch similarity index 100% rename from queue-5.10/igc-add-checking-for-basetime-less-than-zero.patch rename to next-5.10/igc-add-checking-for-basetime-less-than-zero.patch diff --git a/queue-5.10/igc-enhance-qbv-scheduling-by-using-first-flag-bit.patch b/next-5.10/igc-enhance-qbv-scheduling-by-using-first-flag-bit.patch similarity index 100% rename from queue-5.10/igc-enhance-qbv-scheduling-by-using-first-flag-bit.patch rename to next-5.10/igc-enhance-qbv-scheduling-by-using-first-flag-bit.patch diff --git a/queue-5.10/igc-lift-taprio-schedule-restriction.patch b/next-5.10/igc-lift-taprio-schedule-restriction.patch similarity index 100% rename from queue-5.10/igc-lift-taprio-schedule-restriction.patch rename to next-5.10/igc-lift-taprio-schedule-restriction.patch diff --git a/queue-5.10/igc-recalculate-qbv-end_time-by-considering-cycle-ti.patch b/next-5.10/igc-recalculate-qbv-end_time-by-considering-cycle-ti.patch similarity index 100% rename from queue-5.10/igc-recalculate-qbv-end_time-by-considering-cycle-ti.patch rename to next-5.10/igc-recalculate-qbv-end_time-by-considering-cycle-ti.patch diff --git a/queue-5.10/igc-set-qbv-start_time-and-end_time-to-end_time-if-n.patch b/next-5.10/igc-set-qbv-start_time-and-end_time-to-end_time-if-n.patch similarity index 100% rename from queue-5.10/igc-set-qbv-start_time-and-end_time-to-end_time-if-n.patch rename to next-5.10/igc-set-qbv-start_time-and-end_time-to-end_time-if-n.patch diff --git a/queue-5.10/igc-use-strict-cycles-for-qbv-scheduling.patch b/next-5.10/igc-use-strict-cycles-for-qbv-scheduling.patch similarity index 100% rename from queue-5.10/igc-use-strict-cycles-for-qbv-scheduling.patch rename to next-5.10/igc-use-strict-cycles-for-qbv-scheduling.patch diff --git a/queue-5.10/iio-adc-ad_sigma_delta-do-not-use-internal-iio_dev-lock.patch b/next-5.10/iio-adc-ad_sigma_delta-do-not-use-internal-iio_dev-lock.patch similarity index 100% rename from queue-5.10/iio-adc-ad_sigma_delta-do-not-use-internal-iio_dev-lock.patch rename to next-5.10/iio-adc-ad_sigma_delta-do-not-use-internal-iio_dev-lock.patch diff --git a/queue-5.10/iio-adc128s052-add-proper-.data-members-in-adc128_of_match-table.patch b/next-5.10/iio-adc128s052-add-proper-.data-members-in-adc128_of_match-table.patch similarity index 100% rename from queue-5.10/iio-adc128s052-add-proper-.data-members-in-adc128_of_match-table.patch rename to next-5.10/iio-adc128s052-add-proper-.data-members-in-adc128_of_match-table.patch diff --git a/queue-5.10/iio-adis-add-__adis_enable_irq-implementation.patch b/next-5.10/iio-adis-add-__adis_enable_irq-implementation.patch similarity index 100% rename from queue-5.10/iio-adis-add-__adis_enable_irq-implementation.patch rename to next-5.10/iio-adis-add-__adis_enable_irq-implementation.patch diff --git a/queue-5.10/iio-adis-handle-devices-that-cannot-unmask-the-drdy-.patch b/next-5.10/iio-adis-handle-devices-that-cannot-unmask-the-drdy-.patch similarity index 100% rename from queue-5.10/iio-adis-handle-devices-that-cannot-unmask-the-drdy-.patch rename to next-5.10/iio-adis-handle-devices-that-cannot-unmask-the-drdy-.patch diff --git a/queue-5.10/iio-adis-stylistic-changes.patch b/next-5.10/iio-adis-stylistic-changes.patch similarity index 100% rename from queue-5.10/iio-adis-stylistic-changes.patch rename to next-5.10/iio-adis-stylistic-changes.patch diff --git a/queue-5.10/iio-imu-adis-move-exports-into-iio_adislib-namespace.patch b/next-5.10/iio-imu-adis-move-exports-into-iio_adislib-namespace.patch similarity index 100% rename from queue-5.10/iio-imu-adis-move-exports-into-iio_adislib-namespace.patch rename to next-5.10/iio-imu-adis-move-exports-into-iio_adislib-namespace.patch diff --git a/queue-5.10/iio-imu-adis-use-irqf_no_autoen-instead-of-irq-reque.patch b/next-5.10/iio-imu-adis-use-irqf_no_autoen-instead-of-irq-reque.patch similarity index 100% rename from queue-5.10/iio-imu-adis-use-irqf_no_autoen-instead-of-irq-reque.patch rename to next-5.10/iio-imu-adis-use-irqf_no_autoen-instead-of-irq-reque.patch diff --git a/queue-5.10/iio-temperature-ltc2983-make-bulk-write-buffer-dma-s.patch b/next-5.10/iio-temperature-ltc2983-make-bulk-write-buffer-dma-s.patch similarity index 100% rename from queue-5.10/iio-temperature-ltc2983-make-bulk-write-buffer-dma-s.patch rename to next-5.10/iio-temperature-ltc2983-make-bulk-write-buffer-dma-s.patch diff --git a/queue-5.10/ima-fix-fall-through-warnings-for-clang.patch b/next-5.10/ima-fix-fall-through-warnings-for-clang.patch similarity index 100% rename from queue-5.10/ima-fix-fall-through-warnings-for-clang.patch rename to next-5.10/ima-fix-fall-through-warnings-for-clang.patch diff --git a/queue-5.10/ima-fix-misuse-of-dereference-of-pointer-in-template.patch b/next-5.10/ima-fix-misuse-of-dereference-of-pointer-in-template.patch similarity index 100% rename from queue-5.10/ima-fix-misuse-of-dereference-of-pointer-in-template.patch rename to next-5.10/ima-fix-misuse-of-dereference-of-pointer-in-template.patch diff --git a/queue-5.10/ima-handle-estale-returned-by-ima_filter_rule_match.patch b/next-5.10/ima-handle-estale-returned-by-ima_filter_rule_match.patch similarity index 100% rename from queue-5.10/ima-handle-estale-returned-by-ima_filter_rule_match.patch rename to next-5.10/ima-handle-estale-returned-by-ima_filter_rule_match.patch diff --git a/queue-5.10/ima-simplify-ima_lsm_copy_rule.patch b/next-5.10/ima-simplify-ima_lsm_copy_rule.patch similarity index 100% rename from queue-5.10/ima-simplify-ima_lsm_copy_rule.patch rename to next-5.10/ima-simplify-ima_lsm_copy_rule.patch diff --git a/queue-5.10/include-uapi-linux-swab-fix-potentially-missing-__al.patch b/next-5.10/include-uapi-linux-swab-fix-potentially-missing-__al.patch similarity index 100% rename from queue-5.10/include-uapi-linux-swab-fix-potentially-missing-__al.patch rename to next-5.10/include-uapi-linux-swab-fix-potentially-missing-__al.patch diff --git a/queue-5.10/inet-add-read_once-sk-sk_bound_dev_if-in-inet_csk_bi.patch b/next-5.10/inet-add-read_once-sk-sk_bound_dev_if-in-inet_csk_bi.patch similarity index 100% rename from queue-5.10/inet-add-read_once-sk-sk_bound_dev_if-in-inet_csk_bi.patch rename to next-5.10/inet-add-read_once-sk-sk_bound_dev_if-in-inet_csk_bi.patch diff --git a/queue-5.10/input-elants_i2c-properly-handle-the-reset-gpio-when.patch b/next-5.10/input-elants_i2c-properly-handle-the-reset-gpio-when.patch similarity index 100% rename from queue-5.10/input-elants_i2c-properly-handle-the-reset-gpio-when.patch rename to next-5.10/input-elants_i2c-properly-handle-the-reset-gpio-when.patch diff --git a/queue-5.10/input-joystick-fix-kconfig-warning-for-joystick_adc.patch b/next-5.10/input-joystick-fix-kconfig-warning-for-joystick_adc.patch similarity index 100% rename from queue-5.10/input-joystick-fix-kconfig-warning-for-joystick_adc.patch rename to next-5.10/input-joystick-fix-kconfig-warning-for-joystick_adc.patch diff --git a/queue-5.10/integrity-fix-memory-leakage-in-keyring-allocation-e.patch b/next-5.10/integrity-fix-memory-leakage-in-keyring-allocation-e.patch similarity index 100% rename from queue-5.10/integrity-fix-memory-leakage-in-keyring-allocation-e.patch rename to next-5.10/integrity-fix-memory-leakage-in-keyring-allocation-e.patch diff --git a/next-5.10/io_uring-import-5.15-stable-io_uring.patch b/next-5.10/io_uring-import-5.15-stable-io_uring.patch new file mode 100644 index 00000000000..f8bceb60436 --- /dev/null +++ b/next-5.10/io_uring-import-5.15-stable-io_uring.patch @@ -0,0 +1,24573 @@ +From ab51127cad79fdc5ff972e04acb2287c0799efc4 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Thu, 22 Dec 2022 14:30:11 -0700 +Subject: io_uring: import 5.15-stable io_uring + +From: Jens Axboe + +No upstream commit exists. + +This imports the io_uring codebase from 5.15.85, wholesale. Changes +from that code base: + +- Drop IOCB_ALLOC_CACHE, we don't have that in 5.10. +- Drop MKDIRAT/SYMLINKAT/LINKAT. Would require further VFS backports, + and we don't support these in 5.10 to begin with. +- sock_from_file() old style calling convention. +- Use compat_get_bitmap() only for CONFIG_COMPAT=y + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + Makefile | 2 + fs/Makefile | 2 + fs/io-wq.c | 1242 ---- + fs/io-wq.h | 157 + fs/io_uring.c | 9971 ------------------------------------ + include/linux/io_uring.h | 46 + include/linux/sched.h | 3 + include/linux/syscalls.h | 2 + include/trace/events/io_uring.h | 121 + include/uapi/linux/io_uring.h | 115 + io_uring/Makefile | 6 + io_uring/io-wq.c | 1398 +++++ + io_uring/io-wq.h | 160 + io_uring/io_uring.c |10945 ++++++++++++++++++++++++++++++++++++++++ + kernel/exit.c | 2 + kernel/fork.c | 1 + kernel/sched/core.c | 2 + 17 files changed, 12704 insertions(+), 11471 deletions(-) + delete mode 100644 fs/io-wq.c + create mode 100644 io_uring/Makefile + create mode 100644 io_uring/io-wq.c + rename {fs => io_uring}/io-wq.h (81%) + rename {fs => io_uring}/io_uring.c (51%) + +--- a/Makefile ++++ b/Makefile +@@ -1128,7 +1128,7 @@ export MODORDER := $(extmod-prefix)modul + export MODULES_NSDEPS := $(extmod-prefix)modules.nsdeps + + ifeq ($(KBUILD_EXTMOD),) +-core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/ block/ ++core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/ block/ io_uring/ + + vmlinux-dirs := $(patsubst %/,%,$(filter %/, \ + $(core-y) $(core-m) $(drivers-y) $(drivers-m) \ +--- a/fs/Makefile ++++ b/fs/Makefile +@@ -32,8 +32,6 @@ obj-$(CONFIG_TIMERFD) += timerfd.o + obj-$(CONFIG_EVENTFD) += eventfd.o + obj-$(CONFIG_USERFAULTFD) += userfaultfd.o + obj-$(CONFIG_AIO) += aio.o +-obj-$(CONFIG_IO_URING) += io_uring.o +-obj-$(CONFIG_IO_WQ) += io-wq.o + obj-$(CONFIG_FS_DAX) += dax.o + obj-$(CONFIG_FS_ENCRYPTION) += crypto/ + obj-$(CONFIG_FS_VERITY) += verity/ +--- a/fs/io-wq.c ++++ /dev/null +@@ -1,1242 +0,0 @@ +-// SPDX-License-Identifier: GPL-2.0 +-/* +- * Basic worker thread pool for io_uring +- * +- * Copyright (C) 2019 Jens Axboe +- * +- */ +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include "../kernel/sched/sched.h" +-#include "io-wq.h" +- +-#define WORKER_IDLE_TIMEOUT (5 * HZ) +- +-enum { +- IO_WORKER_F_UP = 1, /* up and active */ +- IO_WORKER_F_RUNNING = 2, /* account as running */ +- IO_WORKER_F_FREE = 4, /* worker on free list */ +- IO_WORKER_F_FIXED = 8, /* static idle worker */ +- IO_WORKER_F_BOUND = 16, /* is doing bounded work */ +-}; +- +-enum { +- IO_WQ_BIT_EXIT = 0, /* wq exiting */ +- IO_WQ_BIT_CANCEL = 1, /* cancel work on list */ +- IO_WQ_BIT_ERROR = 2, /* error on setup */ +-}; +- +-enum { +- IO_WQE_FLAG_STALLED = 1, /* stalled on hash */ +-}; +- +-/* +- * One for each thread in a wqe pool +- */ +-struct io_worker { +- refcount_t ref; +- unsigned flags; +- struct hlist_nulls_node nulls_node; +- struct list_head all_list; +- struct task_struct *task; +- struct io_wqe *wqe; +- +- struct io_wq_work *cur_work; +- spinlock_t lock; +- +- struct rcu_head rcu; +- struct mm_struct *mm; +-#ifdef CONFIG_BLK_CGROUP +- struct cgroup_subsys_state *blkcg_css; +-#endif +- const struct cred *cur_creds; +- const struct cred *saved_creds; +- struct files_struct *restore_files; +- struct nsproxy *restore_nsproxy; +- struct fs_struct *restore_fs; +-}; +- +-#if BITS_PER_LONG == 64 +-#define IO_WQ_HASH_ORDER 6 +-#else +-#define IO_WQ_HASH_ORDER 5 +-#endif +- +-#define IO_WQ_NR_HASH_BUCKETS (1u << IO_WQ_HASH_ORDER) +- +-struct io_wqe_acct { +- unsigned nr_workers; +- unsigned max_workers; +- atomic_t nr_running; +-}; +- +-enum { +- IO_WQ_ACCT_BOUND, +- IO_WQ_ACCT_UNBOUND, +-}; +- +-/* +- * Per-node worker thread pool +- */ +-struct io_wqe { +- struct { +- raw_spinlock_t lock; +- struct io_wq_work_list work_list; +- unsigned long hash_map; +- unsigned flags; +- } ____cacheline_aligned_in_smp; +- +- int node; +- struct io_wqe_acct acct[2]; +- +- struct hlist_nulls_head free_list; +- struct list_head all_list; +- +- struct io_wq *wq; +- struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS]; +-}; +- +-/* +- * Per io_wq state +- */ +-struct io_wq { +- struct io_wqe **wqes; +- unsigned long state; +- +- free_work_fn *free_work; +- io_wq_work_fn *do_work; +- +- struct task_struct *manager; +- struct user_struct *user; +- refcount_t refs; +- struct completion done; +- +- struct hlist_node cpuhp_node; +- +- refcount_t use_refs; +-}; +- +-static enum cpuhp_state io_wq_online; +- +-static bool io_worker_get(struct io_worker *worker) +-{ +- return refcount_inc_not_zero(&worker->ref); +-} +- +-static void io_worker_release(struct io_worker *worker) +-{ +- if (refcount_dec_and_test(&worker->ref)) +- wake_up_process(worker->task); +-} +- +-/* +- * Note: drops the wqe->lock if returning true! The caller must re-acquire +- * the lock in that case. Some callers need to restart handling if this +- * happens, so we can't just re-acquire the lock on behalf of the caller. +- */ +-static bool __io_worker_unuse(struct io_wqe *wqe, struct io_worker *worker) +-{ +- bool dropped_lock = false; +- +- if (worker->saved_creds) { +- revert_creds(worker->saved_creds); +- worker->cur_creds = worker->saved_creds = NULL; +- } +- +- if (current->files != worker->restore_files) { +- __acquire(&wqe->lock); +- raw_spin_unlock_irq(&wqe->lock); +- dropped_lock = true; +- +- task_lock(current); +- current->files = worker->restore_files; +- current->nsproxy = worker->restore_nsproxy; +- task_unlock(current); +- } +- +- if (current->fs != worker->restore_fs) +- current->fs = worker->restore_fs; +- +- /* +- * If we have an active mm, we need to drop the wq lock before unusing +- * it. If we do, return true and let the caller retry the idle loop. +- */ +- if (worker->mm) { +- if (!dropped_lock) { +- __acquire(&wqe->lock); +- raw_spin_unlock_irq(&wqe->lock); +- dropped_lock = true; +- } +- __set_current_state(TASK_RUNNING); +- kthread_unuse_mm(worker->mm); +- mmput(worker->mm); +- worker->mm = NULL; +- } +- +-#ifdef CONFIG_BLK_CGROUP +- if (worker->blkcg_css) { +- kthread_associate_blkcg(NULL); +- worker->blkcg_css = NULL; +- } +-#endif +- if (current->signal->rlim[RLIMIT_FSIZE].rlim_cur != RLIM_INFINITY) +- current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY; +- return dropped_lock; +-} +- +-static inline struct io_wqe_acct *io_work_get_acct(struct io_wqe *wqe, +- struct io_wq_work *work) +-{ +- if (work->flags & IO_WQ_WORK_UNBOUND) +- return &wqe->acct[IO_WQ_ACCT_UNBOUND]; +- +- return &wqe->acct[IO_WQ_ACCT_BOUND]; +-} +- +-static inline struct io_wqe_acct *io_wqe_get_acct(struct io_wqe *wqe, +- struct io_worker *worker) +-{ +- if (worker->flags & IO_WORKER_F_BOUND) +- return &wqe->acct[IO_WQ_ACCT_BOUND]; +- +- return &wqe->acct[IO_WQ_ACCT_UNBOUND]; +-} +- +-static void io_worker_exit(struct io_worker *worker) +-{ +- struct io_wqe *wqe = worker->wqe; +- struct io_wqe_acct *acct = io_wqe_get_acct(wqe, worker); +- +- /* +- * If we're not at zero, someone else is holding a brief reference +- * to the worker. Wait for that to go away. +- */ +- set_current_state(TASK_INTERRUPTIBLE); +- if (!refcount_dec_and_test(&worker->ref)) +- schedule(); +- __set_current_state(TASK_RUNNING); +- +- preempt_disable(); +- current->flags &= ~PF_IO_WORKER; +- if (worker->flags & IO_WORKER_F_RUNNING) +- atomic_dec(&acct->nr_running); +- if (!(worker->flags & IO_WORKER_F_BOUND)) +- atomic_dec(&wqe->wq->user->processes); +- worker->flags = 0; +- preempt_enable(); +- +- raw_spin_lock_irq(&wqe->lock); +- hlist_nulls_del_rcu(&worker->nulls_node); +- list_del_rcu(&worker->all_list); +- if (__io_worker_unuse(wqe, worker)) { +- __release(&wqe->lock); +- raw_spin_lock_irq(&wqe->lock); +- } +- acct->nr_workers--; +- raw_spin_unlock_irq(&wqe->lock); +- +- kfree_rcu(worker, rcu); +- if (refcount_dec_and_test(&wqe->wq->refs)) +- complete(&wqe->wq->done); +-} +- +-static inline bool io_wqe_run_queue(struct io_wqe *wqe) +- __must_hold(wqe->lock) +-{ +- if (!wq_list_empty(&wqe->work_list) && +- !(wqe->flags & IO_WQE_FLAG_STALLED)) +- return true; +- return false; +-} +- +-/* +- * Check head of free list for an available worker. If one isn't available, +- * caller must wake up the wq manager to create one. +- */ +-static bool io_wqe_activate_free_worker(struct io_wqe *wqe) +- __must_hold(RCU) +-{ +- struct hlist_nulls_node *n; +- struct io_worker *worker; +- +- n = rcu_dereference(hlist_nulls_first_rcu(&wqe->free_list)); +- if (is_a_nulls(n)) +- return false; +- +- worker = hlist_nulls_entry(n, struct io_worker, nulls_node); +- if (io_worker_get(worker)) { +- wake_up_process(worker->task); +- io_worker_release(worker); +- return true; +- } +- +- return false; +-} +- +-/* +- * We need a worker. If we find a free one, we're good. If not, and we're +- * below the max number of workers, wake up the manager to create one. +- */ +-static void io_wqe_wake_worker(struct io_wqe *wqe, struct io_wqe_acct *acct) +-{ +- bool ret; +- +- /* +- * Most likely an attempt to queue unbounded work on an io_wq that +- * wasn't setup with any unbounded workers. +- */ +- if (unlikely(!acct->max_workers)) +- pr_warn_once("io-wq is not configured for unbound workers"); +- +- rcu_read_lock(); +- ret = io_wqe_activate_free_worker(wqe); +- rcu_read_unlock(); +- +- if (!ret && acct->nr_workers < acct->max_workers) +- wake_up_process(wqe->wq->manager); +-} +- +-static void io_wqe_inc_running(struct io_wqe *wqe, struct io_worker *worker) +-{ +- struct io_wqe_acct *acct = io_wqe_get_acct(wqe, worker); +- +- atomic_inc(&acct->nr_running); +-} +- +-static void io_wqe_dec_running(struct io_wqe *wqe, struct io_worker *worker) +- __must_hold(wqe->lock) +-{ +- struct io_wqe_acct *acct = io_wqe_get_acct(wqe, worker); +- +- if (atomic_dec_and_test(&acct->nr_running) && io_wqe_run_queue(wqe)) +- io_wqe_wake_worker(wqe, acct); +-} +- +-static void io_worker_start(struct io_wqe *wqe, struct io_worker *worker) +-{ +- allow_kernel_signal(SIGINT); +- +- current->flags |= PF_IO_WORKER; +- +- worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); +- worker->restore_files = current->files; +- worker->restore_nsproxy = current->nsproxy; +- worker->restore_fs = current->fs; +- io_wqe_inc_running(wqe, worker); +-} +- +-/* +- * Worker will start processing some work. Move it to the busy list, if +- * it's currently on the freelist +- */ +-static void __io_worker_busy(struct io_wqe *wqe, struct io_worker *worker, +- struct io_wq_work *work) +- __must_hold(wqe->lock) +-{ +- bool worker_bound, work_bound; +- +- if (worker->flags & IO_WORKER_F_FREE) { +- worker->flags &= ~IO_WORKER_F_FREE; +- hlist_nulls_del_init_rcu(&worker->nulls_node); +- } +- +- /* +- * If worker is moving from bound to unbound (or vice versa), then +- * ensure we update the running accounting. +- */ +- worker_bound = (worker->flags & IO_WORKER_F_BOUND) != 0; +- work_bound = (work->flags & IO_WQ_WORK_UNBOUND) == 0; +- if (worker_bound != work_bound) { +- io_wqe_dec_running(wqe, worker); +- if (work_bound) { +- worker->flags |= IO_WORKER_F_BOUND; +- wqe->acct[IO_WQ_ACCT_UNBOUND].nr_workers--; +- wqe->acct[IO_WQ_ACCT_BOUND].nr_workers++; +- atomic_dec(&wqe->wq->user->processes); +- } else { +- worker->flags &= ~IO_WORKER_F_BOUND; +- wqe->acct[IO_WQ_ACCT_UNBOUND].nr_workers++; +- wqe->acct[IO_WQ_ACCT_BOUND].nr_workers--; +- atomic_inc(&wqe->wq->user->processes); +- } +- io_wqe_inc_running(wqe, worker); +- } +-} +- +-/* +- * No work, worker going to sleep. Move to freelist, and unuse mm if we +- * have one attached. Dropping the mm may potentially sleep, so we drop +- * the lock in that case and return success. Since the caller has to +- * retry the loop in that case (we changed task state), we don't regrab +- * the lock if we return success. +- */ +-static bool __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker) +- __must_hold(wqe->lock) +-{ +- if (!(worker->flags & IO_WORKER_F_FREE)) { +- worker->flags |= IO_WORKER_F_FREE; +- hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); +- } +- +- return __io_worker_unuse(wqe, worker); +-} +- +-static inline unsigned int io_get_work_hash(struct io_wq_work *work) +-{ +- return work->flags >> IO_WQ_HASH_SHIFT; +-} +- +-static struct io_wq_work *io_get_next_work(struct io_wqe *wqe) +- __must_hold(wqe->lock) +-{ +- struct io_wq_work_node *node, *prev; +- struct io_wq_work *work, *tail; +- unsigned int hash; +- +- wq_list_for_each(node, prev, &wqe->work_list) { +- work = container_of(node, struct io_wq_work, list); +- +- /* not hashed, can run anytime */ +- if (!io_wq_is_hashed(work)) { +- wq_list_del(&wqe->work_list, node, prev); +- return work; +- } +- +- /* hashed, can run if not already running */ +- hash = io_get_work_hash(work); +- if (!(wqe->hash_map & BIT(hash))) { +- wqe->hash_map |= BIT(hash); +- /* all items with this hash lie in [work, tail] */ +- tail = wqe->hash_tail[hash]; +- wqe->hash_tail[hash] = NULL; +- wq_list_cut(&wqe->work_list, &tail->list, prev); +- return work; +- } +- } +- +- return NULL; +-} +- +-static void io_wq_switch_mm(struct io_worker *worker, struct io_wq_work *work) +-{ +- if (worker->mm) { +- kthread_unuse_mm(worker->mm); +- mmput(worker->mm); +- worker->mm = NULL; +- } +- +- if (mmget_not_zero(work->identity->mm)) { +- kthread_use_mm(work->identity->mm); +- worker->mm = work->identity->mm; +- return; +- } +- +- /* failed grabbing mm, ensure work gets cancelled */ +- work->flags |= IO_WQ_WORK_CANCEL; +-} +- +-static inline void io_wq_switch_blkcg(struct io_worker *worker, +- struct io_wq_work *work) +-{ +-#ifdef CONFIG_BLK_CGROUP +- if (!(work->flags & IO_WQ_WORK_BLKCG)) +- return; +- if (work->identity->blkcg_css != worker->blkcg_css) { +- kthread_associate_blkcg(work->identity->blkcg_css); +- worker->blkcg_css = work->identity->blkcg_css; +- } +-#endif +-} +- +-static void io_wq_switch_creds(struct io_worker *worker, +- struct io_wq_work *work) +-{ +- const struct cred *old_creds = override_creds(work->identity->creds); +- +- worker->cur_creds = work->identity->creds; +- if (worker->saved_creds) +- put_cred(old_creds); /* creds set by previous switch */ +- else +- worker->saved_creds = old_creds; +-} +- +-static void io_impersonate_work(struct io_worker *worker, +- struct io_wq_work *work) +-{ +- if ((work->flags & IO_WQ_WORK_FILES) && +- current->files != work->identity->files) { +- task_lock(current); +- current->files = work->identity->files; +- current->nsproxy = work->identity->nsproxy; +- task_unlock(current); +- if (!work->identity->files) { +- /* failed grabbing files, ensure work gets cancelled */ +- work->flags |= IO_WQ_WORK_CANCEL; +- } +- } +- if ((work->flags & IO_WQ_WORK_FS) && current->fs != work->identity->fs) +- current->fs = work->identity->fs; +- if ((work->flags & IO_WQ_WORK_MM) && work->identity->mm != worker->mm) +- io_wq_switch_mm(worker, work); +- if ((work->flags & IO_WQ_WORK_CREDS) && +- worker->cur_creds != work->identity->creds) +- io_wq_switch_creds(worker, work); +- if (work->flags & IO_WQ_WORK_FSIZE) +- current->signal->rlim[RLIMIT_FSIZE].rlim_cur = work->identity->fsize; +- else if (current->signal->rlim[RLIMIT_FSIZE].rlim_cur != RLIM_INFINITY) +- current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY; +- io_wq_switch_blkcg(worker, work); +-#ifdef CONFIG_AUDIT +- current->loginuid = work->identity->loginuid; +- current->sessionid = work->identity->sessionid; +-#endif +-} +- +-static void io_assign_current_work(struct io_worker *worker, +- struct io_wq_work *work) +-{ +- if (work) { +- /* flush pending signals before assigning new work */ +- if (signal_pending(current)) +- flush_signals(current); +- cond_resched(); +- } +- +-#ifdef CONFIG_AUDIT +- current->loginuid = KUIDT_INIT(AUDIT_UID_UNSET); +- current->sessionid = AUDIT_SID_UNSET; +-#endif +- +- spin_lock_irq(&worker->lock); +- worker->cur_work = work; +- spin_unlock_irq(&worker->lock); +-} +- +-static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work); +- +-static void io_worker_handle_work(struct io_worker *worker) +- __releases(wqe->lock) +-{ +- struct io_wqe *wqe = worker->wqe; +- struct io_wq *wq = wqe->wq; +- +- do { +- struct io_wq_work *work; +-get_next: +- /* +- * If we got some work, mark us as busy. If we didn't, but +- * the list isn't empty, it means we stalled on hashed work. +- * Mark us stalled so we don't keep looking for work when we +- * can't make progress, any work completion or insertion will +- * clear the stalled flag. +- */ +- work = io_get_next_work(wqe); +- if (work) +- __io_worker_busy(wqe, worker, work); +- else if (!wq_list_empty(&wqe->work_list)) +- wqe->flags |= IO_WQE_FLAG_STALLED; +- +- raw_spin_unlock_irq(&wqe->lock); +- if (!work) +- break; +- io_assign_current_work(worker, work); +- +- /* handle a whole dependent link */ +- do { +- struct io_wq_work *old_work, *next_hashed, *linked; +- unsigned int hash = io_get_work_hash(work); +- +- next_hashed = wq_next_work(work); +- io_impersonate_work(worker, work); +- /* +- * OK to set IO_WQ_WORK_CANCEL even for uncancellable +- * work, the worker function will do the right thing. +- */ +- if (test_bit(IO_WQ_BIT_CANCEL, &wq->state)) +- work->flags |= IO_WQ_WORK_CANCEL; +- +- old_work = work; +- linked = wq->do_work(work); +- +- work = next_hashed; +- if (!work && linked && !io_wq_is_hashed(linked)) { +- work = linked; +- linked = NULL; +- } +- io_assign_current_work(worker, work); +- wq->free_work(old_work); +- +- if (linked) +- io_wqe_enqueue(wqe, linked); +- +- if (hash != -1U && !next_hashed) { +- raw_spin_lock_irq(&wqe->lock); +- wqe->hash_map &= ~BIT_ULL(hash); +- wqe->flags &= ~IO_WQE_FLAG_STALLED; +- /* skip unnecessary unlock-lock wqe->lock */ +- if (!work) +- goto get_next; +- raw_spin_unlock_irq(&wqe->lock); +- } +- } while (work); +- +- raw_spin_lock_irq(&wqe->lock); +- } while (1); +-} +- +-static int io_wqe_worker(void *data) +-{ +- struct io_worker *worker = data; +- struct io_wqe *wqe = worker->wqe; +- struct io_wq *wq = wqe->wq; +- +- io_worker_start(wqe, worker); +- +- while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { +- set_current_state(TASK_INTERRUPTIBLE); +-loop: +- raw_spin_lock_irq(&wqe->lock); +- if (io_wqe_run_queue(wqe)) { +- __set_current_state(TASK_RUNNING); +- io_worker_handle_work(worker); +- goto loop; +- } +- /* drops the lock on success, retry */ +- if (__io_worker_idle(wqe, worker)) { +- __release(&wqe->lock); +- goto loop; +- } +- raw_spin_unlock_irq(&wqe->lock); +- if (signal_pending(current)) +- flush_signals(current); +- if (schedule_timeout(WORKER_IDLE_TIMEOUT)) +- continue; +- /* timed out, exit unless we're the fixed worker */ +- if (test_bit(IO_WQ_BIT_EXIT, &wq->state) || +- !(worker->flags & IO_WORKER_F_FIXED)) +- break; +- } +- +- if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) { +- raw_spin_lock_irq(&wqe->lock); +- if (!wq_list_empty(&wqe->work_list)) +- io_worker_handle_work(worker); +- else +- raw_spin_unlock_irq(&wqe->lock); +- } +- +- io_worker_exit(worker); +- return 0; +-} +- +-/* +- * Called when a worker is scheduled in. Mark us as currently running. +- */ +-void io_wq_worker_running(struct task_struct *tsk) +-{ +- struct io_worker *worker = kthread_data(tsk); +- struct io_wqe *wqe = worker->wqe; +- +- if (!(worker->flags & IO_WORKER_F_UP)) +- return; +- if (worker->flags & IO_WORKER_F_RUNNING) +- return; +- worker->flags |= IO_WORKER_F_RUNNING; +- io_wqe_inc_running(wqe, worker); +-} +- +-/* +- * Called when worker is going to sleep. If there are no workers currently +- * running and we have work pending, wake up a free one or have the manager +- * set one up. +- */ +-void io_wq_worker_sleeping(struct task_struct *tsk) +-{ +- struct io_worker *worker = kthread_data(tsk); +- struct io_wqe *wqe = worker->wqe; +- +- if (!(worker->flags & IO_WORKER_F_UP)) +- return; +- if (!(worker->flags & IO_WORKER_F_RUNNING)) +- return; +- +- worker->flags &= ~IO_WORKER_F_RUNNING; +- +- raw_spin_lock_irq(&wqe->lock); +- io_wqe_dec_running(wqe, worker); +- raw_spin_unlock_irq(&wqe->lock); +-} +- +-static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) +-{ +- struct io_wqe_acct *acct = &wqe->acct[index]; +- struct io_worker *worker; +- +- worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, wqe->node); +- if (!worker) +- return false; +- +- refcount_set(&worker->ref, 1); +- worker->nulls_node.pprev = NULL; +- worker->wqe = wqe; +- spin_lock_init(&worker->lock); +- +- worker->task = kthread_create_on_node(io_wqe_worker, worker, wqe->node, +- "io_wqe_worker-%d/%d", index, wqe->node); +- if (IS_ERR(worker->task)) { +- kfree(worker); +- return false; +- } +- kthread_bind_mask(worker->task, cpumask_of_node(wqe->node)); +- +- raw_spin_lock_irq(&wqe->lock); +- hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); +- list_add_tail_rcu(&worker->all_list, &wqe->all_list); +- worker->flags |= IO_WORKER_F_FREE; +- if (index == IO_WQ_ACCT_BOUND) +- worker->flags |= IO_WORKER_F_BOUND; +- if (!acct->nr_workers && (worker->flags & IO_WORKER_F_BOUND)) +- worker->flags |= IO_WORKER_F_FIXED; +- acct->nr_workers++; +- raw_spin_unlock_irq(&wqe->lock); +- +- if (index == IO_WQ_ACCT_UNBOUND) +- atomic_inc(&wq->user->processes); +- +- refcount_inc(&wq->refs); +- wake_up_process(worker->task); +- return true; +-} +- +-static inline bool io_wqe_need_worker(struct io_wqe *wqe, int index) +- __must_hold(wqe->lock) +-{ +- struct io_wqe_acct *acct = &wqe->acct[index]; +- +- /* if we have available workers or no work, no need */ +- if (!hlist_nulls_empty(&wqe->free_list) || !io_wqe_run_queue(wqe)) +- return false; +- return acct->nr_workers < acct->max_workers; +-} +- +-static bool io_wqe_worker_send_sig(struct io_worker *worker, void *data) +-{ +- send_sig(SIGINT, worker->task, 1); +- return false; +-} +- +-/* +- * Iterate the passed in list and call the specific function for each +- * worker that isn't exiting +- */ +-static bool io_wq_for_each_worker(struct io_wqe *wqe, +- bool (*func)(struct io_worker *, void *), +- void *data) +-{ +- struct io_worker *worker; +- bool ret = false; +- +- list_for_each_entry_rcu(worker, &wqe->all_list, all_list) { +- if (io_worker_get(worker)) { +- /* no task if node is/was offline */ +- if (worker->task) +- ret = func(worker, data); +- io_worker_release(worker); +- if (ret) +- break; +- } +- } +- +- return ret; +-} +- +-static bool io_wq_worker_wake(struct io_worker *worker, void *data) +-{ +- wake_up_process(worker->task); +- return false; +-} +- +-/* +- * Manager thread. Tasked with creating new workers, if we need them. +- */ +-static int io_wq_manager(void *data) +-{ +- struct io_wq *wq = data; +- int node; +- +- /* create fixed workers */ +- refcount_set(&wq->refs, 1); +- for_each_node(node) { +- if (!node_online(node)) +- continue; +- if (create_io_worker(wq, wq->wqes[node], IO_WQ_ACCT_BOUND)) +- continue; +- set_bit(IO_WQ_BIT_ERROR, &wq->state); +- set_bit(IO_WQ_BIT_EXIT, &wq->state); +- goto out; +- } +- +- complete(&wq->done); +- +- while (!kthread_should_stop()) { +- if (current->task_works) +- task_work_run(); +- +- for_each_node(node) { +- struct io_wqe *wqe = wq->wqes[node]; +- bool fork_worker[2] = { false, false }; +- +- if (!node_online(node)) +- continue; +- +- raw_spin_lock_irq(&wqe->lock); +- if (io_wqe_need_worker(wqe, IO_WQ_ACCT_BOUND)) +- fork_worker[IO_WQ_ACCT_BOUND] = true; +- if (io_wqe_need_worker(wqe, IO_WQ_ACCT_UNBOUND)) +- fork_worker[IO_WQ_ACCT_UNBOUND] = true; +- raw_spin_unlock_irq(&wqe->lock); +- if (fork_worker[IO_WQ_ACCT_BOUND]) +- create_io_worker(wq, wqe, IO_WQ_ACCT_BOUND); +- if (fork_worker[IO_WQ_ACCT_UNBOUND]) +- create_io_worker(wq, wqe, IO_WQ_ACCT_UNBOUND); +- } +- set_current_state(TASK_INTERRUPTIBLE); +- schedule_timeout(HZ); +- } +- +- if (current->task_works) +- task_work_run(); +- +-out: +- if (refcount_dec_and_test(&wq->refs)) { +- complete(&wq->done); +- return 0; +- } +- /* if ERROR is set and we get here, we have workers to wake */ +- if (test_bit(IO_WQ_BIT_ERROR, &wq->state)) { +- rcu_read_lock(); +- for_each_node(node) +- io_wq_for_each_worker(wq->wqes[node], io_wq_worker_wake, NULL); +- rcu_read_unlock(); +- } +- return 0; +-} +- +-static bool io_wq_can_queue(struct io_wqe *wqe, struct io_wqe_acct *acct, +- struct io_wq_work *work) +-{ +- bool free_worker; +- +- if (!(work->flags & IO_WQ_WORK_UNBOUND)) +- return true; +- if (atomic_read(&acct->nr_running)) +- return true; +- +- rcu_read_lock(); +- free_worker = !hlist_nulls_empty(&wqe->free_list); +- rcu_read_unlock(); +- if (free_worker) +- return true; +- +- if (atomic_read(&wqe->wq->user->processes) >= acct->max_workers && +- !(capable(CAP_SYS_RESOURCE) || capable(CAP_SYS_ADMIN))) +- return false; +- +- return true; +-} +- +-static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe) +-{ +- struct io_wq *wq = wqe->wq; +- +- do { +- struct io_wq_work *old_work = work; +- +- work->flags |= IO_WQ_WORK_CANCEL; +- work = wq->do_work(work); +- wq->free_work(old_work); +- } while (work); +-} +- +-static void io_wqe_insert_work(struct io_wqe *wqe, struct io_wq_work *work) +-{ +- unsigned int hash; +- struct io_wq_work *tail; +- +- if (!io_wq_is_hashed(work)) { +-append: +- wq_list_add_tail(&work->list, &wqe->work_list); +- return; +- } +- +- hash = io_get_work_hash(work); +- tail = wqe->hash_tail[hash]; +- wqe->hash_tail[hash] = work; +- if (!tail) +- goto append; +- +- wq_list_add_after(&work->list, &tail->list, &wqe->work_list); +-} +- +-static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) +-{ +- struct io_wqe_acct *acct = io_work_get_acct(wqe, work); +- bool do_wake; +- unsigned long flags; +- +- /* +- * Do early check to see if we need a new unbound worker, and if we do, +- * if we're allowed to do so. This isn't 100% accurate as there's a +- * gap between this check and incrementing the value, but that's OK. +- * It's close enough to not be an issue, fork() has the same delay. +- */ +- if (unlikely(!io_wq_can_queue(wqe, acct, work))) { +- io_run_cancel(work, wqe); +- return; +- } +- +- raw_spin_lock_irqsave(&wqe->lock, flags); +- io_wqe_insert_work(wqe, work); +- wqe->flags &= ~IO_WQE_FLAG_STALLED; +- do_wake = (work->flags & IO_WQ_WORK_CONCURRENT) || +- !atomic_read(&acct->nr_running); +- raw_spin_unlock_irqrestore(&wqe->lock, flags); +- +- if (do_wake) +- io_wqe_wake_worker(wqe, acct); +-} +- +-void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work) +-{ +- struct io_wqe *wqe = wq->wqes[numa_node_id()]; +- +- io_wqe_enqueue(wqe, work); +-} +- +-/* +- * Work items that hash to the same value will not be done in parallel. +- * Used to limit concurrent writes, generally hashed by inode. +- */ +-void io_wq_hash_work(struct io_wq_work *work, void *val) +-{ +- unsigned int bit; +- +- bit = hash_ptr(val, IO_WQ_HASH_ORDER); +- work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT)); +-} +- +-void io_wq_cancel_all(struct io_wq *wq) +-{ +- int node; +- +- set_bit(IO_WQ_BIT_CANCEL, &wq->state); +- +- rcu_read_lock(); +- for_each_node(node) { +- struct io_wqe *wqe = wq->wqes[node]; +- +- io_wq_for_each_worker(wqe, io_wqe_worker_send_sig, NULL); +- } +- rcu_read_unlock(); +-} +- +-struct io_cb_cancel_data { +- work_cancel_fn *fn; +- void *data; +- int nr_running; +- int nr_pending; +- bool cancel_all; +-}; +- +-static bool io_wq_worker_cancel(struct io_worker *worker, void *data) +-{ +- struct io_cb_cancel_data *match = data; +- unsigned long flags; +- +- /* +- * Hold the lock to avoid ->cur_work going out of scope, caller +- * may dereference the passed in work. +- */ +- spin_lock_irqsave(&worker->lock, flags); +- if (worker->cur_work && +- !(worker->cur_work->flags & IO_WQ_WORK_NO_CANCEL) && +- match->fn(worker->cur_work, match->data)) { +- send_sig(SIGINT, worker->task, 1); +- match->nr_running++; +- } +- spin_unlock_irqrestore(&worker->lock, flags); +- +- return match->nr_running && !match->cancel_all; +-} +- +-static inline void io_wqe_remove_pending(struct io_wqe *wqe, +- struct io_wq_work *work, +- struct io_wq_work_node *prev) +-{ +- unsigned int hash = io_get_work_hash(work); +- struct io_wq_work *prev_work = NULL; +- +- if (io_wq_is_hashed(work) && work == wqe->hash_tail[hash]) { +- if (prev) +- prev_work = container_of(prev, struct io_wq_work, list); +- if (prev_work && io_get_work_hash(prev_work) == hash) +- wqe->hash_tail[hash] = prev_work; +- else +- wqe->hash_tail[hash] = NULL; +- } +- wq_list_del(&wqe->work_list, &work->list, prev); +-} +- +-static void io_wqe_cancel_pending_work(struct io_wqe *wqe, +- struct io_cb_cancel_data *match) +-{ +- struct io_wq_work_node *node, *prev; +- struct io_wq_work *work; +- unsigned long flags; +- +-retry: +- raw_spin_lock_irqsave(&wqe->lock, flags); +- wq_list_for_each(node, prev, &wqe->work_list) { +- work = container_of(node, struct io_wq_work, list); +- if (!match->fn(work, match->data)) +- continue; +- io_wqe_remove_pending(wqe, work, prev); +- raw_spin_unlock_irqrestore(&wqe->lock, flags); +- io_run_cancel(work, wqe); +- match->nr_pending++; +- if (!match->cancel_all) +- return; +- +- /* not safe to continue after unlock */ +- goto retry; +- } +- raw_spin_unlock_irqrestore(&wqe->lock, flags); +-} +- +-static void io_wqe_cancel_running_work(struct io_wqe *wqe, +- struct io_cb_cancel_data *match) +-{ +- rcu_read_lock(); +- io_wq_for_each_worker(wqe, io_wq_worker_cancel, match); +- rcu_read_unlock(); +-} +- +-enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, +- void *data, bool cancel_all) +-{ +- struct io_cb_cancel_data match = { +- .fn = cancel, +- .data = data, +- .cancel_all = cancel_all, +- }; +- int node; +- +- /* +- * First check pending list, if we're lucky we can just remove it +- * from there. CANCEL_OK means that the work is returned as-new, +- * no completion will be posted for it. +- */ +- for_each_node(node) { +- struct io_wqe *wqe = wq->wqes[node]; +- +- io_wqe_cancel_pending_work(wqe, &match); +- if (match.nr_pending && !match.cancel_all) +- return IO_WQ_CANCEL_OK; +- } +- +- /* +- * Now check if a free (going busy) or busy worker has the work +- * currently running. If we find it there, we'll return CANCEL_RUNNING +- * as an indication that we attempt to signal cancellation. The +- * completion will run normally in this case. +- */ +- for_each_node(node) { +- struct io_wqe *wqe = wq->wqes[node]; +- +- io_wqe_cancel_running_work(wqe, &match); +- if (match.nr_running && !match.cancel_all) +- return IO_WQ_CANCEL_RUNNING; +- } +- +- if (match.nr_running) +- return IO_WQ_CANCEL_RUNNING; +- if (match.nr_pending) +- return IO_WQ_CANCEL_OK; +- return IO_WQ_CANCEL_NOTFOUND; +-} +- +-struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) +-{ +- int ret = -ENOMEM, node; +- struct io_wq *wq; +- +- if (WARN_ON_ONCE(!data->free_work || !data->do_work)) +- return ERR_PTR(-EINVAL); +- if (WARN_ON_ONCE(!bounded)) +- return ERR_PTR(-EINVAL); +- +- wq = kzalloc(sizeof(*wq), GFP_KERNEL); +- if (!wq) +- return ERR_PTR(-ENOMEM); +- +- wq->wqes = kcalloc(nr_node_ids, sizeof(struct io_wqe *), GFP_KERNEL); +- if (!wq->wqes) +- goto err_wq; +- +- ret = cpuhp_state_add_instance_nocalls(io_wq_online, &wq->cpuhp_node); +- if (ret) +- goto err_wqes; +- +- wq->free_work = data->free_work; +- wq->do_work = data->do_work; +- +- /* caller must already hold a reference to this */ +- wq->user = data->user; +- +- ret = -ENOMEM; +- for_each_node(node) { +- struct io_wqe *wqe; +- int alloc_node = node; +- +- if (!node_online(alloc_node)) +- alloc_node = NUMA_NO_NODE; +- wqe = kzalloc_node(sizeof(struct io_wqe), GFP_KERNEL, alloc_node); +- if (!wqe) +- goto err; +- wq->wqes[node] = wqe; +- wqe->node = alloc_node; +- wqe->acct[IO_WQ_ACCT_BOUND].max_workers = bounded; +- atomic_set(&wqe->acct[IO_WQ_ACCT_BOUND].nr_running, 0); +- if (wq->user) { +- wqe->acct[IO_WQ_ACCT_UNBOUND].max_workers = +- task_rlimit(current, RLIMIT_NPROC); +- } +- atomic_set(&wqe->acct[IO_WQ_ACCT_UNBOUND].nr_running, 0); +- wqe->wq = wq; +- raw_spin_lock_init(&wqe->lock); +- INIT_WQ_LIST(&wqe->work_list); +- INIT_HLIST_NULLS_HEAD(&wqe->free_list, 0); +- INIT_LIST_HEAD(&wqe->all_list); +- } +- +- init_completion(&wq->done); +- +- wq->manager = kthread_create(io_wq_manager, wq, "io_wq_manager"); +- if (!IS_ERR(wq->manager)) { +- wake_up_process(wq->manager); +- wait_for_completion(&wq->done); +- if (test_bit(IO_WQ_BIT_ERROR, &wq->state)) { +- ret = -ENOMEM; +- goto err; +- } +- refcount_set(&wq->use_refs, 1); +- reinit_completion(&wq->done); +- return wq; +- } +- +- ret = PTR_ERR(wq->manager); +- complete(&wq->done); +-err: +- cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); +- for_each_node(node) +- kfree(wq->wqes[node]); +-err_wqes: +- kfree(wq->wqes); +-err_wq: +- kfree(wq); +- return ERR_PTR(ret); +-} +- +-bool io_wq_get(struct io_wq *wq, struct io_wq_data *data) +-{ +- if (data->free_work != wq->free_work || data->do_work != wq->do_work) +- return false; +- +- return refcount_inc_not_zero(&wq->use_refs); +-} +- +-static void __io_wq_destroy(struct io_wq *wq) +-{ +- int node; +- +- cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); +- +- set_bit(IO_WQ_BIT_EXIT, &wq->state); +- if (wq->manager) +- kthread_stop(wq->manager); +- +- rcu_read_lock(); +- for_each_node(node) +- io_wq_for_each_worker(wq->wqes[node], io_wq_worker_wake, NULL); +- rcu_read_unlock(); +- +- wait_for_completion(&wq->done); +- +- for_each_node(node) +- kfree(wq->wqes[node]); +- kfree(wq->wqes); +- kfree(wq); +-} +- +-void io_wq_destroy(struct io_wq *wq) +-{ +- if (refcount_dec_and_test(&wq->use_refs)) +- __io_wq_destroy(wq); +-} +- +-struct task_struct *io_wq_get_task(struct io_wq *wq) +-{ +- return wq->manager; +-} +- +-static bool io_wq_worker_affinity(struct io_worker *worker, void *data) +-{ +- struct task_struct *task = worker->task; +- struct rq_flags rf; +- struct rq *rq; +- +- rq = task_rq_lock(task, &rf); +- do_set_cpus_allowed(task, cpumask_of_node(worker->wqe->node)); +- task->flags |= PF_NO_SETAFFINITY; +- task_rq_unlock(rq, task, &rf); +- return false; +-} +- +-static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node) +-{ +- struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); +- int i; +- +- rcu_read_lock(); +- for_each_node(i) +- io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, NULL); +- rcu_read_unlock(); +- return 0; +-} +- +-static __init int io_wq_init(void) +-{ +- int ret; +- +- ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online", +- io_wq_cpu_online, NULL); +- if (ret < 0) +- return ret; +- io_wq_online = ret; +- return 0; +-} +-subsys_initcall(io_wq_init); +--- a/fs/io-wq.h ++++ /dev/null +@@ -1,157 +0,0 @@ +-#ifndef INTERNAL_IO_WQ_H +-#define INTERNAL_IO_WQ_H +- +-#include +- +-struct io_wq; +- +-enum { +- IO_WQ_WORK_CANCEL = 1, +- IO_WQ_WORK_HASHED = 2, +- IO_WQ_WORK_UNBOUND = 4, +- IO_WQ_WORK_NO_CANCEL = 8, +- IO_WQ_WORK_CONCURRENT = 16, +- +- IO_WQ_WORK_FILES = 32, +- IO_WQ_WORK_FS = 64, +- IO_WQ_WORK_MM = 128, +- IO_WQ_WORK_CREDS = 256, +- IO_WQ_WORK_BLKCG = 512, +- IO_WQ_WORK_FSIZE = 1024, +- +- IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */ +-}; +- +-enum io_wq_cancel { +- IO_WQ_CANCEL_OK, /* cancelled before started */ +- IO_WQ_CANCEL_RUNNING, /* found, running, and attempted cancelled */ +- IO_WQ_CANCEL_NOTFOUND, /* work not found */ +-}; +- +-struct io_wq_work_node { +- struct io_wq_work_node *next; +-}; +- +-struct io_wq_work_list { +- struct io_wq_work_node *first; +- struct io_wq_work_node *last; +-}; +- +-static inline void wq_list_add_after(struct io_wq_work_node *node, +- struct io_wq_work_node *pos, +- struct io_wq_work_list *list) +-{ +- struct io_wq_work_node *next = pos->next; +- +- pos->next = node; +- node->next = next; +- if (!next) +- list->last = node; +-} +- +-static inline void wq_list_add_tail(struct io_wq_work_node *node, +- struct io_wq_work_list *list) +-{ +- if (!list->first) { +- list->last = node; +- WRITE_ONCE(list->first, node); +- } else { +- list->last->next = node; +- list->last = node; +- } +- node->next = NULL; +-} +- +-static inline void wq_list_cut(struct io_wq_work_list *list, +- struct io_wq_work_node *last, +- struct io_wq_work_node *prev) +-{ +- /* first in the list, if prev==NULL */ +- if (!prev) +- WRITE_ONCE(list->first, last->next); +- else +- prev->next = last->next; +- +- if (last == list->last) +- list->last = prev; +- last->next = NULL; +-} +- +-static inline void wq_list_del(struct io_wq_work_list *list, +- struct io_wq_work_node *node, +- struct io_wq_work_node *prev) +-{ +- wq_list_cut(list, node, prev); +-} +- +-#define wq_list_for_each(pos, prv, head) \ +- for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next) +- +-#define wq_list_empty(list) (READ_ONCE((list)->first) == NULL) +-#define INIT_WQ_LIST(list) do { \ +- (list)->first = NULL; \ +- (list)->last = NULL; \ +-} while (0) +- +-struct io_wq_work { +- struct io_wq_work_node list; +- struct io_identity *identity; +- unsigned flags; +-}; +- +-static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) +-{ +- if (!work->list.next) +- return NULL; +- +- return container_of(work->list.next, struct io_wq_work, list); +-} +- +-typedef void (free_work_fn)(struct io_wq_work *); +-typedef struct io_wq_work *(io_wq_work_fn)(struct io_wq_work *); +- +-struct io_wq_data { +- struct user_struct *user; +- +- io_wq_work_fn *do_work; +- free_work_fn *free_work; +-}; +- +-struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); +-bool io_wq_get(struct io_wq *wq, struct io_wq_data *data); +-void io_wq_destroy(struct io_wq *wq); +- +-void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); +-void io_wq_hash_work(struct io_wq_work *work, void *val); +- +-static inline bool io_wq_is_hashed(struct io_wq_work *work) +-{ +- return work->flags & IO_WQ_WORK_HASHED; +-} +- +-void io_wq_cancel_all(struct io_wq *wq); +- +-typedef bool (work_cancel_fn)(struct io_wq_work *, void *); +- +-enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, +- void *data, bool cancel_all); +- +-struct task_struct *io_wq_get_task(struct io_wq *wq); +- +-#if defined(CONFIG_IO_WQ) +-extern void io_wq_worker_sleeping(struct task_struct *); +-extern void io_wq_worker_running(struct task_struct *); +-#else +-static inline void io_wq_worker_sleeping(struct task_struct *tsk) +-{ +-} +-static inline void io_wq_worker_running(struct task_struct *tsk) +-{ +-} +-#endif +- +-static inline bool io_wq_current_is_worker(void) +-{ +- return in_task() && (current->flags & PF_IO_WORKER); +-} +-#endif +--- a/fs/io_uring.c ++++ /dev/null +@@ -1,9971 +0,0 @@ +-// SPDX-License-Identifier: GPL-2.0 +-/* +- * Shared application/kernel submission and completion ring pairs, for +- * supporting fast/efficient IO. +- * +- * A note on the read/write ordering memory barriers that are matched between +- * the application and kernel side. +- * +- * After the application reads the CQ ring tail, it must use an +- * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses +- * before writing the tail (using smp_load_acquire to read the tail will +- * do). It also needs a smp_mb() before updating CQ head (ordering the +- * entry load(s) with the head store), pairing with an implicit barrier +- * through a control-dependency in io_get_cqring (smp_store_release to +- * store head will do). Failure to do so could lead to reading invalid +- * CQ entries. +- * +- * Likewise, the application must use an appropriate smp_wmb() before +- * writing the SQ tail (ordering SQ entry stores with the tail store), +- * which pairs with smp_load_acquire in io_get_sqring (smp_store_release +- * to store the tail will do). And it needs a barrier ordering the SQ +- * head load before writing new SQ entries (smp_load_acquire to read +- * head will do). +- * +- * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application +- * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after* +- * updating the SQ tail; a full memory barrier smp_mb() is needed +- * between. +- * +- * Also see the examples in the liburing library: +- * +- * git://git.kernel.dk/liburing +- * +- * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens +- * from data shared between the kernel and application. This is done both +- * for ordering purposes, but also to ensure that once a value is loaded from +- * data that the application could potentially modify, it remains stable. +- * +- * Copyright (C) 2018-2019 Jens Axboe +- * Copyright (c) 2018-2019 Christoph Hellwig +- */ +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#define CREATE_TRACE_POINTS +-#include +- +-#include +- +-#include "internal.h" +-#include "io-wq.h" +- +-#define IORING_MAX_ENTRIES 32768 +-#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) +- +-/* +- * Shift of 9 is 512 entries, or exactly one page on 64-bit archs +- */ +-#define IORING_FILE_TABLE_SHIFT 9 +-#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT) +-#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1) +-#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE) +-#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ +- IORING_REGISTER_LAST + IORING_OP_LAST) +- +-struct io_uring { +- u32 head ____cacheline_aligned_in_smp; +- u32 tail ____cacheline_aligned_in_smp; +-}; +- +-/* +- * This data is shared with the application through the mmap at offsets +- * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING. +- * +- * The offsets to the member fields are published through struct +- * io_sqring_offsets when calling io_uring_setup. +- */ +-struct io_rings { +- /* +- * Head and tail offsets into the ring; the offsets need to be +- * masked to get valid indices. +- * +- * The kernel controls head of the sq ring and the tail of the cq ring, +- * and the application controls tail of the sq ring and the head of the +- * cq ring. +- */ +- struct io_uring sq, cq; +- /* +- * Bitmasks to apply to head and tail offsets (constant, equals +- * ring_entries - 1) +- */ +- u32 sq_ring_mask, cq_ring_mask; +- /* Ring sizes (constant, power of 2) */ +- u32 sq_ring_entries, cq_ring_entries; +- /* +- * Number of invalid entries dropped by the kernel due to +- * invalid index stored in array +- * +- * Written by the kernel, shouldn't be modified by the +- * application (i.e. get number of "new events" by comparing to +- * cached value). +- * +- * After a new SQ head value was read by the application this +- * counter includes all submissions that were dropped reaching +- * the new SQ head (and possibly more). +- */ +- u32 sq_dropped; +- /* +- * Runtime SQ flags +- * +- * Written by the kernel, shouldn't be modified by the +- * application. +- * +- * The application needs a full memory barrier before checking +- * for IORING_SQ_NEED_WAKEUP after updating the sq tail. +- */ +- u32 sq_flags; +- /* +- * Runtime CQ flags +- * +- * Written by the application, shouldn't be modified by the +- * kernel. +- */ +- u32 cq_flags; +- /* +- * Number of completion events lost because the queue was full; +- * this should be avoided by the application by making sure +- * there are not more requests pending than there is space in +- * the completion queue. +- * +- * Written by the kernel, shouldn't be modified by the +- * application (i.e. get number of "new events" by comparing to +- * cached value). +- * +- * As completion events come in out of order this counter is not +- * ordered with any other data. +- */ +- u32 cq_overflow; +- /* +- * Ring buffer of completion events. +- * +- * The kernel writes completion events fresh every time they are +- * produced, so the application is allowed to modify pending +- * entries. +- */ +- struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp; +-}; +- +-struct io_mapped_ubuf { +- u64 ubuf; +- size_t len; +- struct bio_vec *bvec; +- unsigned int nr_bvecs; +- unsigned long acct_pages; +-}; +- +-struct fixed_file_table { +- struct file **files; +-}; +- +-struct fixed_file_ref_node { +- struct percpu_ref refs; +- struct list_head node; +- struct list_head file_list; +- struct fixed_file_data *file_data; +- struct llist_node llist; +- bool done; +-}; +- +-struct fixed_file_data { +- struct fixed_file_table *table; +- struct io_ring_ctx *ctx; +- +- struct fixed_file_ref_node *node; +- struct percpu_ref refs; +- struct completion done; +- struct list_head ref_list; +- spinlock_t lock; +- bool quiesce; +-}; +- +-struct io_buffer { +- struct list_head list; +- __u64 addr; +- __u32 len; +- __u16 bid; +-}; +- +-struct io_restriction { +- DECLARE_BITMAP(register_op, IORING_REGISTER_LAST); +- DECLARE_BITMAP(sqe_op, IORING_OP_LAST); +- u8 sqe_flags_allowed; +- u8 sqe_flags_required; +- bool registered; +-}; +- +-struct io_sq_data { +- refcount_t refs; +- struct mutex lock; +- +- /* ctx's that are using this sqd */ +- struct list_head ctx_list; +- struct list_head ctx_new_list; +- struct mutex ctx_lock; +- +- struct task_struct *thread; +- struct wait_queue_head wait; +-}; +- +-struct io_ring_ctx { +- struct { +- struct percpu_ref refs; +- } ____cacheline_aligned_in_smp; +- +- struct { +- unsigned int flags; +- unsigned int compat: 1; +- unsigned int limit_mem: 1; +- unsigned int cq_overflow_flushed: 1; +- unsigned int drain_next: 1; +- unsigned int eventfd_async: 1; +- unsigned int restricted: 1; +- unsigned int sqo_dead: 1; +- +- /* +- * Ring buffer of indices into array of io_uring_sqe, which is +- * mmapped by the application using the IORING_OFF_SQES offset. +- * +- * This indirection could e.g. be used to assign fixed +- * io_uring_sqe entries to operations and only submit them to +- * the queue when needed. +- * +- * The kernel modifies neither the indices array nor the entries +- * array. +- */ +- u32 *sq_array; +- unsigned cached_sq_head; +- unsigned sq_entries; +- unsigned sq_mask; +- unsigned sq_thread_idle; +- unsigned cached_sq_dropped; +- unsigned cached_cq_overflow; +- unsigned long sq_check_overflow; +- +- struct list_head defer_list; +- struct list_head timeout_list; +- struct list_head cq_overflow_list; +- +- struct io_uring_sqe *sq_sqes; +- } ____cacheline_aligned_in_smp; +- +- struct io_rings *rings; +- +- /* IO offload */ +- struct io_wq *io_wq; +- +- /* +- * For SQPOLL usage - we hold a reference to the parent task, so we +- * have access to the ->files +- */ +- struct task_struct *sqo_task; +- +- /* Only used for accounting purposes */ +- struct mm_struct *mm_account; +- +-#ifdef CONFIG_BLK_CGROUP +- struct cgroup_subsys_state *sqo_blkcg_css; +-#endif +- +- struct io_sq_data *sq_data; /* if using sq thread polling */ +- +- struct wait_queue_head sqo_sq_wait; +- struct wait_queue_entry sqo_wait_entry; +- struct list_head sqd_list; +- +- /* +- * If used, fixed file set. Writers must ensure that ->refs is dead, +- * readers must ensure that ->refs is alive as long as the file* is +- * used. Only updated through io_uring_register(2). +- */ +- struct fixed_file_data *file_data; +- unsigned nr_user_files; +- +- /* if used, fixed mapped user buffers */ +- unsigned nr_user_bufs; +- struct io_mapped_ubuf *user_bufs; +- +- struct user_struct *user; +- +- const struct cred *creds; +- +-#ifdef CONFIG_AUDIT +- kuid_t loginuid; +- unsigned int sessionid; +-#endif +- +- struct completion ref_comp; +- struct completion sq_thread_comp; +- +- /* if all else fails... */ +- struct io_kiocb *fallback_req; +- +-#if defined(CONFIG_UNIX) +- struct socket *ring_sock; +-#endif +- +- struct xarray io_buffers; +- +- struct xarray personalities; +- u32 pers_next; +- +- struct { +- unsigned cached_cq_tail; +- unsigned cq_entries; +- unsigned cq_mask; +- atomic_t cq_timeouts; +- unsigned cq_last_tm_flush; +- unsigned long cq_check_overflow; +- struct wait_queue_head cq_wait; +- struct fasync_struct *cq_fasync; +- struct eventfd_ctx *cq_ev_fd; +- } ____cacheline_aligned_in_smp; +- +- struct { +- struct mutex uring_lock; +- wait_queue_head_t wait; +- } ____cacheline_aligned_in_smp; +- +- struct { +- spinlock_t completion_lock; +- +- /* +- * ->iopoll_list is protected by the ctx->uring_lock for +- * io_uring instances that don't use IORING_SETUP_SQPOLL. +- * For SQPOLL, only the single threaded io_sq_thread() will +- * manipulate the list, hence no extra locking is needed there. +- */ +- struct list_head iopoll_list; +- struct hlist_head *cancel_hash; +- unsigned cancel_hash_bits; +- bool poll_multi_file; +- +- spinlock_t inflight_lock; +- struct list_head inflight_list; +- } ____cacheline_aligned_in_smp; +- +- struct delayed_work file_put_work; +- struct llist_head file_put_llist; +- +- struct work_struct exit_work; +- struct io_restriction restrictions; +-}; +- +-/* +- * First field must be the file pointer in all the +- * iocb unions! See also 'struct kiocb' in +- */ +-struct io_poll_iocb { +- struct file *file; +- union { +- struct wait_queue_head *head; +- u64 addr; +- }; +- __poll_t events; +- bool done; +- bool canceled; +- struct wait_queue_entry wait; +-}; +- +-struct io_close { +- struct file *file; +- struct file *put_file; +- int fd; +-}; +- +-struct io_timeout_data { +- struct io_kiocb *req; +- struct hrtimer timer; +- struct timespec64 ts; +- enum hrtimer_mode mode; +-}; +- +-struct io_accept { +- struct file *file; +- struct sockaddr __user *addr; +- int __user *addr_len; +- int flags; +- unsigned long nofile; +-}; +- +-struct io_sync { +- struct file *file; +- loff_t len; +- loff_t off; +- int flags; +- int mode; +-}; +- +-struct io_cancel { +- struct file *file; +- u64 addr; +-}; +- +-struct io_timeout { +- struct file *file; +- u32 off; +- u32 target_seq; +- struct list_head list; +-}; +- +-struct io_timeout_rem { +- struct file *file; +- u64 addr; +-}; +- +-struct io_rw { +- /* NOTE: kiocb has the file as the first member, so don't do it here */ +- struct kiocb kiocb; +- u64 addr; +- u64 len; +-}; +- +-struct io_connect { +- struct file *file; +- struct sockaddr __user *addr; +- int addr_len; +-}; +- +-struct io_sr_msg { +- struct file *file; +- union { +- struct user_msghdr __user *umsg; +- void __user *buf; +- }; +- int msg_flags; +- int bgid; +- size_t len; +- struct io_buffer *kbuf; +-}; +- +-struct io_open { +- struct file *file; +- int dfd; +- bool ignore_nonblock; +- struct filename *filename; +- struct open_how how; +- unsigned long nofile; +-}; +- +-struct io_files_update { +- struct file *file; +- u64 arg; +- u32 nr_args; +- u32 offset; +-}; +- +-struct io_fadvise { +- struct file *file; +- u64 offset; +- u32 len; +- u32 advice; +-}; +- +-struct io_madvise { +- struct file *file; +- u64 addr; +- u32 len; +- u32 advice; +-}; +- +-struct io_epoll { +- struct file *file; +- int epfd; +- int op; +- int fd; +- struct epoll_event event; +-}; +- +-struct io_splice { +- struct file *file_out; +- struct file *file_in; +- loff_t off_out; +- loff_t off_in; +- u64 len; +- unsigned int flags; +-}; +- +-struct io_provide_buf { +- struct file *file; +- __u64 addr; +- __u32 len; +- __u32 bgid; +- __u16 nbufs; +- __u16 bid; +-}; +- +-struct io_statx { +- struct file *file; +- int dfd; +- unsigned int mask; +- unsigned int flags; +- const char __user *filename; +- struct statx __user *buffer; +-}; +- +-struct io_completion { +- struct file *file; +- struct list_head list; +- u32 cflags; +-}; +- +-struct io_async_connect { +- struct sockaddr_storage address; +-}; +- +-struct io_async_msghdr { +- struct iovec fast_iov[UIO_FASTIOV]; +- struct iovec *iov; +- struct sockaddr __user *uaddr; +- struct msghdr msg; +- struct sockaddr_storage addr; +-}; +- +-struct io_async_rw { +- struct iovec fast_iov[UIO_FASTIOV]; +- const struct iovec *free_iovec; +- struct iov_iter iter; +- size_t bytes_done; +- struct wait_page_queue wpq; +-}; +- +-enum { +- REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT, +- REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT, +- REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT, +- REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT, +- REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT, +- REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT, +- +- REQ_F_LINK_HEAD_BIT, +- REQ_F_FAIL_LINK_BIT, +- REQ_F_INFLIGHT_BIT, +- REQ_F_CUR_POS_BIT, +- REQ_F_NOWAIT_BIT, +- REQ_F_LINK_TIMEOUT_BIT, +- REQ_F_ISREG_BIT, +- REQ_F_NEED_CLEANUP_BIT, +- REQ_F_POLLED_BIT, +- REQ_F_BUFFER_SELECTED_BIT, +- REQ_F_NO_FILE_TABLE_BIT, +- REQ_F_WORK_INITIALIZED_BIT, +- REQ_F_LTIMEOUT_ACTIVE_BIT, +- +- /* not a real bit, just to check we're not overflowing the space */ +- __REQ_F_LAST_BIT, +-}; +- +-enum { +- /* ctx owns file */ +- REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT), +- /* drain existing IO first */ +- REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT), +- /* linked sqes */ +- REQ_F_LINK = BIT(REQ_F_LINK_BIT), +- /* doesn't sever on completion < 0 */ +- REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT), +- /* IOSQE_ASYNC */ +- REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT), +- /* IOSQE_BUFFER_SELECT */ +- REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT), +- +- /* head of a link */ +- REQ_F_LINK_HEAD = BIT(REQ_F_LINK_HEAD_BIT), +- /* fail rest of links */ +- REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT), +- /* on inflight list */ +- REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT), +- /* read/write uses file position */ +- REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT), +- /* must not punt to workers */ +- REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT), +- /* has or had linked timeout */ +- REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT), +- /* regular file */ +- REQ_F_ISREG = BIT(REQ_F_ISREG_BIT), +- /* needs cleanup */ +- REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT), +- /* already went through poll handler */ +- REQ_F_POLLED = BIT(REQ_F_POLLED_BIT), +- /* buffer already selected */ +- REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT), +- /* doesn't need file table for this request */ +- REQ_F_NO_FILE_TABLE = BIT(REQ_F_NO_FILE_TABLE_BIT), +- /* io_wq_work is initialized */ +- REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT), +- /* linked timeout is active, i.e. prepared by link's head */ +- REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT), +-}; +- +-struct async_poll { +- struct io_poll_iocb poll; +- struct io_poll_iocb *double_poll; +-}; +- +-/* +- * NOTE! Each of the iocb union members has the file pointer +- * as the first entry in their struct definition. So you can +- * access the file pointer through any of the sub-structs, +- * or directly as just 'ki_filp' in this struct. +- */ +-struct io_kiocb { +- union { +- struct file *file; +- struct io_rw rw; +- struct io_poll_iocb poll; +- struct io_accept accept; +- struct io_sync sync; +- struct io_cancel cancel; +- struct io_timeout timeout; +- struct io_timeout_rem timeout_rem; +- struct io_connect connect; +- struct io_sr_msg sr_msg; +- struct io_open open; +- struct io_close close; +- struct io_files_update files_update; +- struct io_fadvise fadvise; +- struct io_madvise madvise; +- struct io_epoll epoll; +- struct io_splice splice; +- struct io_provide_buf pbuf; +- struct io_statx statx; +- /* use only after cleaning per-op data, see io_clean_op() */ +- struct io_completion compl; +- }; +- +- /* opcode allocated if it needs to store data for async defer */ +- void *async_data; +- u8 opcode; +- /* polled IO has completed */ +- u8 iopoll_completed; +- +- u16 buf_index; +- u32 result; +- +- struct io_ring_ctx *ctx; +- unsigned int flags; +- refcount_t refs; +- struct task_struct *task; +- u64 user_data; +- +- struct list_head link_list; +- +- /* +- * 1. used with ctx->iopoll_list with reads/writes +- * 2. to track reqs with ->files (see io_op_def::file_table) +- */ +- struct list_head inflight_entry; +- +- struct list_head iopoll_entry; +- +- struct percpu_ref *fixed_file_refs; +- struct callback_head task_work; +- /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ +- struct hlist_node hash_node; +- struct async_poll *apoll; +- struct io_wq_work work; +-}; +- +-struct io_defer_entry { +- struct list_head list; +- struct io_kiocb *req; +- u32 seq; +-}; +- +-#define IO_IOPOLL_BATCH 8 +- +-struct io_comp_state { +- unsigned int nr; +- struct list_head list; +- struct io_ring_ctx *ctx; +-}; +- +-struct io_submit_state { +- struct blk_plug plug; +- +- /* +- * io_kiocb alloc cache +- */ +- void *reqs[IO_IOPOLL_BATCH]; +- unsigned int free_reqs; +- +- /* +- * Batch completion logic +- */ +- struct io_comp_state comp; +- +- /* +- * File reference cache +- */ +- struct file *file; +- unsigned int fd; +- unsigned int has_refs; +- unsigned int ios_left; +-}; +- +-struct io_op_def { +- /* needs req->file assigned */ +- unsigned needs_file : 1; +- /* don't fail if file grab fails */ +- unsigned needs_file_no_error : 1; +- /* hash wq insertion if file is a regular file */ +- unsigned hash_reg_file : 1; +- /* unbound wq insertion if file is a non-regular file */ +- unsigned unbound_nonreg_file : 1; +- /* opcode is not supported by this kernel */ +- unsigned not_supported : 1; +- /* set if opcode supports polled "wait" */ +- unsigned pollin : 1; +- unsigned pollout : 1; +- /* op supports buffer selection */ +- unsigned buffer_select : 1; +- /* must always have async data allocated */ +- unsigned needs_async_data : 1; +- /* size of async data needed, if any */ +- unsigned short async_size; +- unsigned work_flags; +-}; +- +-static const struct io_op_def io_op_defs[] = { +- [IORING_OP_NOP] = {}, +- [IORING_OP_READV] = { +- .needs_file = 1, +- .unbound_nonreg_file = 1, +- .pollin = 1, +- .buffer_select = 1, +- .needs_async_data = 1, +- .async_size = sizeof(struct io_async_rw), +- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | +- IO_WQ_WORK_FILES, +- }, +- [IORING_OP_WRITEV] = { +- .needs_file = 1, +- .hash_reg_file = 1, +- .unbound_nonreg_file = 1, +- .pollout = 1, +- .needs_async_data = 1, +- .async_size = sizeof(struct io_async_rw), +- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | +- IO_WQ_WORK_FSIZE | IO_WQ_WORK_FILES, +- }, +- [IORING_OP_FSYNC] = { +- .needs_file = 1, +- .work_flags = IO_WQ_WORK_BLKCG, +- }, +- [IORING_OP_READ_FIXED] = { +- .needs_file = 1, +- .unbound_nonreg_file = 1, +- .pollin = 1, +- .async_size = sizeof(struct io_async_rw), +- .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_MM | +- IO_WQ_WORK_FILES, +- }, +- [IORING_OP_WRITE_FIXED] = { +- .needs_file = 1, +- .hash_reg_file = 1, +- .unbound_nonreg_file = 1, +- .pollout = 1, +- .async_size = sizeof(struct io_async_rw), +- .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE | +- IO_WQ_WORK_MM | IO_WQ_WORK_FILES, +- }, +- [IORING_OP_POLL_ADD] = { +- .needs_file = 1, +- .unbound_nonreg_file = 1, +- }, +- [IORING_OP_POLL_REMOVE] = {}, +- [IORING_OP_SYNC_FILE_RANGE] = { +- .needs_file = 1, +- .work_flags = IO_WQ_WORK_BLKCG, +- }, +- [IORING_OP_SENDMSG] = { +- .needs_file = 1, +- .unbound_nonreg_file = 1, +- .pollout = 1, +- .needs_async_data = 1, +- .async_size = sizeof(struct io_async_msghdr), +- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | +- IO_WQ_WORK_FS, +- }, +- [IORING_OP_RECVMSG] = { +- .needs_file = 1, +- .unbound_nonreg_file = 1, +- .pollin = 1, +- .buffer_select = 1, +- .needs_async_data = 1, +- .async_size = sizeof(struct io_async_msghdr), +- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | +- IO_WQ_WORK_FS, +- }, +- [IORING_OP_TIMEOUT] = { +- .needs_async_data = 1, +- .async_size = sizeof(struct io_timeout_data), +- .work_flags = IO_WQ_WORK_MM, +- }, +- [IORING_OP_TIMEOUT_REMOVE] = {}, +- [IORING_OP_ACCEPT] = { +- .needs_file = 1, +- .unbound_nonreg_file = 1, +- .pollin = 1, +- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_FILES, +- }, +- [IORING_OP_ASYNC_CANCEL] = {}, +- [IORING_OP_LINK_TIMEOUT] = { +- .needs_async_data = 1, +- .async_size = sizeof(struct io_timeout_data), +- .work_flags = IO_WQ_WORK_MM, +- }, +- [IORING_OP_CONNECT] = { +- .needs_file = 1, +- .unbound_nonreg_file = 1, +- .pollout = 1, +- .needs_async_data = 1, +- .async_size = sizeof(struct io_async_connect), +- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_FS, +- }, +- [IORING_OP_FALLOCATE] = { +- .needs_file = 1, +- .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE, +- }, +- [IORING_OP_OPENAT] = { +- .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_BLKCG | +- IO_WQ_WORK_FS, +- }, +- [IORING_OP_CLOSE] = { +- .needs_file = 1, +- .needs_file_no_error = 1, +- .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_BLKCG, +- }, +- [IORING_OP_FILES_UPDATE] = { +- .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_MM, +- }, +- [IORING_OP_STATX] = { +- .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_MM | +- IO_WQ_WORK_FS | IO_WQ_WORK_BLKCG, +- }, +- [IORING_OP_READ] = { +- .needs_file = 1, +- .unbound_nonreg_file = 1, +- .pollin = 1, +- .buffer_select = 1, +- .async_size = sizeof(struct io_async_rw), +- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | +- IO_WQ_WORK_FILES, +- }, +- [IORING_OP_WRITE] = { +- .needs_file = 1, +- .hash_reg_file = 1, +- .unbound_nonreg_file = 1, +- .pollout = 1, +- .async_size = sizeof(struct io_async_rw), +- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | +- IO_WQ_WORK_FSIZE | IO_WQ_WORK_FILES, +- }, +- [IORING_OP_FADVISE] = { +- .needs_file = 1, +- .work_flags = IO_WQ_WORK_BLKCG, +- }, +- [IORING_OP_MADVISE] = { +- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG, +- }, +- [IORING_OP_SEND] = { +- .needs_file = 1, +- .unbound_nonreg_file = 1, +- .pollout = 1, +- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | +- IO_WQ_WORK_FS, +- }, +- [IORING_OP_RECV] = { +- .needs_file = 1, +- .unbound_nonreg_file = 1, +- .pollin = 1, +- .buffer_select = 1, +- .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | +- IO_WQ_WORK_FS, +- }, +- [IORING_OP_OPENAT2] = { +- .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_FS | +- IO_WQ_WORK_BLKCG, +- }, +- [IORING_OP_EPOLL_CTL] = { +- .unbound_nonreg_file = 1, +- .work_flags = IO_WQ_WORK_FILES, +- }, +- [IORING_OP_SPLICE] = { +- .needs_file = 1, +- .hash_reg_file = 1, +- .unbound_nonreg_file = 1, +- .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FILES, +- }, +- [IORING_OP_PROVIDE_BUFFERS] = {}, +- [IORING_OP_REMOVE_BUFFERS] = {}, +- [IORING_OP_TEE] = { +- .needs_file = 1, +- .hash_reg_file = 1, +- .unbound_nonreg_file = 1, +- }, +-}; +- +-enum io_mem_account { +- ACCT_LOCKED, +- ACCT_PINNED, +-}; +- +-static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node); +-static struct fixed_file_ref_node *alloc_fixed_file_ref_node( +- struct io_ring_ctx *ctx); +- +-static void __io_complete_rw(struct io_kiocb *req, long res, long res2, +- struct io_comp_state *cs); +-static void io_cqring_fill_event(struct io_kiocb *req, long res); +-static void io_put_req(struct io_kiocb *req); +-static void io_put_req_deferred(struct io_kiocb *req, int nr); +-static void io_double_put_req(struct io_kiocb *req); +-static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req); +-static void __io_queue_linked_timeout(struct io_kiocb *req); +-static void io_queue_linked_timeout(struct io_kiocb *req); +-static int __io_sqe_files_update(struct io_ring_ctx *ctx, +- struct io_uring_files_update *ip, +- unsigned nr_args); +-static void __io_clean_op(struct io_kiocb *req); +-static struct file *io_file_get(struct io_submit_state *state, +- struct io_kiocb *req, int fd, bool fixed); +-static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs); +-static void io_file_put_work(struct work_struct *work); +- +-static ssize_t io_import_iovec(int rw, struct io_kiocb *req, +- struct iovec **iovec, struct iov_iter *iter, +- bool needs_lock); +-static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, +- const struct iovec *fast_iov, +- struct iov_iter *iter, bool force); +-static void io_req_drop_files(struct io_kiocb *req); +-static void io_req_task_queue(struct io_kiocb *req); +- +-static struct kmem_cache *req_cachep; +- +-static const struct file_operations io_uring_fops; +- +-struct sock *io_uring_get_socket(struct file *file) +-{ +-#if defined(CONFIG_UNIX) +- if (file->f_op == &io_uring_fops) { +- struct io_ring_ctx *ctx = file->private_data; +- +- return ctx->ring_sock->sk; +- } +-#endif +- return NULL; +-} +-EXPORT_SYMBOL(io_uring_get_socket); +- +-static inline void io_clean_op(struct io_kiocb *req) +-{ +- if (req->flags & (REQ_F_NEED_CLEANUP | REQ_F_BUFFER_SELECTED)) +- __io_clean_op(req); +-} +- +-static inline bool __io_match_files(struct io_kiocb *req, +- struct files_struct *files) +-{ +- if (req->file && req->file->f_op == &io_uring_fops) +- return true; +- +- return ((req->flags & REQ_F_WORK_INITIALIZED) && +- (req->work.flags & IO_WQ_WORK_FILES)) && +- req->work.identity->files == files; +-} +- +-static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl) +-{ +- bool got = percpu_ref_tryget(ref); +- +- /* already at zero, wait for ->release() */ +- if (!got) +- wait_for_completion(compl); +- percpu_ref_resurrect(ref); +- if (got) +- percpu_ref_put(ref); +-} +- +-static bool io_match_task(struct io_kiocb *head, +- struct task_struct *task, +- struct files_struct *files) +-{ +- struct io_kiocb *link; +- +- if (task && head->task != task) { +- /* in terms of cancelation, always match if req task is dead */ +- if (head->task->flags & PF_EXITING) +- return true; +- return false; +- } +- if (!files) +- return true; +- if (__io_match_files(head, files)) +- return true; +- if (head->flags & REQ_F_LINK_HEAD) { +- list_for_each_entry(link, &head->link_list, link_list) { +- if (__io_match_files(link, files)) +- return true; +- } +- } +- return false; +-} +- +- +-static void io_sq_thread_drop_mm(void) +-{ +- struct mm_struct *mm = current->mm; +- +- if (mm) { +- kthread_unuse_mm(mm); +- mmput(mm); +- current->mm = NULL; +- } +-} +- +-static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx) +-{ +- struct mm_struct *mm; +- +- if (current->flags & PF_EXITING) +- return -EFAULT; +- if (current->mm) +- return 0; +- +- /* Should never happen */ +- if (unlikely(!(ctx->flags & IORING_SETUP_SQPOLL))) +- return -EFAULT; +- +- task_lock(ctx->sqo_task); +- mm = ctx->sqo_task->mm; +- if (unlikely(!mm || !mmget_not_zero(mm))) +- mm = NULL; +- task_unlock(ctx->sqo_task); +- +- if (mm) { +- kthread_use_mm(mm); +- return 0; +- } +- +- return -EFAULT; +-} +- +-static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx, +- struct io_kiocb *req) +-{ +- if (!(io_op_defs[req->opcode].work_flags & IO_WQ_WORK_MM)) +- return 0; +- return __io_sq_thread_acquire_mm(ctx); +-} +- +-static void io_sq_thread_associate_blkcg(struct io_ring_ctx *ctx, +- struct cgroup_subsys_state **cur_css) +- +-{ +-#ifdef CONFIG_BLK_CGROUP +- /* puts the old one when swapping */ +- if (*cur_css != ctx->sqo_blkcg_css) { +- kthread_associate_blkcg(ctx->sqo_blkcg_css); +- *cur_css = ctx->sqo_blkcg_css; +- } +-#endif +-} +- +-static void io_sq_thread_unassociate_blkcg(void) +-{ +-#ifdef CONFIG_BLK_CGROUP +- kthread_associate_blkcg(NULL); +-#endif +-} +- +-static inline void req_set_fail_links(struct io_kiocb *req) +-{ +- if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK) +- req->flags |= REQ_F_FAIL_LINK; +-} +- +-/* +- * None of these are dereferenced, they are simply used to check if any of +- * them have changed. If we're under current and check they are still the +- * same, we're fine to grab references to them for actual out-of-line use. +- */ +-static void io_init_identity(struct io_identity *id) +-{ +- id->files = current->files; +- id->mm = current->mm; +-#ifdef CONFIG_BLK_CGROUP +- rcu_read_lock(); +- id->blkcg_css = blkcg_css(); +- rcu_read_unlock(); +-#endif +- id->creds = current_cred(); +- id->nsproxy = current->nsproxy; +- id->fs = current->fs; +- id->fsize = rlimit(RLIMIT_FSIZE); +-#ifdef CONFIG_AUDIT +- id->loginuid = current->loginuid; +- id->sessionid = current->sessionid; +-#endif +- refcount_set(&id->count, 1); +-} +- +-static inline void __io_req_init_async(struct io_kiocb *req) +-{ +- memset(&req->work, 0, sizeof(req->work)); +- req->flags |= REQ_F_WORK_INITIALIZED; +-} +- +-/* +- * Note: must call io_req_init_async() for the first time you +- * touch any members of io_wq_work. +- */ +-static inline void io_req_init_async(struct io_kiocb *req) +-{ +- struct io_uring_task *tctx = req->task->io_uring; +- +- if (req->flags & REQ_F_WORK_INITIALIZED) +- return; +- +- __io_req_init_async(req); +- +- /* Grab a ref if this isn't our static identity */ +- req->work.identity = tctx->identity; +- if (tctx->identity != &tctx->__identity) +- refcount_inc(&req->work.identity->count); +-} +- +-static inline bool io_async_submit(struct io_ring_ctx *ctx) +-{ +- return ctx->flags & IORING_SETUP_SQPOLL; +-} +- +-static void io_ring_ctx_ref_free(struct percpu_ref *ref) +-{ +- struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); +- +- complete(&ctx->ref_comp); +-} +- +-static inline bool io_is_timeout_noseq(struct io_kiocb *req) +-{ +- return !req->timeout.off; +-} +- +-static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) +-{ +- struct io_ring_ctx *ctx; +- int hash_bits; +- +- ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); +- if (!ctx) +- return NULL; +- +- ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL); +- if (!ctx->fallback_req) +- goto err; +- +- /* +- * Use 5 bits less than the max cq entries, that should give us around +- * 32 entries per hash list if totally full and uniformly spread. +- */ +- hash_bits = ilog2(p->cq_entries); +- hash_bits -= 5; +- if (hash_bits <= 0) +- hash_bits = 1; +- ctx->cancel_hash_bits = hash_bits; +- ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head), +- GFP_KERNEL); +- if (!ctx->cancel_hash) +- goto err; +- __hash_init(ctx->cancel_hash, 1U << hash_bits); +- +- if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free, +- PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) +- goto err; +- +- ctx->flags = p->flags; +- init_waitqueue_head(&ctx->sqo_sq_wait); +- INIT_LIST_HEAD(&ctx->sqd_list); +- init_waitqueue_head(&ctx->cq_wait); +- INIT_LIST_HEAD(&ctx->cq_overflow_list); +- init_completion(&ctx->ref_comp); +- init_completion(&ctx->sq_thread_comp); +- xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1); +- xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1); +- mutex_init(&ctx->uring_lock); +- init_waitqueue_head(&ctx->wait); +- spin_lock_init(&ctx->completion_lock); +- INIT_LIST_HEAD(&ctx->iopoll_list); +- INIT_LIST_HEAD(&ctx->defer_list); +- INIT_LIST_HEAD(&ctx->timeout_list); +- spin_lock_init(&ctx->inflight_lock); +- INIT_LIST_HEAD(&ctx->inflight_list); +- INIT_DELAYED_WORK(&ctx->file_put_work, io_file_put_work); +- init_llist_head(&ctx->file_put_llist); +- return ctx; +-err: +- if (ctx->fallback_req) +- kmem_cache_free(req_cachep, ctx->fallback_req); +- kfree(ctx->cancel_hash); +- kfree(ctx); +- return NULL; +-} +- +-static bool req_need_defer(struct io_kiocb *req, u32 seq) +-{ +- if (unlikely(req->flags & REQ_F_IO_DRAIN)) { +- struct io_ring_ctx *ctx = req->ctx; +- +- return seq != ctx->cached_cq_tail +- + READ_ONCE(ctx->cached_cq_overflow); +- } +- +- return false; +-} +- +-static void __io_commit_cqring(struct io_ring_ctx *ctx) +-{ +- struct io_rings *rings = ctx->rings; +- +- /* order cqe stores with ring update */ +- smp_store_release(&rings->cq.tail, ctx->cached_cq_tail); +-} +- +-static void io_put_identity(struct io_uring_task *tctx, struct io_kiocb *req) +-{ +- if (req->work.identity == &tctx->__identity) +- return; +- if (refcount_dec_and_test(&req->work.identity->count)) +- kfree(req->work.identity); +-} +- +-static void io_req_clean_work(struct io_kiocb *req) +-{ +- if (!(req->flags & REQ_F_WORK_INITIALIZED)) +- return; +- +- req->flags &= ~REQ_F_WORK_INITIALIZED; +- +- if (req->work.flags & IO_WQ_WORK_MM) { +- mmdrop(req->work.identity->mm); +- req->work.flags &= ~IO_WQ_WORK_MM; +- } +-#ifdef CONFIG_BLK_CGROUP +- if (req->work.flags & IO_WQ_WORK_BLKCG) { +- css_put(req->work.identity->blkcg_css); +- req->work.flags &= ~IO_WQ_WORK_BLKCG; +- } +-#endif +- if (req->work.flags & IO_WQ_WORK_CREDS) { +- put_cred(req->work.identity->creds); +- req->work.flags &= ~IO_WQ_WORK_CREDS; +- } +- if (req->work.flags & IO_WQ_WORK_FS) { +- struct fs_struct *fs = req->work.identity->fs; +- +- spin_lock(&req->work.identity->fs->lock); +- if (--fs->users) +- fs = NULL; +- spin_unlock(&req->work.identity->fs->lock); +- if (fs) +- free_fs_struct(fs); +- req->work.flags &= ~IO_WQ_WORK_FS; +- } +- if (req->flags & REQ_F_INFLIGHT) +- io_req_drop_files(req); +- +- io_put_identity(req->task->io_uring, req); +-} +- +-/* +- * Create a private copy of io_identity, since some fields don't match +- * the current context. +- */ +-static bool io_identity_cow(struct io_kiocb *req) +-{ +- struct io_uring_task *tctx = req->task->io_uring; +- const struct cred *creds = NULL; +- struct io_identity *id; +- +- if (req->work.flags & IO_WQ_WORK_CREDS) +- creds = req->work.identity->creds; +- +- id = kmemdup(req->work.identity, sizeof(*id), GFP_KERNEL); +- if (unlikely(!id)) { +- req->work.flags |= IO_WQ_WORK_CANCEL; +- return false; +- } +- +- /* +- * We can safely just re-init the creds we copied Either the field +- * matches the current one, or we haven't grabbed it yet. The only +- * exception is ->creds, through registered personalities, so handle +- * that one separately. +- */ +- io_init_identity(id); +- if (creds) +- id->creds = creds; +- +- /* add one for this request */ +- refcount_inc(&id->count); +- +- /* drop tctx and req identity references, if needed */ +- if (tctx->identity != &tctx->__identity && +- refcount_dec_and_test(&tctx->identity->count)) +- kfree(tctx->identity); +- if (req->work.identity != &tctx->__identity && +- refcount_dec_and_test(&req->work.identity->count)) +- kfree(req->work.identity); +- +- req->work.identity = id; +- tctx->identity = id; +- return true; +-} +- +-static bool io_grab_identity(struct io_kiocb *req) +-{ +- const struct io_op_def *def = &io_op_defs[req->opcode]; +- struct io_identity *id = req->work.identity; +- struct io_ring_ctx *ctx = req->ctx; +- +- if (def->work_flags & IO_WQ_WORK_FSIZE) { +- if (id->fsize != rlimit(RLIMIT_FSIZE)) +- return false; +- req->work.flags |= IO_WQ_WORK_FSIZE; +- } +-#ifdef CONFIG_BLK_CGROUP +- if (!(req->work.flags & IO_WQ_WORK_BLKCG) && +- (def->work_flags & IO_WQ_WORK_BLKCG)) { +- rcu_read_lock(); +- if (id->blkcg_css != blkcg_css()) { +- rcu_read_unlock(); +- return false; +- } +- /* +- * This should be rare, either the cgroup is dying or the task +- * is moving cgroups. Just punt to root for the handful of ios. +- */ +- if (css_tryget_online(id->blkcg_css)) +- req->work.flags |= IO_WQ_WORK_BLKCG; +- rcu_read_unlock(); +- } +-#endif +- if (!(req->work.flags & IO_WQ_WORK_CREDS)) { +- if (id->creds != current_cred()) +- return false; +- get_cred(id->creds); +- req->work.flags |= IO_WQ_WORK_CREDS; +- } +-#ifdef CONFIG_AUDIT +- if (!uid_eq(current->loginuid, id->loginuid) || +- current->sessionid != id->sessionid) +- return false; +-#endif +- if (!(req->work.flags & IO_WQ_WORK_FS) && +- (def->work_flags & IO_WQ_WORK_FS)) { +- if (current->fs != id->fs) +- return false; +- spin_lock(&id->fs->lock); +- if (!id->fs->in_exec) { +- id->fs->users++; +- req->work.flags |= IO_WQ_WORK_FS; +- } else { +- req->work.flags |= IO_WQ_WORK_CANCEL; +- } +- spin_unlock(¤t->fs->lock); +- } +- if (!(req->work.flags & IO_WQ_WORK_FILES) && +- (def->work_flags & IO_WQ_WORK_FILES) && +- !(req->flags & REQ_F_NO_FILE_TABLE)) { +- if (id->files != current->files || +- id->nsproxy != current->nsproxy) +- return false; +- atomic_inc(&id->files->count); +- get_nsproxy(id->nsproxy); +- +- if (!(req->flags & REQ_F_INFLIGHT)) { +- req->flags |= REQ_F_INFLIGHT; +- +- spin_lock_irq(&ctx->inflight_lock); +- list_add(&req->inflight_entry, &ctx->inflight_list); +- spin_unlock_irq(&ctx->inflight_lock); +- } +- req->work.flags |= IO_WQ_WORK_FILES; +- } +- if (!(req->work.flags & IO_WQ_WORK_MM) && +- (def->work_flags & IO_WQ_WORK_MM)) { +- if (id->mm != current->mm) +- return false; +- mmgrab(id->mm); +- req->work.flags |= IO_WQ_WORK_MM; +- } +- +- return true; +-} +- +-static void io_prep_async_work(struct io_kiocb *req) +-{ +- const struct io_op_def *def = &io_op_defs[req->opcode]; +- struct io_ring_ctx *ctx = req->ctx; +- struct io_identity *id; +- +- io_req_init_async(req); +- id = req->work.identity; +- +- if (req->flags & REQ_F_FORCE_ASYNC) +- req->work.flags |= IO_WQ_WORK_CONCURRENT; +- +- if (req->flags & REQ_F_ISREG) { +- if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL)) +- io_wq_hash_work(&req->work, file_inode(req->file)); +- } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) { +- if (def->unbound_nonreg_file) +- req->work.flags |= IO_WQ_WORK_UNBOUND; +- } +- +- /* if we fail grabbing identity, we must COW, regrab, and retry */ +- if (io_grab_identity(req)) +- return; +- +- if (!io_identity_cow(req)) +- return; +- +- /* can't fail at this point */ +- if (!io_grab_identity(req)) +- WARN_ON(1); +-} +- +-static void io_prep_async_link(struct io_kiocb *req) +-{ +- struct io_kiocb *cur; +- +- io_prep_async_work(req); +- if (req->flags & REQ_F_LINK_HEAD) +- list_for_each_entry(cur, &req->link_list, link_list) +- io_prep_async_work(cur); +-} +- +-static struct io_kiocb *__io_queue_async_work(struct io_kiocb *req) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- struct io_kiocb *link = io_prep_linked_timeout(req); +- +- trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req, +- &req->work, req->flags); +- io_wq_enqueue(ctx->io_wq, &req->work); +- return link; +-} +- +-static void io_queue_async_work(struct io_kiocb *req) +-{ +- struct io_kiocb *link; +- +- /* init ->work of the whole link before punting */ +- io_prep_async_link(req); +- link = __io_queue_async_work(req); +- +- if (link) +- io_queue_linked_timeout(link); +-} +- +-static void io_kill_timeout(struct io_kiocb *req, int status) +-{ +- struct io_timeout_data *io = req->async_data; +- int ret; +- +- ret = hrtimer_try_to_cancel(&io->timer); +- if (ret != -1) { +- if (status) +- req_set_fail_links(req); +- atomic_set(&req->ctx->cq_timeouts, +- atomic_read(&req->ctx->cq_timeouts) + 1); +- list_del_init(&req->timeout.list); +- io_cqring_fill_event(req, status); +- io_put_req_deferred(req, 1); +- } +-} +- +-/* +- * Returns true if we found and killed one or more timeouts +- */ +-static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, +- struct files_struct *files) +-{ +- struct io_kiocb *req, *tmp; +- int canceled = 0; +- +- spin_lock_irq(&ctx->completion_lock); +- list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { +- if (io_match_task(req, tsk, files)) { +- io_kill_timeout(req, -ECANCELED); +- canceled++; +- } +- } +- spin_unlock_irq(&ctx->completion_lock); +- return canceled != 0; +-} +- +-static void __io_queue_deferred(struct io_ring_ctx *ctx) +-{ +- do { +- struct io_defer_entry *de = list_first_entry(&ctx->defer_list, +- struct io_defer_entry, list); +- +- if (req_need_defer(de->req, de->seq)) +- break; +- list_del_init(&de->list); +- io_req_task_queue(de->req); +- kfree(de); +- } while (!list_empty(&ctx->defer_list)); +-} +- +-static void io_flush_timeouts(struct io_ring_ctx *ctx) +-{ +- struct io_kiocb *req, *tmp; +- u32 seq; +- +- if (list_empty(&ctx->timeout_list)) +- return; +- +- seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); +- +- list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { +- u32 events_needed, events_got; +- +- if (io_is_timeout_noseq(req)) +- break; +- +- /* +- * Since seq can easily wrap around over time, subtract +- * the last seq at which timeouts were flushed before comparing. +- * Assuming not more than 2^31-1 events have happened since, +- * these subtractions won't have wrapped, so we can check if +- * target is in [last_seq, current_seq] by comparing the two. +- */ +- events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush; +- events_got = seq - ctx->cq_last_tm_flush; +- if (events_got < events_needed) +- break; +- +- io_kill_timeout(req, 0); +- } +- +- ctx->cq_last_tm_flush = seq; +-} +- +-static void io_commit_cqring(struct io_ring_ctx *ctx) +-{ +- io_flush_timeouts(ctx); +- __io_commit_cqring(ctx); +- +- if (unlikely(!list_empty(&ctx->defer_list))) +- __io_queue_deferred(ctx); +-} +- +-static inline bool io_sqring_full(struct io_ring_ctx *ctx) +-{ +- struct io_rings *r = ctx->rings; +- +- return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == r->sq_ring_entries; +-} +- +-static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx) +-{ +- struct io_rings *rings = ctx->rings; +- unsigned tail; +- +- tail = ctx->cached_cq_tail; +- /* +- * writes to the cq entry need to come after reading head; the +- * control dependency is enough as we're using WRITE_ONCE to +- * fill the cq entry +- */ +- if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries) +- return NULL; +- +- ctx->cached_cq_tail++; +- return &rings->cqes[tail & ctx->cq_mask]; +-} +- +-static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx) +-{ +- if (!ctx->cq_ev_fd) +- return false; +- if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) +- return false; +- if (!ctx->eventfd_async) +- return true; +- return io_wq_current_is_worker(); +-} +- +-static void io_cqring_ev_posted(struct io_ring_ctx *ctx) +-{ +- if (wq_has_sleeper(&ctx->cq_wait)) { +- wake_up_interruptible(&ctx->cq_wait); +- kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN); +- } +- if (waitqueue_active(&ctx->wait)) +- wake_up(&ctx->wait); +- if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait)) +- wake_up(&ctx->sq_data->wait); +- if (io_should_trigger_evfd(ctx)) +- eventfd_signal(ctx->cq_ev_fd, 1); +-} +- +-static void io_cqring_mark_overflow(struct io_ring_ctx *ctx) +-{ +- if (list_empty(&ctx->cq_overflow_list)) { +- clear_bit(0, &ctx->sq_check_overflow); +- clear_bit(0, &ctx->cq_check_overflow); +- ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW; +- } +-} +- +-/* Returns true if there are no backlogged entries after the flush */ +-static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, +- struct task_struct *tsk, +- struct files_struct *files) +-{ +- struct io_rings *rings = ctx->rings; +- struct io_kiocb *req, *tmp; +- struct io_uring_cqe *cqe; +- unsigned long flags; +- LIST_HEAD(list); +- +- if (!force) { +- if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) == +- rings->cq_ring_entries)) +- return false; +- } +- +- spin_lock_irqsave(&ctx->completion_lock, flags); +- +- cqe = NULL; +- list_for_each_entry_safe(req, tmp, &ctx->cq_overflow_list, compl.list) { +- if (!io_match_task(req, tsk, files)) +- continue; +- +- cqe = io_get_cqring(ctx); +- if (!cqe && !force) +- break; +- +- list_move(&req->compl.list, &list); +- if (cqe) { +- WRITE_ONCE(cqe->user_data, req->user_data); +- WRITE_ONCE(cqe->res, req->result); +- WRITE_ONCE(cqe->flags, req->compl.cflags); +- } else { +- ctx->cached_cq_overflow++; +- WRITE_ONCE(ctx->rings->cq_overflow, +- ctx->cached_cq_overflow); +- } +- } +- +- io_commit_cqring(ctx); +- io_cqring_mark_overflow(ctx); +- +- spin_unlock_irqrestore(&ctx->completion_lock, flags); +- io_cqring_ev_posted(ctx); +- +- while (!list_empty(&list)) { +- req = list_first_entry(&list, struct io_kiocb, compl.list); +- list_del(&req->compl.list); +- io_put_req(req); +- } +- +- return cqe != NULL; +-} +- +-static void io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, +- struct task_struct *tsk, +- struct files_struct *files) +-{ +- if (test_bit(0, &ctx->cq_check_overflow)) { +- /* iopoll syncs against uring_lock, not completion_lock */ +- if (ctx->flags & IORING_SETUP_IOPOLL) +- mutex_lock(&ctx->uring_lock); +- __io_cqring_overflow_flush(ctx, force, tsk, files); +- if (ctx->flags & IORING_SETUP_IOPOLL) +- mutex_unlock(&ctx->uring_lock); +- } +-} +- +-static void __io_cqring_fill_event(struct io_kiocb *req, long res, +- unsigned int cflags) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- struct io_uring_cqe *cqe; +- +- trace_io_uring_complete(ctx, req->user_data, res); +- +- /* +- * If we can't get a cq entry, userspace overflowed the +- * submission (by quite a lot). Increment the overflow count in +- * the ring. +- */ +- cqe = io_get_cqring(ctx); +- if (likely(cqe)) { +- WRITE_ONCE(cqe->user_data, req->user_data); +- WRITE_ONCE(cqe->res, res); +- WRITE_ONCE(cqe->flags, cflags); +- } else if (ctx->cq_overflow_flushed || +- atomic_read(&req->task->io_uring->in_idle)) { +- /* +- * If we're in ring overflow flush mode, or in task cancel mode, +- * then we cannot store the request for later flushing, we need +- * to drop it on the floor. +- */ +- ctx->cached_cq_overflow++; +- WRITE_ONCE(ctx->rings->cq_overflow, ctx->cached_cq_overflow); +- } else { +- if (list_empty(&ctx->cq_overflow_list)) { +- set_bit(0, &ctx->sq_check_overflow); +- set_bit(0, &ctx->cq_check_overflow); +- ctx->rings->sq_flags |= IORING_SQ_CQ_OVERFLOW; +- } +- io_clean_op(req); +- req->result = res; +- req->compl.cflags = cflags; +- refcount_inc(&req->refs); +- list_add_tail(&req->compl.list, &ctx->cq_overflow_list); +- } +-} +- +-static void io_cqring_fill_event(struct io_kiocb *req, long res) +-{ +- __io_cqring_fill_event(req, res, 0); +-} +- +-static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- unsigned long flags; +- +- spin_lock_irqsave(&ctx->completion_lock, flags); +- __io_cqring_fill_event(req, res, cflags); +- io_commit_cqring(ctx); +- spin_unlock_irqrestore(&ctx->completion_lock, flags); +- +- io_cqring_ev_posted(ctx); +-} +- +-static void io_submit_flush_completions(struct io_comp_state *cs) +-{ +- struct io_ring_ctx *ctx = cs->ctx; +- +- spin_lock_irq(&ctx->completion_lock); +- while (!list_empty(&cs->list)) { +- struct io_kiocb *req; +- +- req = list_first_entry(&cs->list, struct io_kiocb, compl.list); +- list_del(&req->compl.list); +- __io_cqring_fill_event(req, req->result, req->compl.cflags); +- +- /* +- * io_free_req() doesn't care about completion_lock unless one +- * of these flags is set. REQ_F_WORK_INITIALIZED is in the list +- * because of a potential deadlock with req->work.fs->lock +- */ +- if (req->flags & (REQ_F_FAIL_LINK|REQ_F_LINK_TIMEOUT +- |REQ_F_WORK_INITIALIZED)) { +- spin_unlock_irq(&ctx->completion_lock); +- io_put_req(req); +- spin_lock_irq(&ctx->completion_lock); +- } else { +- io_put_req(req); +- } +- } +- io_commit_cqring(ctx); +- spin_unlock_irq(&ctx->completion_lock); +- +- io_cqring_ev_posted(ctx); +- cs->nr = 0; +-} +- +-static void __io_req_complete(struct io_kiocb *req, long res, unsigned cflags, +- struct io_comp_state *cs) +-{ +- if (!cs) { +- io_cqring_add_event(req, res, cflags); +- io_put_req(req); +- } else { +- io_clean_op(req); +- req->result = res; +- req->compl.cflags = cflags; +- list_add_tail(&req->compl.list, &cs->list); +- if (++cs->nr >= 32) +- io_submit_flush_completions(cs); +- } +-} +- +-static void io_req_complete(struct io_kiocb *req, long res) +-{ +- __io_req_complete(req, res, 0, NULL); +-} +- +-static inline bool io_is_fallback_req(struct io_kiocb *req) +-{ +- return req == (struct io_kiocb *) +- ((unsigned long) req->ctx->fallback_req & ~1UL); +-} +- +-static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx) +-{ +- struct io_kiocb *req; +- +- req = ctx->fallback_req; +- if (!test_and_set_bit_lock(0, (unsigned long *) &ctx->fallback_req)) +- return req; +- +- return NULL; +-} +- +-static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx, +- struct io_submit_state *state) +-{ +- if (!state->free_reqs) { +- gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; +- size_t sz; +- int ret; +- +- sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs)); +- ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs); +- +- /* +- * Bulk alloc is all-or-nothing. If we fail to get a batch, +- * retry single alloc to be on the safe side. +- */ +- if (unlikely(ret <= 0)) { +- state->reqs[0] = kmem_cache_alloc(req_cachep, gfp); +- if (!state->reqs[0]) +- goto fallback; +- ret = 1; +- } +- state->free_reqs = ret; +- } +- +- state->free_reqs--; +- return state->reqs[state->free_reqs]; +-fallback: +- return io_get_fallback_req(ctx); +-} +- +-static inline void io_put_file(struct io_kiocb *req, struct file *file, +- bool fixed) +-{ +- if (fixed) +- percpu_ref_put(req->fixed_file_refs); +- else +- fput(file); +-} +- +-static void io_dismantle_req(struct io_kiocb *req) +-{ +- io_clean_op(req); +- +- if (req->async_data) +- kfree(req->async_data); +- if (req->file) +- io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE)); +- +- io_req_clean_work(req); +-} +- +-static void __io_free_req(struct io_kiocb *req) +-{ +- struct io_uring_task *tctx = req->task->io_uring; +- struct io_ring_ctx *ctx = req->ctx; +- +- io_dismantle_req(req); +- +- percpu_counter_dec(&tctx->inflight); +- if (atomic_read(&tctx->in_idle)) +- wake_up(&tctx->wait); +- put_task_struct(req->task); +- +- if (likely(!io_is_fallback_req(req))) +- kmem_cache_free(req_cachep, req); +- else +- clear_bit_unlock(0, (unsigned long *) &ctx->fallback_req); +- percpu_ref_put(&ctx->refs); +-} +- +-static void io_kill_linked_timeout(struct io_kiocb *req) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- struct io_kiocb *link; +- bool cancelled = false; +- unsigned long flags; +- +- spin_lock_irqsave(&ctx->completion_lock, flags); +- link = list_first_entry_or_null(&req->link_list, struct io_kiocb, +- link_list); +- /* +- * Can happen if a linked timeout fired and link had been like +- * req -> link t-out -> link t-out [-> ...] +- */ +- if (link && (link->flags & REQ_F_LTIMEOUT_ACTIVE)) { +- struct io_timeout_data *io = link->async_data; +- int ret; +- +- list_del_init(&link->link_list); +- ret = hrtimer_try_to_cancel(&io->timer); +- if (ret != -1) { +- io_cqring_fill_event(link, -ECANCELED); +- io_commit_cqring(ctx); +- cancelled = true; +- } +- } +- req->flags &= ~REQ_F_LINK_TIMEOUT; +- spin_unlock_irqrestore(&ctx->completion_lock, flags); +- +- if (cancelled) { +- io_cqring_ev_posted(ctx); +- io_put_req(link); +- } +-} +- +-static struct io_kiocb *io_req_link_next(struct io_kiocb *req) +-{ +- struct io_kiocb *nxt; +- +- /* +- * The list should never be empty when we are called here. But could +- * potentially happen if the chain is messed up, check to be on the +- * safe side. +- */ +- if (unlikely(list_empty(&req->link_list))) +- return NULL; +- +- nxt = list_first_entry(&req->link_list, struct io_kiocb, link_list); +- list_del_init(&req->link_list); +- if (!list_empty(&nxt->link_list)) +- nxt->flags |= REQ_F_LINK_HEAD; +- return nxt; +-} +- +-/* +- * Called if REQ_F_LINK_HEAD is set, and we fail the head request +- */ +-static void io_fail_links(struct io_kiocb *req) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- unsigned long flags; +- +- spin_lock_irqsave(&ctx->completion_lock, flags); +- while (!list_empty(&req->link_list)) { +- struct io_kiocb *link = list_first_entry(&req->link_list, +- struct io_kiocb, link_list); +- +- list_del_init(&link->link_list); +- trace_io_uring_fail_link(req, link); +- +- io_cqring_fill_event(link, -ECANCELED); +- +- /* +- * It's ok to free under spinlock as they're not linked anymore, +- * but avoid REQ_F_WORK_INITIALIZED because it may deadlock on +- * work.fs->lock. +- */ +- if (link->flags & REQ_F_WORK_INITIALIZED) +- io_put_req_deferred(link, 2); +- else +- io_double_put_req(link); +- } +- +- io_commit_cqring(ctx); +- spin_unlock_irqrestore(&ctx->completion_lock, flags); +- +- io_cqring_ev_posted(ctx); +-} +- +-static struct io_kiocb *__io_req_find_next(struct io_kiocb *req) +-{ +- req->flags &= ~REQ_F_LINK_HEAD; +- if (req->flags & REQ_F_LINK_TIMEOUT) +- io_kill_linked_timeout(req); +- +- /* +- * If LINK is set, we have dependent requests in this chain. If we +- * didn't fail this request, queue the first one up, moving any other +- * dependencies to the next request. In case of failure, fail the rest +- * of the chain. +- */ +- if (likely(!(req->flags & REQ_F_FAIL_LINK))) +- return io_req_link_next(req); +- io_fail_links(req); +- return NULL; +-} +- +-static struct io_kiocb *io_req_find_next(struct io_kiocb *req) +-{ +- if (likely(!(req->flags & REQ_F_LINK_HEAD))) +- return NULL; +- return __io_req_find_next(req); +-} +- +-static int io_req_task_work_add(struct io_kiocb *req, bool twa_signal_ok) +-{ +- struct task_struct *tsk = req->task; +- struct io_ring_ctx *ctx = req->ctx; +- enum task_work_notify_mode notify; +- int ret; +- +- if (tsk->flags & PF_EXITING) +- return -ESRCH; +- +- /* +- * SQPOLL kernel thread doesn't need notification, just a wakeup. For +- * all other cases, use TWA_SIGNAL unconditionally to ensure we're +- * processing task_work. There's no reliable way to tell if TWA_RESUME +- * will do the job. +- */ +- notify = TWA_NONE; +- if (!(ctx->flags & IORING_SETUP_SQPOLL) && twa_signal_ok) +- notify = TWA_SIGNAL; +- +- ret = task_work_add(tsk, &req->task_work, notify); +- if (!ret) +- wake_up_process(tsk); +- +- return ret; +-} +- +-static void __io_req_task_cancel(struct io_kiocb *req, int error) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- +- spin_lock_irq(&ctx->completion_lock); +- io_cqring_fill_event(req, error); +- io_commit_cqring(ctx); +- spin_unlock_irq(&ctx->completion_lock); +- +- io_cqring_ev_posted(ctx); +- req_set_fail_links(req); +- io_double_put_req(req); +-} +- +-static void io_req_task_cancel(struct callback_head *cb) +-{ +- struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); +- struct io_ring_ctx *ctx = req->ctx; +- +- mutex_lock(&ctx->uring_lock); +- __io_req_task_cancel(req, -ECANCELED); +- mutex_unlock(&ctx->uring_lock); +- percpu_ref_put(&ctx->refs); +-} +- +-static void __io_req_task_submit(struct io_kiocb *req) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- +- mutex_lock(&ctx->uring_lock); +- if (!ctx->sqo_dead && !__io_sq_thread_acquire_mm(ctx)) +- __io_queue_sqe(req, NULL); +- else +- __io_req_task_cancel(req, -EFAULT); +- mutex_unlock(&ctx->uring_lock); +- +- if (ctx->flags & IORING_SETUP_SQPOLL) +- io_sq_thread_drop_mm(); +-} +- +-static void io_req_task_submit(struct callback_head *cb) +-{ +- struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); +- struct io_ring_ctx *ctx = req->ctx; +- +- __io_req_task_submit(req); +- percpu_ref_put(&ctx->refs); +-} +- +-static void io_req_task_queue(struct io_kiocb *req) +-{ +- int ret; +- +- init_task_work(&req->task_work, io_req_task_submit); +- percpu_ref_get(&req->ctx->refs); +- +- ret = io_req_task_work_add(req, true); +- if (unlikely(ret)) { +- struct task_struct *tsk; +- +- init_task_work(&req->task_work, io_req_task_cancel); +- tsk = io_wq_get_task(req->ctx->io_wq); +- task_work_add(tsk, &req->task_work, TWA_NONE); +- wake_up_process(tsk); +- } +-} +- +-static void io_queue_next(struct io_kiocb *req) +-{ +- struct io_kiocb *nxt = io_req_find_next(req); +- +- if (nxt) +- io_req_task_queue(nxt); +-} +- +-static void io_free_req(struct io_kiocb *req) +-{ +- io_queue_next(req); +- __io_free_req(req); +-} +- +-struct req_batch { +- void *reqs[IO_IOPOLL_BATCH]; +- int to_free; +- +- struct task_struct *task; +- int task_refs; +-}; +- +-static inline void io_init_req_batch(struct req_batch *rb) +-{ +- rb->to_free = 0; +- rb->task_refs = 0; +- rb->task = NULL; +-} +- +-static void __io_req_free_batch_flush(struct io_ring_ctx *ctx, +- struct req_batch *rb) +-{ +- kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs); +- percpu_ref_put_many(&ctx->refs, rb->to_free); +- rb->to_free = 0; +-} +- +-static void io_req_free_batch_finish(struct io_ring_ctx *ctx, +- struct req_batch *rb) +-{ +- if (rb->to_free) +- __io_req_free_batch_flush(ctx, rb); +- if (rb->task) { +- struct io_uring_task *tctx = rb->task->io_uring; +- +- percpu_counter_sub(&tctx->inflight, rb->task_refs); +- if (atomic_read(&tctx->in_idle)) +- wake_up(&tctx->wait); +- put_task_struct_many(rb->task, rb->task_refs); +- rb->task = NULL; +- } +-} +- +-static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) +-{ +- if (unlikely(io_is_fallback_req(req))) { +- io_free_req(req); +- return; +- } +- if (req->flags & REQ_F_LINK_HEAD) +- io_queue_next(req); +- +- if (req->task != rb->task) { +- if (rb->task) { +- struct io_uring_task *tctx = rb->task->io_uring; +- +- percpu_counter_sub(&tctx->inflight, rb->task_refs); +- if (atomic_read(&tctx->in_idle)) +- wake_up(&tctx->wait); +- put_task_struct_many(rb->task, rb->task_refs); +- } +- rb->task = req->task; +- rb->task_refs = 0; +- } +- rb->task_refs++; +- +- io_dismantle_req(req); +- rb->reqs[rb->to_free++] = req; +- if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs))) +- __io_req_free_batch_flush(req->ctx, rb); +-} +- +-/* +- * Drop reference to request, return next in chain (if there is one) if this +- * was the last reference to this request. +- */ +-static struct io_kiocb *io_put_req_find_next(struct io_kiocb *req) +-{ +- struct io_kiocb *nxt = NULL; +- +- if (refcount_dec_and_test(&req->refs)) { +- nxt = io_req_find_next(req); +- __io_free_req(req); +- } +- return nxt; +-} +- +-static void io_put_req(struct io_kiocb *req) +-{ +- if (refcount_dec_and_test(&req->refs)) +- io_free_req(req); +-} +- +-static void io_put_req_deferred_cb(struct callback_head *cb) +-{ +- struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); +- +- io_free_req(req); +-} +- +-static void io_free_req_deferred(struct io_kiocb *req) +-{ +- int ret; +- +- init_task_work(&req->task_work, io_put_req_deferred_cb); +- ret = io_req_task_work_add(req, true); +- if (unlikely(ret)) { +- struct task_struct *tsk; +- +- tsk = io_wq_get_task(req->ctx->io_wq); +- task_work_add(tsk, &req->task_work, TWA_NONE); +- wake_up_process(tsk); +- } +-} +- +-static inline void io_put_req_deferred(struct io_kiocb *req, int refs) +-{ +- if (refcount_sub_and_test(refs, &req->refs)) +- io_free_req_deferred(req); +-} +- +-static struct io_wq_work *io_steal_work(struct io_kiocb *req) +-{ +- struct io_kiocb *nxt; +- +- /* +- * A ref is owned by io-wq in which context we're. So, if that's the +- * last one, it's safe to steal next work. False negatives are Ok, +- * it just will be re-punted async in io_put_work() +- */ +- if (refcount_read(&req->refs) != 1) +- return NULL; +- +- nxt = io_req_find_next(req); +- return nxt ? &nxt->work : NULL; +-} +- +-static void io_double_put_req(struct io_kiocb *req) +-{ +- /* drop both submit and complete references */ +- if (refcount_sub_and_test(2, &req->refs)) +- io_free_req(req); +-} +- +-static unsigned io_cqring_events(struct io_ring_ctx *ctx) +-{ +- struct io_rings *rings = ctx->rings; +- +- /* See comment at the top of this file */ +- smp_rmb(); +- return ctx->cached_cq_tail - READ_ONCE(rings->cq.head); +-} +- +-static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) +-{ +- struct io_rings *rings = ctx->rings; +- +- /* make sure SQ entry isn't read before tail */ +- return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; +-} +- +-static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf) +-{ +- unsigned int cflags; +- +- cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT; +- cflags |= IORING_CQE_F_BUFFER; +- req->flags &= ~REQ_F_BUFFER_SELECTED; +- kfree(kbuf); +- return cflags; +-} +- +-static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req) +-{ +- struct io_buffer *kbuf; +- +- kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; +- return io_put_kbuf(req, kbuf); +-} +- +-static inline bool io_run_task_work(void) +-{ +- /* +- * Not safe to run on exiting task, and the task_work handling will +- * not add work to such a task. +- */ +- if (unlikely(current->flags & PF_EXITING)) +- return false; +- if (current->task_works) { +- __set_current_state(TASK_RUNNING); +- task_work_run(); +- return true; +- } +- +- return false; +-} +- +-static void io_iopoll_queue(struct list_head *again) +-{ +- struct io_kiocb *req; +- +- do { +- req = list_first_entry(again, struct io_kiocb, iopoll_entry); +- list_del(&req->iopoll_entry); +- __io_complete_rw(req, -EAGAIN, 0, NULL); +- } while (!list_empty(again)); +-} +- +-/* +- * Find and free completed poll iocbs +- */ +-static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, +- struct list_head *done) +-{ +- struct req_batch rb; +- struct io_kiocb *req; +- LIST_HEAD(again); +- +- /* order with ->result store in io_complete_rw_iopoll() */ +- smp_rmb(); +- +- io_init_req_batch(&rb); +- while (!list_empty(done)) { +- int cflags = 0; +- +- req = list_first_entry(done, struct io_kiocb, iopoll_entry); +- if (READ_ONCE(req->result) == -EAGAIN) { +- req->result = 0; +- req->iopoll_completed = 0; +- list_move_tail(&req->iopoll_entry, &again); +- continue; +- } +- list_del(&req->iopoll_entry); +- +- if (req->flags & REQ_F_BUFFER_SELECTED) +- cflags = io_put_rw_kbuf(req); +- +- __io_cqring_fill_event(req, req->result, cflags); +- (*nr_events)++; +- +- if (refcount_dec_and_test(&req->refs)) +- io_req_free_batch(&rb, req); +- } +- +- io_commit_cqring(ctx); +- if (ctx->flags & IORING_SETUP_SQPOLL) +- io_cqring_ev_posted(ctx); +- io_req_free_batch_finish(ctx, &rb); +- +- if (!list_empty(&again)) +- io_iopoll_queue(&again); +-} +- +-static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, +- long min) +-{ +- struct io_kiocb *req, *tmp; +- LIST_HEAD(done); +- bool spin; +- int ret; +- +- /* +- * Only spin for completions if we don't have multiple devices hanging +- * off our complete list, and we're under the requested amount. +- */ +- spin = !ctx->poll_multi_file && *nr_events < min; +- +- ret = 0; +- list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, iopoll_entry) { +- struct kiocb *kiocb = &req->rw.kiocb; +- +- /* +- * Move completed and retryable entries to our local lists. +- * If we find a request that requires polling, break out +- * and complete those lists first, if we have entries there. +- */ +- if (READ_ONCE(req->iopoll_completed)) { +- list_move_tail(&req->iopoll_entry, &done); +- continue; +- } +- if (!list_empty(&done)) +- break; +- +- ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin); +- if (ret < 0) +- break; +- +- /* iopoll may have completed current req */ +- if (READ_ONCE(req->iopoll_completed)) +- list_move_tail(&req->iopoll_entry, &done); +- +- if (ret && spin) +- spin = false; +- ret = 0; +- } +- +- if (!list_empty(&done)) +- io_iopoll_complete(ctx, nr_events, &done); +- +- return ret; +-} +- +-/* +- * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a +- * non-spinning poll check - we'll still enter the driver poll loop, but only +- * as a non-spinning completion check. +- */ +-static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events, +- long min) +-{ +- while (!list_empty(&ctx->iopoll_list) && !need_resched()) { +- int ret; +- +- ret = io_do_iopoll(ctx, nr_events, min); +- if (ret < 0) +- return ret; +- if (*nr_events >= min) +- return 0; +- } +- +- return 1; +-} +- +-/* +- * We can't just wait for polled events to come to us, we have to actively +- * find and complete them. +- */ +-static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx) +-{ +- if (!(ctx->flags & IORING_SETUP_IOPOLL)) +- return; +- +- mutex_lock(&ctx->uring_lock); +- while (!list_empty(&ctx->iopoll_list)) { +- unsigned int nr_events = 0; +- +- io_do_iopoll(ctx, &nr_events, 0); +- +- /* let it sleep and repeat later if can't complete a request */ +- if (nr_events == 0) +- break; +- /* +- * Ensure we allow local-to-the-cpu processing to take place, +- * in this case we need to ensure that we reap all events. +- * Also let task_work, etc. to progress by releasing the mutex +- */ +- if (need_resched()) { +- mutex_unlock(&ctx->uring_lock); +- cond_resched(); +- mutex_lock(&ctx->uring_lock); +- } +- } +- mutex_unlock(&ctx->uring_lock); +-} +- +-static int io_iopoll_check(struct io_ring_ctx *ctx, long min) +-{ +- unsigned int nr_events = 0; +- int iters = 0, ret = 0; +- +- /* +- * We disallow the app entering submit/complete with polling, but we +- * still need to lock the ring to prevent racing with polled issue +- * that got punted to a workqueue. +- */ +- mutex_lock(&ctx->uring_lock); +- do { +- /* +- * Don't enter poll loop if we already have events pending. +- * If we do, we can potentially be spinning for commands that +- * already triggered a CQE (eg in error). +- */ +- if (test_bit(0, &ctx->cq_check_overflow)) +- __io_cqring_overflow_flush(ctx, false, NULL, NULL); +- if (io_cqring_events(ctx)) +- break; +- +- /* +- * If a submit got punted to a workqueue, we can have the +- * application entering polling for a command before it gets +- * issued. That app will hold the uring_lock for the duration +- * of the poll right here, so we need to take a breather every +- * now and then to ensure that the issue has a chance to add +- * the poll to the issued list. Otherwise we can spin here +- * forever, while the workqueue is stuck trying to acquire the +- * very same mutex. +- */ +- if (!(++iters & 7)) { +- mutex_unlock(&ctx->uring_lock); +- io_run_task_work(); +- mutex_lock(&ctx->uring_lock); +- } +- +- ret = io_iopoll_getevents(ctx, &nr_events, min); +- if (ret <= 0) +- break; +- ret = 0; +- } while (min && !nr_events && !need_resched()); +- +- mutex_unlock(&ctx->uring_lock); +- return ret; +-} +- +-static void kiocb_end_write(struct io_kiocb *req) +-{ +- /* +- * Tell lockdep we inherited freeze protection from submission +- * thread. +- */ +- if (req->flags & REQ_F_ISREG) { +- struct inode *inode = file_inode(req->file); +- +- __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE); +- } +- file_end_write(req->file); +-} +- +-static void io_complete_rw_common(struct kiocb *kiocb, long res, +- struct io_comp_state *cs) +-{ +- struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); +- int cflags = 0; +- +- if (kiocb->ki_flags & IOCB_WRITE) +- kiocb_end_write(req); +- +- if (res != req->result) +- req_set_fail_links(req); +- if (req->flags & REQ_F_BUFFER_SELECTED) +- cflags = io_put_rw_kbuf(req); +- __io_req_complete(req, res, cflags, cs); +-} +- +-#ifdef CONFIG_BLOCK +-static bool io_resubmit_prep(struct io_kiocb *req, int error) +-{ +- req_set_fail_links(req); +- return false; +-} +-#endif +- +-static bool io_rw_reissue(struct io_kiocb *req, long res) +-{ +-#ifdef CONFIG_BLOCK +- umode_t mode = file_inode(req->file)->i_mode; +- int ret; +- +- if (!S_ISBLK(mode) && !S_ISREG(mode)) +- return false; +- if ((res != -EAGAIN && res != -EOPNOTSUPP) || io_wq_current_is_worker()) +- return false; +- /* +- * If ref is dying, we might be running poll reap from the exit work. +- * Don't attempt to reissue from that path, just let it fail with +- * -EAGAIN. +- */ +- if (percpu_ref_is_dying(&req->ctx->refs)) +- return false; +- +- ret = io_sq_thread_acquire_mm(req->ctx, req); +- +- if (io_resubmit_prep(req, ret)) { +- refcount_inc(&req->refs); +- io_queue_async_work(req); +- return true; +- } +- +-#endif +- return false; +-} +- +-static void __io_complete_rw(struct io_kiocb *req, long res, long res2, +- struct io_comp_state *cs) +-{ +- if (!io_rw_reissue(req, res)) +- io_complete_rw_common(&req->rw.kiocb, res, cs); +-} +- +-static void io_complete_rw(struct kiocb *kiocb, long res, long res2) +-{ +- struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); +- +- __io_complete_rw(req, res, res2, NULL); +-} +- +-static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) +-{ +- struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); +- +- if (kiocb->ki_flags & IOCB_WRITE) +- kiocb_end_write(req); +- +- if (res != -EAGAIN && res != req->result) +- req_set_fail_links(req); +- +- WRITE_ONCE(req->result, res); +- /* order with io_poll_complete() checking ->result */ +- smp_wmb(); +- WRITE_ONCE(req->iopoll_completed, 1); +-} +- +-/* +- * After the iocb has been issued, it's safe to be found on the poll list. +- * Adding the kiocb to the list AFTER submission ensures that we don't +- * find it from a io_iopoll_getevents() thread before the issuer is done +- * accessing the kiocb cookie. +- */ +-static void io_iopoll_req_issued(struct io_kiocb *req) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- +- /* +- * Track whether we have multiple files in our lists. This will impact +- * how we do polling eventually, not spinning if we're on potentially +- * different devices. +- */ +- if (list_empty(&ctx->iopoll_list)) { +- ctx->poll_multi_file = false; +- } else if (!ctx->poll_multi_file) { +- struct io_kiocb *list_req; +- +- list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb, +- iopoll_entry); +- if (list_req->file != req->file) +- ctx->poll_multi_file = true; +- } +- +- /* +- * For fast devices, IO may have already completed. If it has, add +- * it to the front so we find it first. +- */ +- if (READ_ONCE(req->iopoll_completed)) +- list_add(&req->iopoll_entry, &ctx->iopoll_list); +- else +- list_add_tail(&req->iopoll_entry, &ctx->iopoll_list); +- +- if ((ctx->flags & IORING_SETUP_SQPOLL) && +- wq_has_sleeper(&ctx->sq_data->wait)) +- wake_up(&ctx->sq_data->wait); +-} +- +-static void __io_state_file_put(struct io_submit_state *state) +-{ +- if (state->has_refs) +- fput_many(state->file, state->has_refs); +- state->file = NULL; +-} +- +-static inline void io_state_file_put(struct io_submit_state *state) +-{ +- if (state->file) +- __io_state_file_put(state); +-} +- +-/* +- * Get as many references to a file as we have IOs left in this submission, +- * assuming most submissions are for one file, or at least that each file +- * has more than one submission. +- */ +-static struct file *__io_file_get(struct io_submit_state *state, int fd) +-{ +- if (!state) +- return fget(fd); +- +- if (state->file) { +- if (state->fd == fd) { +- state->has_refs--; +- return state->file; +- } +- __io_state_file_put(state); +- } +- state->file = fget_many(fd, state->ios_left); +- if (!state->file) +- return NULL; +- +- state->fd = fd; +- state->has_refs = state->ios_left - 1; +- return state->file; +-} +- +-static bool io_bdev_nowait(struct block_device *bdev) +-{ +-#ifdef CONFIG_BLOCK +- return !bdev || blk_queue_nowait(bdev_get_queue(bdev)); +-#else +- return true; +-#endif +-} +- +-/* +- * If we tracked the file through the SCM inflight mechanism, we could support +- * any file. For now, just ensure that anything potentially problematic is done +- * inline. +- */ +-static bool io_file_supports_async(struct file *file, int rw) +-{ +- umode_t mode = file_inode(file)->i_mode; +- +- if (S_ISBLK(mode)) { +- if (io_bdev_nowait(file->f_inode->i_bdev)) +- return true; +- return false; +- } +- if (S_ISSOCK(mode)) +- return true; +- if (S_ISREG(mode)) { +- if (io_bdev_nowait(file->f_inode->i_sb->s_bdev) && +- file->f_op != &io_uring_fops) +- return true; +- return false; +- } +- +- /* any ->read/write should understand O_NONBLOCK */ +- if (file->f_flags & O_NONBLOCK) +- return true; +- +- if (!(file->f_mode & FMODE_NOWAIT)) +- return false; +- +- if (rw == READ) +- return file->f_op->read_iter != NULL; +- +- return file->f_op->write_iter != NULL; +-} +- +-static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- struct kiocb *kiocb = &req->rw.kiocb; +- unsigned ioprio; +- int ret; +- +- if (S_ISREG(file_inode(req->file)->i_mode)) +- req->flags |= REQ_F_ISREG; +- +- kiocb->ki_pos = READ_ONCE(sqe->off); +- if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) { +- req->flags |= REQ_F_CUR_POS; +- kiocb->ki_pos = req->file->f_pos; +- } +- kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp)); +- kiocb->ki_flags = iocb_flags(kiocb->ki_filp); +- ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags)); +- if (unlikely(ret)) +- return ret; +- +- ioprio = READ_ONCE(sqe->ioprio); +- if (ioprio) { +- ret = ioprio_check_cap(ioprio); +- if (ret) +- return ret; +- +- kiocb->ki_ioprio = ioprio; +- } else +- kiocb->ki_ioprio = get_current_ioprio(); +- +- /* don't allow async punt if RWF_NOWAIT was requested */ +- if (kiocb->ki_flags & IOCB_NOWAIT) +- req->flags |= REQ_F_NOWAIT; +- +- if (ctx->flags & IORING_SETUP_IOPOLL) { +- if (!(kiocb->ki_flags & IOCB_DIRECT) || +- !kiocb->ki_filp->f_op->iopoll) +- return -EOPNOTSUPP; +- +- kiocb->ki_flags |= IOCB_HIPRI; +- kiocb->ki_complete = io_complete_rw_iopoll; +- req->iopoll_completed = 0; +- } else { +- if (kiocb->ki_flags & IOCB_HIPRI) +- return -EINVAL; +- kiocb->ki_complete = io_complete_rw; +- } +- +- req->rw.addr = READ_ONCE(sqe->addr); +- req->rw.len = READ_ONCE(sqe->len); +- req->buf_index = READ_ONCE(sqe->buf_index); +- return 0; +-} +- +-static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) +-{ +- switch (ret) { +- case -EIOCBQUEUED: +- break; +- case -ERESTARTSYS: +- case -ERESTARTNOINTR: +- case -ERESTARTNOHAND: +- case -ERESTART_RESTARTBLOCK: +- /* +- * We can't just restart the syscall, since previously +- * submitted sqes may already be in progress. Just fail this +- * IO with EINTR. +- */ +- ret = -EINTR; +- fallthrough; +- default: +- kiocb->ki_complete(kiocb, ret, 0); +- } +-} +- +-static void kiocb_done(struct kiocb *kiocb, ssize_t ret, +- struct io_comp_state *cs) +-{ +- struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); +- struct io_async_rw *io = req->async_data; +- +- /* add previously done IO, if any */ +- if (io && io->bytes_done > 0) { +- if (ret < 0) +- ret = io->bytes_done; +- else +- ret += io->bytes_done; +- } +- +- if (req->flags & REQ_F_CUR_POS) +- req->file->f_pos = kiocb->ki_pos; +- if (ret >= 0 && kiocb->ki_complete == io_complete_rw) +- __io_complete_rw(req, ret, 0, cs); +- else +- io_rw_done(kiocb, ret); +-} +- +-static ssize_t io_import_fixed(struct io_kiocb *req, int rw, +- struct iov_iter *iter) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- size_t len = req->rw.len; +- struct io_mapped_ubuf *imu; +- u16 index, buf_index = req->buf_index; +- size_t offset; +- u64 buf_addr; +- +- if (unlikely(buf_index >= ctx->nr_user_bufs)) +- return -EFAULT; +- index = array_index_nospec(buf_index, ctx->nr_user_bufs); +- imu = &ctx->user_bufs[index]; +- buf_addr = req->rw.addr; +- +- /* overflow */ +- if (buf_addr + len < buf_addr) +- return -EFAULT; +- /* not inside the mapped region */ +- if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len) +- return -EFAULT; +- +- /* +- * May not be a start of buffer, set size appropriately +- * and advance us to the beginning. +- */ +- offset = buf_addr - imu->ubuf; +- iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len); +- +- if (offset) { +- /* +- * Don't use iov_iter_advance() here, as it's really slow for +- * using the latter parts of a big fixed buffer - it iterates +- * over each segment manually. We can cheat a bit here, because +- * we know that: +- * +- * 1) it's a BVEC iter, we set it up +- * 2) all bvecs are PAGE_SIZE in size, except potentially the +- * first and last bvec +- * +- * So just find our index, and adjust the iterator afterwards. +- * If the offset is within the first bvec (or the whole first +- * bvec, just use iov_iter_advance(). This makes it easier +- * since we can just skip the first segment, which may not +- * be PAGE_SIZE aligned. +- */ +- const struct bio_vec *bvec = imu->bvec; +- +- if (offset <= bvec->bv_len) { +- iov_iter_advance(iter, offset); +- } else { +- unsigned long seg_skip; +- +- /* skip first vec */ +- offset -= bvec->bv_len; +- seg_skip = 1 + (offset >> PAGE_SHIFT); +- +- iter->bvec = bvec + seg_skip; +- iter->nr_segs -= seg_skip; +- iter->count -= bvec->bv_len + offset; +- iter->iov_offset = offset & ~PAGE_MASK; +- } +- } +- +- return len; +-} +- +-static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock) +-{ +- if (needs_lock) +- mutex_unlock(&ctx->uring_lock); +-} +- +-static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock) +-{ +- /* +- * "Normal" inline submissions always hold the uring_lock, since we +- * grab it from the system call. Same is true for the SQPOLL offload. +- * The only exception is when we've detached the request and issue it +- * from an async worker thread, grab the lock for that case. +- */ +- if (needs_lock) +- mutex_lock(&ctx->uring_lock); +-} +- +-static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len, +- int bgid, struct io_buffer *kbuf, +- bool needs_lock) +-{ +- struct io_buffer *head; +- +- if (req->flags & REQ_F_BUFFER_SELECTED) +- return kbuf; +- +- io_ring_submit_lock(req->ctx, needs_lock); +- +- lockdep_assert_held(&req->ctx->uring_lock); +- +- head = xa_load(&req->ctx->io_buffers, bgid); +- if (head) { +- if (!list_empty(&head->list)) { +- kbuf = list_last_entry(&head->list, struct io_buffer, +- list); +- list_del(&kbuf->list); +- } else { +- kbuf = head; +- xa_erase(&req->ctx->io_buffers, bgid); +- } +- if (*len > kbuf->len) +- *len = kbuf->len; +- } else { +- kbuf = ERR_PTR(-ENOBUFS); +- } +- +- io_ring_submit_unlock(req->ctx, needs_lock); +- +- return kbuf; +-} +- +-static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len, +- bool needs_lock) +-{ +- struct io_buffer *kbuf; +- u16 bgid; +- +- kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; +- bgid = req->buf_index; +- kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock); +- if (IS_ERR(kbuf)) +- return kbuf; +- req->rw.addr = (u64) (unsigned long) kbuf; +- req->flags |= REQ_F_BUFFER_SELECTED; +- return u64_to_user_ptr(kbuf->addr); +-} +- +-#ifdef CONFIG_COMPAT +-static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, +- bool needs_lock) +-{ +- struct compat_iovec __user *uiov; +- compat_ssize_t clen; +- void __user *buf; +- ssize_t len; +- +- uiov = u64_to_user_ptr(req->rw.addr); +- if (!access_ok(uiov, sizeof(*uiov))) +- return -EFAULT; +- if (__get_user(clen, &uiov->iov_len)) +- return -EFAULT; +- if (clen < 0) +- return -EINVAL; +- +- len = clen; +- buf = io_rw_buffer_select(req, &len, needs_lock); +- if (IS_ERR(buf)) +- return PTR_ERR(buf); +- iov[0].iov_base = buf; +- iov[0].iov_len = (compat_size_t) len; +- return 0; +-} +-#endif +- +-static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, +- bool needs_lock) +-{ +- struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr); +- void __user *buf; +- ssize_t len; +- +- if (copy_from_user(iov, uiov, sizeof(*uiov))) +- return -EFAULT; +- +- len = iov[0].iov_len; +- if (len < 0) +- return -EINVAL; +- buf = io_rw_buffer_select(req, &len, needs_lock); +- if (IS_ERR(buf)) +- return PTR_ERR(buf); +- iov[0].iov_base = buf; +- iov[0].iov_len = len; +- return 0; +-} +- +-static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, +- bool needs_lock) +-{ +- if (req->flags & REQ_F_BUFFER_SELECTED) { +- struct io_buffer *kbuf; +- +- kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; +- iov[0].iov_base = u64_to_user_ptr(kbuf->addr); +- iov[0].iov_len = kbuf->len; +- return 0; +- } +- if (req->rw.len != 1) +- return -EINVAL; +- +-#ifdef CONFIG_COMPAT +- if (req->ctx->compat) +- return io_compat_import(req, iov, needs_lock); +-#endif +- +- return __io_iov_buffer_select(req, iov, needs_lock); +-} +- +-static ssize_t __io_import_iovec(int rw, struct io_kiocb *req, +- struct iovec **iovec, struct iov_iter *iter, +- bool needs_lock) +-{ +- void __user *buf = u64_to_user_ptr(req->rw.addr); +- size_t sqe_len = req->rw.len; +- ssize_t ret; +- u8 opcode; +- +- opcode = req->opcode; +- if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) { +- *iovec = NULL; +- return io_import_fixed(req, rw, iter); +- } +- +- /* buffer index only valid with fixed read/write, or buffer select */ +- if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT)) +- return -EINVAL; +- +- if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) { +- if (req->flags & REQ_F_BUFFER_SELECT) { +- buf = io_rw_buffer_select(req, &sqe_len, needs_lock); +- if (IS_ERR(buf)) +- return PTR_ERR(buf); +- req->rw.len = sqe_len; +- } +- +- ret = import_single_range(rw, buf, sqe_len, *iovec, iter); +- *iovec = NULL; +- return ret; +- } +- +- if (req->flags & REQ_F_BUFFER_SELECT) { +- ret = io_iov_buffer_select(req, *iovec, needs_lock); +- if (!ret) { +- ret = (*iovec)->iov_len; +- iov_iter_init(iter, rw, *iovec, 1, ret); +- } +- *iovec = NULL; +- return ret; +- } +- +- return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter, +- req->ctx->compat); +-} +- +-static ssize_t io_import_iovec(int rw, struct io_kiocb *req, +- struct iovec **iovec, struct iov_iter *iter, +- bool needs_lock) +-{ +- struct io_async_rw *iorw = req->async_data; +- +- if (!iorw) +- return __io_import_iovec(rw, req, iovec, iter, needs_lock); +- *iovec = NULL; +- return 0; +-} +- +-static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb) +-{ +- return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos; +-} +- +-/* +- * For files that don't have ->read_iter() and ->write_iter(), handle them +- * by looping over ->read() or ->write() manually. +- */ +-static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) +-{ +- struct kiocb *kiocb = &req->rw.kiocb; +- struct file *file = req->file; +- ssize_t ret = 0; +- +- /* +- * Don't support polled IO through this interface, and we can't +- * support non-blocking either. For the latter, this just causes +- * the kiocb to be handled from an async context. +- */ +- if (kiocb->ki_flags & IOCB_HIPRI) +- return -EOPNOTSUPP; +- if (kiocb->ki_flags & IOCB_NOWAIT) +- return -EAGAIN; +- +- while (iov_iter_count(iter)) { +- struct iovec iovec; +- ssize_t nr; +- +- if (!iov_iter_is_bvec(iter)) { +- iovec = iov_iter_iovec(iter); +- } else { +- iovec.iov_base = u64_to_user_ptr(req->rw.addr); +- iovec.iov_len = req->rw.len; +- } +- +- if (rw == READ) { +- nr = file->f_op->read(file, iovec.iov_base, +- iovec.iov_len, io_kiocb_ppos(kiocb)); +- } else { +- nr = file->f_op->write(file, iovec.iov_base, +- iovec.iov_len, io_kiocb_ppos(kiocb)); +- } +- +- if (nr < 0) { +- if (!ret) +- ret = nr; +- break; +- } +- ret += nr; +- if (!iov_iter_is_bvec(iter)) { +- iov_iter_advance(iter, nr); +- } else { +- req->rw.addr += nr; +- req->rw.len -= nr; +- if (!req->rw.len) +- break; +- } +- if (nr != iovec.iov_len) +- break; +- } +- +- return ret; +-} +- +-static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, +- const struct iovec *fast_iov, struct iov_iter *iter) +-{ +- struct io_async_rw *rw = req->async_data; +- +- memcpy(&rw->iter, iter, sizeof(*iter)); +- rw->free_iovec = iovec; +- rw->bytes_done = 0; +- /* can only be fixed buffers, no need to do anything */ +- if (iov_iter_is_bvec(iter)) +- return; +- if (!iovec) { +- unsigned iov_off = 0; +- +- rw->iter.iov = rw->fast_iov; +- if (iter->iov != fast_iov) { +- iov_off = iter->iov - fast_iov; +- rw->iter.iov += iov_off; +- } +- if (rw->fast_iov != fast_iov) +- memcpy(rw->fast_iov + iov_off, fast_iov + iov_off, +- sizeof(struct iovec) * iter->nr_segs); +- } else { +- req->flags |= REQ_F_NEED_CLEANUP; +- } +-} +- +-static inline int __io_alloc_async_data(struct io_kiocb *req) +-{ +- WARN_ON_ONCE(!io_op_defs[req->opcode].async_size); +- req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL); +- return req->async_data == NULL; +-} +- +-static int io_alloc_async_data(struct io_kiocb *req) +-{ +- if (!io_op_defs[req->opcode].needs_async_data) +- return 0; +- +- return __io_alloc_async_data(req); +-} +- +-static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, +- const struct iovec *fast_iov, +- struct iov_iter *iter, bool force) +-{ +- if (!force && !io_op_defs[req->opcode].needs_async_data) +- return 0; +- if (!req->async_data) { +- if (__io_alloc_async_data(req)) +- return -ENOMEM; +- +- io_req_map_rw(req, iovec, fast_iov, iter); +- } +- return 0; +-} +- +-static inline int io_rw_prep_async(struct io_kiocb *req, int rw) +-{ +- struct io_async_rw *iorw = req->async_data; +- struct iovec *iov = iorw->fast_iov; +- ssize_t ret; +- +- ret = __io_import_iovec(rw, req, &iov, &iorw->iter, false); +- if (unlikely(ret < 0)) +- return ret; +- +- iorw->bytes_done = 0; +- iorw->free_iovec = iov; +- if (iov) +- req->flags |= REQ_F_NEED_CLEANUP; +- return 0; +-} +- +-static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- ssize_t ret; +- +- ret = io_prep_rw(req, sqe); +- if (ret) +- return ret; +- +- if (unlikely(!(req->file->f_mode & FMODE_READ))) +- return -EBADF; +- +- /* either don't need iovec imported or already have it */ +- if (!req->async_data) +- return 0; +- return io_rw_prep_async(req, READ); +-} +- +-/* +- * This is our waitqueue callback handler, registered through lock_page_async() +- * when we initially tried to do the IO with the iocb armed our waitqueue. +- * This gets called when the page is unlocked, and we generally expect that to +- * happen when the page IO is completed and the page is now uptodate. This will +- * queue a task_work based retry of the operation, attempting to copy the data +- * again. If the latter fails because the page was NOT uptodate, then we will +- * do a thread based blocking retry of the operation. That's the unexpected +- * slow path. +- */ +-static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, +- int sync, void *arg) +-{ +- struct wait_page_queue *wpq; +- struct io_kiocb *req = wait->private; +- struct wait_page_key *key = arg; +- int ret; +- +- wpq = container_of(wait, struct wait_page_queue, wait); +- +- if (!wake_page_match(wpq, key)) +- return 0; +- +- req->rw.kiocb.ki_flags &= ~IOCB_WAITQ; +- list_del_init(&wait->entry); +- +- init_task_work(&req->task_work, io_req_task_submit); +- percpu_ref_get(&req->ctx->refs); +- +- /* submit ref gets dropped, acquire a new one */ +- refcount_inc(&req->refs); +- ret = io_req_task_work_add(req, true); +- if (unlikely(ret)) { +- struct task_struct *tsk; +- +- /* queue just for cancelation */ +- init_task_work(&req->task_work, io_req_task_cancel); +- tsk = io_wq_get_task(req->ctx->io_wq); +- task_work_add(tsk, &req->task_work, TWA_NONE); +- wake_up_process(tsk); +- } +- return 1; +-} +- +-/* +- * This controls whether a given IO request should be armed for async page +- * based retry. If we return false here, the request is handed to the async +- * worker threads for retry. If we're doing buffered reads on a regular file, +- * we prepare a private wait_page_queue entry and retry the operation. This +- * will either succeed because the page is now uptodate and unlocked, or it +- * will register a callback when the page is unlocked at IO completion. Through +- * that callback, io_uring uses task_work to setup a retry of the operation. +- * That retry will attempt the buffered read again. The retry will generally +- * succeed, or in rare cases where it fails, we then fall back to using the +- * async worker threads for a blocking retry. +- */ +-static bool io_rw_should_retry(struct io_kiocb *req) +-{ +- struct io_async_rw *rw = req->async_data; +- struct wait_page_queue *wait = &rw->wpq; +- struct kiocb *kiocb = &req->rw.kiocb; +- +- /* never retry for NOWAIT, we just complete with -EAGAIN */ +- if (req->flags & REQ_F_NOWAIT) +- return false; +- +- /* Only for buffered IO */ +- if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI)) +- return false; +- +- /* +- * just use poll if we can, and don't attempt if the fs doesn't +- * support callback based unlocks +- */ +- if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC)) +- return false; +- +- wait->wait.func = io_async_buf_func; +- wait->wait.private = req; +- wait->wait.flags = 0; +- INIT_LIST_HEAD(&wait->wait.entry); +- kiocb->ki_flags |= IOCB_WAITQ; +- kiocb->ki_flags &= ~IOCB_NOWAIT; +- kiocb->ki_waitq = wait; +- return true; +-} +- +-static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter) +-{ +- if (req->file->f_op->read_iter) +- return call_read_iter(req->file, &req->rw.kiocb, iter); +- else if (req->file->f_op->read) +- return loop_rw_iter(READ, req, iter); +- else +- return -EINVAL; +-} +- +-static int io_read(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; +- struct kiocb *kiocb = &req->rw.kiocb; +- struct iov_iter __iter, *iter = &__iter; +- struct iov_iter iter_cp; +- struct io_async_rw *rw = req->async_data; +- ssize_t io_size, ret, ret2; +- bool no_async; +- +- if (rw) +- iter = &rw->iter; +- +- ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock); +- if (ret < 0) +- return ret; +- iter_cp = *iter; +- io_size = iov_iter_count(iter); +- req->result = io_size; +- ret = 0; +- +- /* Ensure we clear previously set non-block flag */ +- if (!force_nonblock) +- kiocb->ki_flags &= ~IOCB_NOWAIT; +- else +- kiocb->ki_flags |= IOCB_NOWAIT; +- +- +- /* If the file doesn't support async, just async punt */ +- no_async = force_nonblock && !io_file_supports_async(req->file, READ); +- if (no_async) +- goto copy_iov; +- +- ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size); +- if (unlikely(ret)) +- goto out_free; +- +- ret = io_iter_do_read(req, iter); +- +- if (!ret) { +- goto done; +- } else if (ret == -EIOCBQUEUED) { +- ret = 0; +- goto out_free; +- } else if (ret == -EAGAIN) { +- /* IOPOLL retry should happen for io-wq threads */ +- if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) +- goto done; +- /* no retry on NONBLOCK marked file */ +- if (req->file->f_flags & O_NONBLOCK) +- goto done; +- /* some cases will consume bytes even on error returns */ +- *iter = iter_cp; +- ret = 0; +- goto copy_iov; +- } else if (ret < 0) { +- /* make sure -ERESTARTSYS -> -EINTR is done */ +- goto done; +- } +- +- /* read it all, or we did blocking attempt. no retry. */ +- if (!iov_iter_count(iter) || !force_nonblock || +- (req->file->f_flags & O_NONBLOCK) || !(req->flags & REQ_F_ISREG)) +- goto done; +- +- io_size -= ret; +-copy_iov: +- ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true); +- if (ret2) { +- ret = ret2; +- goto out_free; +- } +- if (no_async) +- return -EAGAIN; +- rw = req->async_data; +- /* it's copied and will be cleaned with ->io */ +- iovec = NULL; +- /* now use our persistent iterator, if we aren't already */ +- iter = &rw->iter; +-retry: +- rw->bytes_done += ret; +- /* if we can retry, do so with the callbacks armed */ +- if (!io_rw_should_retry(req)) { +- kiocb->ki_flags &= ~IOCB_WAITQ; +- return -EAGAIN; +- } +- +- /* +- * Now retry read with the IOCB_WAITQ parts set in the iocb. If we +- * get -EIOCBQUEUED, then we'll get a notification when the desired +- * page gets unlocked. We can also get a partial read here, and if we +- * do, then just retry at the new offset. +- */ +- ret = io_iter_do_read(req, iter); +- if (ret == -EIOCBQUEUED) { +- ret = 0; +- goto out_free; +- } else if (ret > 0 && ret < io_size) { +- /* we got some bytes, but not all. retry. */ +- kiocb->ki_flags &= ~IOCB_WAITQ; +- goto retry; +- } +-done: +- kiocb_done(kiocb, ret, cs); +- ret = 0; +-out_free: +- /* it's reportedly faster than delegating the null check to kfree() */ +- if (iovec) +- kfree(iovec); +- return ret; +-} +- +-static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- ssize_t ret; +- +- ret = io_prep_rw(req, sqe); +- if (ret) +- return ret; +- +- if (unlikely(!(req->file->f_mode & FMODE_WRITE))) +- return -EBADF; +- +- /* either don't need iovec imported or already have it */ +- if (!req->async_data) +- return 0; +- return io_rw_prep_async(req, WRITE); +-} +- +-static int io_write(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; +- struct kiocb *kiocb = &req->rw.kiocb; +- struct iov_iter __iter, *iter = &__iter; +- struct iov_iter iter_cp; +- struct io_async_rw *rw = req->async_data; +- ssize_t ret, ret2, io_size; +- +- if (rw) +- iter = &rw->iter; +- +- ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock); +- if (ret < 0) +- return ret; +- iter_cp = *iter; +- io_size = iov_iter_count(iter); +- req->result = io_size; +- +- /* Ensure we clear previously set non-block flag */ +- if (!force_nonblock) +- kiocb->ki_flags &= ~IOCB_NOWAIT; +- else +- kiocb->ki_flags |= IOCB_NOWAIT; +- +- /* If the file doesn't support async, just async punt */ +- if (force_nonblock && !io_file_supports_async(req->file, WRITE)) +- goto copy_iov; +- +- /* file path doesn't support NOWAIT for non-direct_IO */ +- if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) && +- (req->flags & REQ_F_ISREG)) +- goto copy_iov; +- +- ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), io_size); +- if (unlikely(ret)) +- goto out_free; +- +- /* +- * Open-code file_start_write here to grab freeze protection, +- * which will be released by another thread in +- * io_complete_rw(). Fool lockdep by telling it the lock got +- * released so that it doesn't complain about the held lock when +- * we return to userspace. +- */ +- if (req->flags & REQ_F_ISREG) { +- sb_start_write(file_inode(req->file)->i_sb); +- __sb_writers_release(file_inode(req->file)->i_sb, +- SB_FREEZE_WRITE); +- } +- kiocb->ki_flags |= IOCB_WRITE; +- +- if (req->file->f_op->write_iter) +- ret2 = call_write_iter(req->file, kiocb, iter); +- else if (req->file->f_op->write) +- ret2 = loop_rw_iter(WRITE, req, iter); +- else +- ret2 = -EINVAL; +- +- /* +- * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just +- * retry them without IOCB_NOWAIT. +- */ +- if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) +- ret2 = -EAGAIN; +- /* no retry on NONBLOCK marked file */ +- if (ret2 == -EAGAIN && (req->file->f_flags & O_NONBLOCK)) +- goto done; +- if (!force_nonblock || ret2 != -EAGAIN) { +- /* IOPOLL retry should happen for io-wq threads */ +- if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN) +- goto copy_iov; +-done: +- kiocb_done(kiocb, ret2, cs); +- } else { +-copy_iov: +- /* some cases will consume bytes even on error returns */ +- *iter = iter_cp; +- ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false); +- if (!ret) +- return -EAGAIN; +- } +-out_free: +- /* it's reportedly faster than delegating the null check to kfree() */ +- if (iovec) +- kfree(iovec); +- return ret; +-} +- +-static int __io_splice_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +- struct io_splice* sp = &req->splice; +- unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL; +- +- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- +- sp->file_in = NULL; +- sp->len = READ_ONCE(sqe->len); +- sp->flags = READ_ONCE(sqe->splice_flags); +- +- if (unlikely(sp->flags & ~valid_flags)) +- return -EINVAL; +- +- sp->file_in = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in), +- (sp->flags & SPLICE_F_FD_IN_FIXED)); +- if (!sp->file_in) +- return -EBADF; +- req->flags |= REQ_F_NEED_CLEANUP; +- +- if (!S_ISREG(file_inode(sp->file_in)->i_mode)) { +- /* +- * Splice operation will be punted aync, and here need to +- * modify io_wq_work.flags, so initialize io_wq_work firstly. +- */ +- io_req_init_async(req); +- req->work.flags |= IO_WQ_WORK_UNBOUND; +- } +- +- return 0; +-} +- +-static int io_tee_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +- if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off)) +- return -EINVAL; +- return __io_splice_prep(req, sqe); +-} +- +-static int io_tee(struct io_kiocb *req, bool force_nonblock) +-{ +- struct io_splice *sp = &req->splice; +- struct file *in = sp->file_in; +- struct file *out = sp->file_out; +- unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; +- long ret = 0; +- +- if (force_nonblock) +- return -EAGAIN; +- if (sp->len) +- ret = do_tee(in, out, sp->len, flags); +- +- io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED)); +- req->flags &= ~REQ_F_NEED_CLEANUP; +- +- if (ret != sp->len) +- req_set_fail_links(req); +- io_req_complete(req, ret); +- return 0; +-} +- +-static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- struct io_splice* sp = &req->splice; +- +- sp->off_in = READ_ONCE(sqe->splice_off_in); +- sp->off_out = READ_ONCE(sqe->off); +- return __io_splice_prep(req, sqe); +-} +- +-static int io_splice(struct io_kiocb *req, bool force_nonblock) +-{ +- struct io_splice *sp = &req->splice; +- struct file *in = sp->file_in; +- struct file *out = sp->file_out; +- unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; +- loff_t *poff_in, *poff_out; +- long ret = 0; +- +- if (force_nonblock) +- return -EAGAIN; +- +- poff_in = (sp->off_in == -1) ? NULL : &sp->off_in; +- poff_out = (sp->off_out == -1) ? NULL : &sp->off_out; +- +- if (sp->len) +- ret = do_splice(in, poff_in, out, poff_out, sp->len, flags); +- +- io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED)); +- req->flags &= ~REQ_F_NEED_CLEANUP; +- +- if (ret != sp->len) +- req_set_fail_links(req); +- io_req_complete(req, ret); +- return 0; +-} +- +-/* +- * IORING_OP_NOP just posts a completion event, nothing else. +- */ +-static int io_nop(struct io_kiocb *req, struct io_comp_state *cs) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- +- if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- +- __io_req_complete(req, 0, 0, cs); +- return 0; +-} +- +-static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- +- if (!req->file) +- return -EBADF; +- +- if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index || +- sqe->splice_fd_in)) +- return -EINVAL; +- +- req->sync.flags = READ_ONCE(sqe->fsync_flags); +- if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC)) +- return -EINVAL; +- +- req->sync.off = READ_ONCE(sqe->off); +- req->sync.len = READ_ONCE(sqe->len); +- return 0; +-} +- +-static int io_fsync(struct io_kiocb *req, bool force_nonblock) +-{ +- loff_t end = req->sync.off + req->sync.len; +- int ret; +- +- /* fsync always requires a blocking context */ +- if (force_nonblock) +- return -EAGAIN; +- +- ret = vfs_fsync_range(req->file, req->sync.off, +- end > 0 ? end : LLONG_MAX, +- req->sync.flags & IORING_FSYNC_DATASYNC); +- if (ret < 0) +- req_set_fail_links(req); +- io_req_complete(req, ret); +- return 0; +-} +- +-static int io_fallocate_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +- if (sqe->ioprio || sqe->buf_index || sqe->rw_flags || +- sqe->splice_fd_in) +- return -EINVAL; +- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- +- req->sync.off = READ_ONCE(sqe->off); +- req->sync.len = READ_ONCE(sqe->addr); +- req->sync.mode = READ_ONCE(sqe->len); +- return 0; +-} +- +-static int io_fallocate(struct io_kiocb *req, bool force_nonblock) +-{ +- int ret; +- +- /* fallocate always requiring blocking context */ +- if (force_nonblock) +- return -EAGAIN; +- ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off, +- req->sync.len); +- if (ret < 0) +- req_set_fail_links(req); +- io_req_complete(req, ret); +- return 0; +-} +- +-static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- const char __user *fname; +- int ret; +- +- if (unlikely(sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)) +- return -EINVAL; +- if (unlikely(req->flags & REQ_F_FIXED_FILE)) +- return -EBADF; +- +- /* open.how should be already initialised */ +- if (!(req->open.how.flags & O_PATH) && force_o_largefile()) +- req->open.how.flags |= O_LARGEFILE; +- +- req->open.dfd = READ_ONCE(sqe->fd); +- fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); +- req->open.filename = getname(fname); +- if (IS_ERR(req->open.filename)) { +- ret = PTR_ERR(req->open.filename); +- req->open.filename = NULL; +- return ret; +- } +- req->open.nofile = rlimit(RLIMIT_NOFILE); +- req->open.ignore_nonblock = false; +- req->flags |= REQ_F_NEED_CLEANUP; +- return 0; +-} +- +-static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- u64 flags, mode; +- +- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL))) +- return -EINVAL; +- mode = READ_ONCE(sqe->len); +- flags = READ_ONCE(sqe->open_flags); +- req->open.how = build_open_how(flags, mode); +- return __io_openat_prep(req, sqe); +-} +- +-static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- struct open_how __user *how; +- size_t len; +- int ret; +- +- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL))) +- return -EINVAL; +- how = u64_to_user_ptr(READ_ONCE(sqe->addr2)); +- len = READ_ONCE(sqe->len); +- if (len < OPEN_HOW_SIZE_VER0) +- return -EINVAL; +- +- ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how, +- len); +- if (ret) +- return ret; +- +- return __io_openat_prep(req, sqe); +-} +- +-static int io_openat2(struct io_kiocb *req, bool force_nonblock) +-{ +- struct open_flags op; +- struct file *file; +- int ret; +- +- if (force_nonblock && !req->open.ignore_nonblock) +- return -EAGAIN; +- +- ret = build_open_flags(&req->open.how, &op); +- if (ret) +- goto err; +- +- ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile); +- if (ret < 0) +- goto err; +- +- file = do_filp_open(req->open.dfd, req->open.filename, &op); +- if (IS_ERR(file)) { +- put_unused_fd(ret); +- ret = PTR_ERR(file); +- /* +- * A work-around to ensure that /proc/self works that way +- * that it should - if we get -EOPNOTSUPP back, then assume +- * that proc_self_get_link() failed us because we're in async +- * context. We should be safe to retry this from the task +- * itself with force_nonblock == false set, as it should not +- * block on lookup. Would be nice to know this upfront and +- * avoid the async dance, but doesn't seem feasible. +- */ +- if (ret == -EOPNOTSUPP && io_wq_current_is_worker()) { +- req->open.ignore_nonblock = true; +- refcount_inc(&req->refs); +- io_req_task_queue(req); +- return 0; +- } +- } else { +- fsnotify_open(file); +- fd_install(ret, file); +- } +-err: +- putname(req->open.filename); +- req->flags &= ~REQ_F_NEED_CLEANUP; +- if (ret < 0) +- req_set_fail_links(req); +- io_req_complete(req, ret); +- return 0; +-} +- +-static int io_openat(struct io_kiocb *req, bool force_nonblock) +-{ +- return io_openat2(req, force_nonblock); +-} +- +-static int io_remove_buffers_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +- struct io_provide_buf *p = &req->pbuf; +- u64 tmp; +- +- if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off || +- sqe->splice_fd_in) +- return -EINVAL; +- +- tmp = READ_ONCE(sqe->fd); +- if (!tmp || tmp > USHRT_MAX) +- return -EINVAL; +- +- memset(p, 0, sizeof(*p)); +- p->nbufs = tmp; +- p->bgid = READ_ONCE(sqe->buf_group); +- return 0; +-} +- +-static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf, +- int bgid, unsigned nbufs) +-{ +- unsigned i = 0; +- +- /* shouldn't happen */ +- if (!nbufs) +- return 0; +- +- /* the head kbuf is the list itself */ +- while (!list_empty(&buf->list)) { +- struct io_buffer *nxt; +- +- nxt = list_first_entry(&buf->list, struct io_buffer, list); +- list_del(&nxt->list); +- kfree(nxt); +- if (++i == nbufs) +- return i; +- } +- i++; +- kfree(buf); +- xa_erase(&ctx->io_buffers, bgid); +- +- return i; +-} +- +-static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct io_provide_buf *p = &req->pbuf; +- struct io_ring_ctx *ctx = req->ctx; +- struct io_buffer *head; +- int ret = 0; +- +- io_ring_submit_lock(ctx, !force_nonblock); +- +- lockdep_assert_held(&ctx->uring_lock); +- +- ret = -ENOENT; +- head = xa_load(&ctx->io_buffers, p->bgid); +- if (head) +- ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs); +- if (ret < 0) +- req_set_fail_links(req); +- +- /* need to hold the lock to complete IOPOLL requests */ +- if (ctx->flags & IORING_SETUP_IOPOLL) { +- __io_req_complete(req, ret, 0, cs); +- io_ring_submit_unlock(ctx, !force_nonblock); +- } else { +- io_ring_submit_unlock(ctx, !force_nonblock); +- __io_req_complete(req, ret, 0, cs); +- } +- return 0; +-} +- +-static int io_provide_buffers_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +- unsigned long size, tmp_check; +- struct io_provide_buf *p = &req->pbuf; +- u64 tmp; +- +- if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in) +- return -EINVAL; +- +- tmp = READ_ONCE(sqe->fd); +- if (!tmp || tmp > USHRT_MAX) +- return -E2BIG; +- p->nbufs = tmp; +- p->addr = READ_ONCE(sqe->addr); +- p->len = READ_ONCE(sqe->len); +- +- if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs, +- &size)) +- return -EOVERFLOW; +- if (check_add_overflow((unsigned long)p->addr, size, &tmp_check)) +- return -EOVERFLOW; +- +- size = (unsigned long)p->len * p->nbufs; +- if (!access_ok(u64_to_user_ptr(p->addr), size)) +- return -EFAULT; +- +- p->bgid = READ_ONCE(sqe->buf_group); +- tmp = READ_ONCE(sqe->off); +- if (tmp > USHRT_MAX) +- return -E2BIG; +- p->bid = tmp; +- return 0; +-} +- +-static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head) +-{ +- struct io_buffer *buf; +- u64 addr = pbuf->addr; +- int i, bid = pbuf->bid; +- +- for (i = 0; i < pbuf->nbufs; i++) { +- buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT); +- if (!buf) +- break; +- +- buf->addr = addr; +- buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT); +- buf->bid = bid; +- addr += pbuf->len; +- bid++; +- if (!*head) { +- INIT_LIST_HEAD(&buf->list); +- *head = buf; +- } else { +- list_add_tail(&buf->list, &(*head)->list); +- } +- cond_resched(); +- } +- +- return i ? i : -ENOMEM; +-} +- +-static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct io_provide_buf *p = &req->pbuf; +- struct io_ring_ctx *ctx = req->ctx; +- struct io_buffer *head, *list; +- int ret = 0; +- +- io_ring_submit_lock(ctx, !force_nonblock); +- +- lockdep_assert_held(&ctx->uring_lock); +- +- list = head = xa_load(&ctx->io_buffers, p->bgid); +- +- ret = io_add_buffers(p, &head); +- if (ret >= 0 && !list) { +- ret = xa_insert(&ctx->io_buffers, p->bgid, head, GFP_KERNEL); +- if (ret < 0) +- __io_remove_buffers(ctx, head, p->bgid, -1U); +- } +- if (ret < 0) +- req_set_fail_links(req); +- +- /* need to hold the lock to complete IOPOLL requests */ +- if (ctx->flags & IORING_SETUP_IOPOLL) { +- __io_req_complete(req, ret, 0, cs); +- io_ring_submit_unlock(ctx, !force_nonblock); +- } else { +- io_ring_submit_unlock(ctx, !force_nonblock); +- __io_req_complete(req, ret, 0, cs); +- } +- return 0; +-} +- +-static int io_epoll_ctl_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +-#if defined(CONFIG_EPOLL) +- if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) +- return -EINVAL; +- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL))) +- return -EINVAL; +- +- req->epoll.epfd = READ_ONCE(sqe->fd); +- req->epoll.op = READ_ONCE(sqe->len); +- req->epoll.fd = READ_ONCE(sqe->off); +- +- if (ep_op_has_event(req->epoll.op)) { +- struct epoll_event __user *ev; +- +- ev = u64_to_user_ptr(READ_ONCE(sqe->addr)); +- if (copy_from_user(&req->epoll.event, ev, sizeof(*ev))) +- return -EFAULT; +- } +- +- return 0; +-#else +- return -EOPNOTSUPP; +-#endif +-} +- +-static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +-#if defined(CONFIG_EPOLL) +- struct io_epoll *ie = &req->epoll; +- int ret; +- +- ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock); +- if (force_nonblock && ret == -EAGAIN) +- return -EAGAIN; +- +- if (ret < 0) +- req_set_fail_links(req); +- __io_req_complete(req, ret, 0, cs); +- return 0; +-#else +- return -EOPNOTSUPP; +-#endif +-} +- +-static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +-#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) +- if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->splice_fd_in) +- return -EINVAL; +- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- +- req->madvise.addr = READ_ONCE(sqe->addr); +- req->madvise.len = READ_ONCE(sqe->len); +- req->madvise.advice = READ_ONCE(sqe->fadvise_advice); +- return 0; +-#else +- return -EOPNOTSUPP; +-#endif +-} +- +-static int io_madvise(struct io_kiocb *req, bool force_nonblock) +-{ +-#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) +- struct io_madvise *ma = &req->madvise; +- int ret; +- +- if (force_nonblock) +- return -EAGAIN; +- +- ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice); +- if (ret < 0) +- req_set_fail_links(req); +- io_req_complete(req, ret); +- return 0; +-#else +- return -EOPNOTSUPP; +-#endif +-} +- +-static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- if (sqe->ioprio || sqe->buf_index || sqe->addr || sqe->splice_fd_in) +- return -EINVAL; +- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- +- req->fadvise.offset = READ_ONCE(sqe->off); +- req->fadvise.len = READ_ONCE(sqe->len); +- req->fadvise.advice = READ_ONCE(sqe->fadvise_advice); +- return 0; +-} +- +-static int io_fadvise(struct io_kiocb *req, bool force_nonblock) +-{ +- struct io_fadvise *fa = &req->fadvise; +- int ret; +- +- if (force_nonblock) { +- switch (fa->advice) { +- case POSIX_FADV_NORMAL: +- case POSIX_FADV_RANDOM: +- case POSIX_FADV_SEQUENTIAL: +- break; +- default: +- return -EAGAIN; +- } +- } +- +- ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice); +- if (ret < 0) +- req_set_fail_links(req); +- io_req_complete(req, ret); +- return 0; +-} +- +-static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL))) +- return -EINVAL; +- if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) +- return -EINVAL; +- if (req->flags & REQ_F_FIXED_FILE) +- return -EBADF; +- +- req->statx.dfd = READ_ONCE(sqe->fd); +- req->statx.mask = READ_ONCE(sqe->len); +- req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr)); +- req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2)); +- req->statx.flags = READ_ONCE(sqe->statx_flags); +- +- return 0; +-} +- +-static int io_statx(struct io_kiocb *req, bool force_nonblock) +-{ +- struct io_statx *ctx = &req->statx; +- int ret; +- +- if (force_nonblock) +- return -EAGAIN; +- +- ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask, +- ctx->buffer); +- +- if (ret < 0) +- req_set_fail_links(req); +- io_req_complete(req, ret); +- return 0; +-} +- +-static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- /* +- * If we queue this for async, it must not be cancellable. That would +- * leave the 'file' in an undeterminate state, and here need to modify +- * io_wq_work.flags, so initialize io_wq_work firstly. +- */ +- io_req_init_async(req); +- +- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL))) +- return -EINVAL; +- if (sqe->ioprio || sqe->off || sqe->addr || sqe->len || +- sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in) +- return -EINVAL; +- if (req->flags & REQ_F_FIXED_FILE) +- return -EBADF; +- +- req->close.fd = READ_ONCE(sqe->fd); +- if ((req->file && req->file->f_op == &io_uring_fops)) +- return -EBADF; +- +- req->close.put_file = NULL; +- return 0; +-} +- +-static int io_close(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct io_close *close = &req->close; +- int ret; +- +- /* might be already done during nonblock submission */ +- if (!close->put_file) { +- ret = close_fd_get_file(close->fd, &close->put_file); +- if (ret < 0) +- return (ret == -ENOENT) ? -EBADF : ret; +- } +- +- /* if the file has a flush method, be safe and punt to async */ +- if (close->put_file->f_op->flush && force_nonblock) { +- /* not safe to cancel at this point */ +- req->work.flags |= IO_WQ_WORK_NO_CANCEL; +- /* was never set, but play safe */ +- req->flags &= ~REQ_F_NOWAIT; +- /* avoid grabbing files - we don't need the files */ +- req->flags |= REQ_F_NO_FILE_TABLE; +- return -EAGAIN; +- } +- +- /* No ->flush() or already async, safely close from here */ +- ret = filp_close(close->put_file, req->work.identity->files); +- if (ret < 0) +- req_set_fail_links(req); +- fput(close->put_file); +- close->put_file = NULL; +- __io_req_complete(req, ret, 0, cs); +- return 0; +-} +- +-static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- +- if (!req->file) +- return -EBADF; +- +- if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index || +- sqe->splice_fd_in)) +- return -EINVAL; +- +- req->sync.off = READ_ONCE(sqe->off); +- req->sync.len = READ_ONCE(sqe->len); +- req->sync.flags = READ_ONCE(sqe->sync_range_flags); +- return 0; +-} +- +-static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock) +-{ +- int ret; +- +- /* sync_file_range always requires a blocking context */ +- if (force_nonblock) +- return -EAGAIN; +- +- ret = sync_file_range(req->file, req->sync.off, req->sync.len, +- req->sync.flags); +- if (ret < 0) +- req_set_fail_links(req); +- io_req_complete(req, ret); +- return 0; +-} +- +-#if defined(CONFIG_NET) +-static int io_setup_async_msg(struct io_kiocb *req, +- struct io_async_msghdr *kmsg) +-{ +- struct io_async_msghdr *async_msg = req->async_data; +- +- if (async_msg) +- return -EAGAIN; +- if (io_alloc_async_data(req)) { +- if (kmsg->iov != kmsg->fast_iov) +- kfree(kmsg->iov); +- return -ENOMEM; +- } +- async_msg = req->async_data; +- req->flags |= REQ_F_NEED_CLEANUP; +- memcpy(async_msg, kmsg, sizeof(*kmsg)); +- return -EAGAIN; +-} +- +-static int io_sendmsg_copy_hdr(struct io_kiocb *req, +- struct io_async_msghdr *iomsg) +-{ +- iomsg->iov = iomsg->fast_iov; +- iomsg->msg.msg_name = &iomsg->addr; +- return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg, +- req->sr_msg.msg_flags, &iomsg->iov); +-} +- +-static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- struct io_async_msghdr *async_msg = req->async_data; +- struct io_sr_msg *sr = &req->sr_msg; +- int ret; +- +- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- if (unlikely(sqe->addr2 || sqe->splice_fd_in || sqe->ioprio)) +- return -EINVAL; +- +- sr->msg_flags = READ_ONCE(sqe->msg_flags); +- sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); +- sr->len = READ_ONCE(sqe->len); +- +-#ifdef CONFIG_COMPAT +- if (req->ctx->compat) +- sr->msg_flags |= MSG_CMSG_COMPAT; +-#endif +- +- if (!async_msg || !io_op_defs[req->opcode].needs_async_data) +- return 0; +- ret = io_sendmsg_copy_hdr(req, async_msg); +- if (!ret) +- req->flags |= REQ_F_NEED_CLEANUP; +- return ret; +-} +- +-static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct io_async_msghdr iomsg, *kmsg; +- struct socket *sock; +- unsigned flags; +- int min_ret = 0; +- int ret; +- +- sock = sock_from_file(req->file, &ret); +- if (unlikely(!sock)) +- return ret; +- +- if (req->async_data) { +- kmsg = req->async_data; +- kmsg->msg.msg_name = &kmsg->addr; +- /* if iov is set, it's allocated already */ +- if (!kmsg->iov) +- kmsg->iov = kmsg->fast_iov; +- kmsg->msg.msg_iter.iov = kmsg->iov; +- } else { +- ret = io_sendmsg_copy_hdr(req, &iomsg); +- if (ret) +- return ret; +- kmsg = &iomsg; +- } +- +- flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; +- if (flags & MSG_DONTWAIT) +- req->flags |= REQ_F_NOWAIT; +- else if (force_nonblock) +- flags |= MSG_DONTWAIT; +- +- if (flags & MSG_WAITALL) +- min_ret = iov_iter_count(&kmsg->msg.msg_iter); +- +- ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); +- if (force_nonblock && ret == -EAGAIN) +- return io_setup_async_msg(req, kmsg); +- if (ret == -ERESTARTSYS) +- ret = -EINTR; +- +- if (kmsg->iov != kmsg->fast_iov) +- kfree(kmsg->iov); +- req->flags &= ~REQ_F_NEED_CLEANUP; +- if (ret < min_ret) +- req_set_fail_links(req); +- __io_req_complete(req, ret, 0, cs); +- return 0; +-} +- +-static int io_send(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct io_sr_msg *sr = &req->sr_msg; +- struct msghdr msg; +- struct iovec iov; +- struct socket *sock; +- unsigned flags; +- int min_ret = 0; +- int ret; +- +- sock = sock_from_file(req->file, &ret); +- if (unlikely(!sock)) +- return ret; +- +- ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter); +- if (unlikely(ret)) +- return ret; +- +- msg.msg_name = NULL; +- msg.msg_control = NULL; +- msg.msg_controllen = 0; +- msg.msg_namelen = 0; +- +- flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; +- if (flags & MSG_DONTWAIT) +- req->flags |= REQ_F_NOWAIT; +- else if (force_nonblock) +- flags |= MSG_DONTWAIT; +- +- if (flags & MSG_WAITALL) +- min_ret = iov_iter_count(&msg.msg_iter); +- +- msg.msg_flags = flags; +- ret = sock_sendmsg(sock, &msg); +- if (force_nonblock && ret == -EAGAIN) +- return -EAGAIN; +- if (ret == -ERESTARTSYS) +- ret = -EINTR; +- +- if (ret < min_ret) +- req_set_fail_links(req); +- __io_req_complete(req, ret, 0, cs); +- return 0; +-} +- +-static int __io_recvmsg_copy_hdr(struct io_kiocb *req, +- struct io_async_msghdr *iomsg) +-{ +- struct io_sr_msg *sr = &req->sr_msg; +- struct iovec __user *uiov; +- size_t iov_len; +- int ret; +- +- ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg, +- &iomsg->uaddr, &uiov, &iov_len); +- if (ret) +- return ret; +- +- if (req->flags & REQ_F_BUFFER_SELECT) { +- if (iov_len > 1) +- return -EINVAL; +- if (copy_from_user(iomsg->iov, uiov, sizeof(*uiov))) +- return -EFAULT; +- sr->len = iomsg->iov[0].iov_len; +- iov_iter_init(&iomsg->msg.msg_iter, READ, iomsg->iov, 1, +- sr->len); +- iomsg->iov = NULL; +- } else { +- ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV, +- &iomsg->iov, &iomsg->msg.msg_iter, +- false); +- if (ret > 0) +- ret = 0; +- } +- +- return ret; +-} +- +-#ifdef CONFIG_COMPAT +-static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, +- struct io_async_msghdr *iomsg) +-{ +- struct compat_msghdr __user *msg_compat; +- struct io_sr_msg *sr = &req->sr_msg; +- struct compat_iovec __user *uiov; +- compat_uptr_t ptr; +- compat_size_t len; +- int ret; +- +- msg_compat = (struct compat_msghdr __user *) sr->umsg; +- ret = __get_compat_msghdr(&iomsg->msg, msg_compat, &iomsg->uaddr, +- &ptr, &len); +- if (ret) +- return ret; +- +- uiov = compat_ptr(ptr); +- if (req->flags & REQ_F_BUFFER_SELECT) { +- compat_ssize_t clen; +- +- if (len > 1) +- return -EINVAL; +- if (!access_ok(uiov, sizeof(*uiov))) +- return -EFAULT; +- if (__get_user(clen, &uiov->iov_len)) +- return -EFAULT; +- if (clen < 0) +- return -EINVAL; +- sr->len = clen; +- iomsg->iov[0].iov_len = clen; +- iomsg->iov = NULL; +- } else { +- ret = __import_iovec(READ, (struct iovec __user *)uiov, len, +- UIO_FASTIOV, &iomsg->iov, +- &iomsg->msg.msg_iter, true); +- if (ret < 0) +- return ret; +- } +- +- return 0; +-} +-#endif +- +-static int io_recvmsg_copy_hdr(struct io_kiocb *req, +- struct io_async_msghdr *iomsg) +-{ +- iomsg->msg.msg_name = &iomsg->addr; +- iomsg->iov = iomsg->fast_iov; +- +-#ifdef CONFIG_COMPAT +- if (req->ctx->compat) +- return __io_compat_recvmsg_copy_hdr(req, iomsg); +-#endif +- +- return __io_recvmsg_copy_hdr(req, iomsg); +-} +- +-static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req, +- bool needs_lock) +-{ +- struct io_sr_msg *sr = &req->sr_msg; +- struct io_buffer *kbuf; +- +- kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock); +- if (IS_ERR(kbuf)) +- return kbuf; +- +- sr->kbuf = kbuf; +- req->flags |= REQ_F_BUFFER_SELECTED; +- return kbuf; +-} +- +-static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req) +-{ +- return io_put_kbuf(req, req->sr_msg.kbuf); +-} +- +-static int io_recvmsg_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +- struct io_async_msghdr *async_msg = req->async_data; +- struct io_sr_msg *sr = &req->sr_msg; +- int ret; +- +- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- if (unlikely(sqe->addr2 || sqe->splice_fd_in || sqe->ioprio)) +- return -EINVAL; +- +- sr->msg_flags = READ_ONCE(sqe->msg_flags); +- sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); +- sr->len = READ_ONCE(sqe->len); +- sr->bgid = READ_ONCE(sqe->buf_group); +- +-#ifdef CONFIG_COMPAT +- if (req->ctx->compat) +- sr->msg_flags |= MSG_CMSG_COMPAT; +-#endif +- +- if (!async_msg || !io_op_defs[req->opcode].needs_async_data) +- return 0; +- ret = io_recvmsg_copy_hdr(req, async_msg); +- if (!ret) +- req->flags |= REQ_F_NEED_CLEANUP; +- return ret; +-} +- +-static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct io_async_msghdr iomsg, *kmsg; +- struct socket *sock; +- struct io_buffer *kbuf; +- unsigned flags; +- int min_ret = 0; +- int ret, cflags = 0; +- +- sock = sock_from_file(req->file, &ret); +- if (unlikely(!sock)) +- return ret; +- +- if (req->async_data) { +- kmsg = req->async_data; +- kmsg->msg.msg_name = &kmsg->addr; +- /* if iov is set, it's allocated already */ +- if (!kmsg->iov) +- kmsg->iov = kmsg->fast_iov; +- kmsg->msg.msg_iter.iov = kmsg->iov; +- } else { +- ret = io_recvmsg_copy_hdr(req, &iomsg); +- if (ret) +- return ret; +- kmsg = &iomsg; +- } +- +- if (req->flags & REQ_F_BUFFER_SELECT) { +- kbuf = io_recv_buffer_select(req, !force_nonblock); +- if (IS_ERR(kbuf)) +- return PTR_ERR(kbuf); +- kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr); +- iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov, +- 1, req->sr_msg.len); +- } +- +- flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; +- if (flags & MSG_DONTWAIT) +- req->flags |= REQ_F_NOWAIT; +- else if (force_nonblock) +- flags |= MSG_DONTWAIT; +- +- if (flags & MSG_WAITALL) +- min_ret = iov_iter_count(&kmsg->msg.msg_iter); +- +- ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg, +- kmsg->uaddr, flags); +- if (force_nonblock && ret == -EAGAIN) +- return io_setup_async_msg(req, kmsg); +- if (ret == -ERESTARTSYS) +- ret = -EINTR; +- +- if (req->flags & REQ_F_BUFFER_SELECTED) +- cflags = io_put_recv_kbuf(req); +- if (kmsg->iov != kmsg->fast_iov) +- kfree(kmsg->iov); +- req->flags &= ~REQ_F_NEED_CLEANUP; +- if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)))) +- req_set_fail_links(req); +- __io_req_complete(req, ret, cflags, cs); +- return 0; +-} +- +-static int io_recv(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct io_buffer *kbuf; +- struct io_sr_msg *sr = &req->sr_msg; +- struct msghdr msg; +- void __user *buf = sr->buf; +- struct socket *sock; +- struct iovec iov; +- unsigned flags; +- int min_ret = 0; +- int ret, cflags = 0; +- +- sock = sock_from_file(req->file, &ret); +- if (unlikely(!sock)) +- return ret; +- +- if (req->flags & REQ_F_BUFFER_SELECT) { +- kbuf = io_recv_buffer_select(req, !force_nonblock); +- if (IS_ERR(kbuf)) +- return PTR_ERR(kbuf); +- buf = u64_to_user_ptr(kbuf->addr); +- } +- +- ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter); +- if (unlikely(ret)) +- goto out_free; +- +- msg.msg_name = NULL; +- msg.msg_control = NULL; +- msg.msg_controllen = 0; +- msg.msg_namelen = 0; +- msg.msg_iocb = NULL; +- msg.msg_flags = 0; +- +- flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; +- if (flags & MSG_DONTWAIT) +- req->flags |= REQ_F_NOWAIT; +- else if (force_nonblock) +- flags |= MSG_DONTWAIT; +- +- if (flags & MSG_WAITALL) +- min_ret = iov_iter_count(&msg.msg_iter); +- +- ret = sock_recvmsg(sock, &msg, flags); +- if (force_nonblock && ret == -EAGAIN) +- return -EAGAIN; +- if (ret == -ERESTARTSYS) +- ret = -EINTR; +-out_free: +- if (req->flags & REQ_F_BUFFER_SELECTED) +- cflags = io_put_recv_kbuf(req); +- if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)))) +- req_set_fail_links(req); +- __io_req_complete(req, ret, cflags, cs); +- return 0; +-} +- +-static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- struct io_accept *accept = &req->accept; +- +- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL))) +- return -EINVAL; +- if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->splice_fd_in) +- return -EINVAL; +- +- accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); +- accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); +- accept->flags = READ_ONCE(sqe->accept_flags); +- accept->nofile = rlimit(RLIMIT_NOFILE); +- return 0; +-} +- +-static int io_accept(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct io_accept *accept = &req->accept; +- unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; +- int ret; +- +- if (req->file->f_flags & O_NONBLOCK) +- req->flags |= REQ_F_NOWAIT; +- +- ret = __sys_accept4_file(req->file, file_flags, accept->addr, +- accept->addr_len, accept->flags, +- accept->nofile); +- if (ret == -EAGAIN && force_nonblock) +- return -EAGAIN; +- if (ret < 0) { +- if (ret == -ERESTARTSYS) +- ret = -EINTR; +- req_set_fail_links(req); +- } +- __io_req_complete(req, ret, 0, cs); +- return 0; +-} +- +-static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- struct io_connect *conn = &req->connect; +- struct io_async_connect *io = req->async_data; +- +- if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL))) +- return -EINVAL; +- if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags || +- sqe->splice_fd_in) +- return -EINVAL; +- +- conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); +- conn->addr_len = READ_ONCE(sqe->addr2); +- +- if (!io) +- return 0; +- +- return move_addr_to_kernel(conn->addr, conn->addr_len, +- &io->address); +-} +- +-static int io_connect(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct io_async_connect __io, *io; +- unsigned file_flags; +- int ret; +- +- if (req->async_data) { +- io = req->async_data; +- } else { +- ret = move_addr_to_kernel(req->connect.addr, +- req->connect.addr_len, +- &__io.address); +- if (ret) +- goto out; +- io = &__io; +- } +- +- file_flags = force_nonblock ? O_NONBLOCK : 0; +- +- ret = __sys_connect_file(req->file, &io->address, +- req->connect.addr_len, file_flags); +- if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) { +- if (req->async_data) +- return -EAGAIN; +- if (io_alloc_async_data(req)) { +- ret = -ENOMEM; +- goto out; +- } +- io = req->async_data; +- memcpy(req->async_data, &__io, sizeof(__io)); +- return -EAGAIN; +- } +- if (ret == -ERESTARTSYS) +- ret = -EINTR; +-out: +- if (ret < 0) +- req_set_fail_links(req); +- __io_req_complete(req, ret, 0, cs); +- return 0; +-} +-#else /* !CONFIG_NET */ +-static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- return -EOPNOTSUPP; +-} +- +-static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- return -EOPNOTSUPP; +-} +- +-static int io_send(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- return -EOPNOTSUPP; +-} +- +-static int io_recvmsg_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +- return -EOPNOTSUPP; +-} +- +-static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- return -EOPNOTSUPP; +-} +- +-static int io_recv(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- return -EOPNOTSUPP; +-} +- +-static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- return -EOPNOTSUPP; +-} +- +-static int io_accept(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- return -EOPNOTSUPP; +-} +- +-static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- return -EOPNOTSUPP; +-} +- +-static int io_connect(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- return -EOPNOTSUPP; +-} +-#endif /* CONFIG_NET */ +- +-struct io_poll_table { +- struct poll_table_struct pt; +- struct io_kiocb *req; +- int nr_entries; +- int error; +-}; +- +-static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll, +- __poll_t mask, task_work_func_t func) +-{ +- bool twa_signal_ok; +- int ret; +- +- /* for instances that support it check for an event match first: */ +- if (mask && !(mask & poll->events)) +- return 0; +- +- trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask); +- +- list_del_init(&poll->wait.entry); +- +- req->result = mask; +- init_task_work(&req->task_work, func); +- percpu_ref_get(&req->ctx->refs); +- +- /* +- * If we using the signalfd wait_queue_head for this wakeup, then +- * it's not safe to use TWA_SIGNAL as we could be recursing on the +- * tsk->sighand->siglock on doing the wakeup. Should not be needed +- * either, as the normal wakeup will suffice. +- */ +- twa_signal_ok = (poll->head != &req->task->sighand->signalfd_wqh); +- +- /* +- * If this fails, then the task is exiting. When a task exits, the +- * work gets canceled, so just cancel this request as well instead +- * of executing it. We can't safely execute it anyway, as we may not +- * have the needed state needed for it anyway. +- */ +- ret = io_req_task_work_add(req, twa_signal_ok); +- if (unlikely(ret)) { +- struct task_struct *tsk; +- +- WRITE_ONCE(poll->canceled, true); +- tsk = io_wq_get_task(req->ctx->io_wq); +- task_work_add(tsk, &req->task_work, TWA_NONE); +- wake_up_process(tsk); +- } +- return 1; +-} +- +-static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll) +- __acquires(&req->ctx->completion_lock) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- +- if (!req->result && !READ_ONCE(poll->canceled)) { +- struct poll_table_struct pt = { ._key = poll->events }; +- +- req->result = vfs_poll(req->file, &pt) & poll->events; +- } +- +- spin_lock_irq(&ctx->completion_lock); +- if (!req->result && !READ_ONCE(poll->canceled)) { +- add_wait_queue(poll->head, &poll->wait); +- return true; +- } +- +- return false; +-} +- +-static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req) +-{ +- /* pure poll stashes this in ->async_data, poll driven retry elsewhere */ +- if (req->opcode == IORING_OP_POLL_ADD) +- return req->async_data; +- return req->apoll->double_poll; +-} +- +-static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req) +-{ +- if (req->opcode == IORING_OP_POLL_ADD) +- return &req->poll; +- return &req->apoll->poll; +-} +- +-static void io_poll_remove_double(struct io_kiocb *req) +-{ +- struct io_poll_iocb *poll = io_poll_get_double(req); +- +- lockdep_assert_held(&req->ctx->completion_lock); +- +- if (poll && poll->head) { +- struct wait_queue_head *head = poll->head; +- +- spin_lock(&head->lock); +- list_del_init(&poll->wait.entry); +- if (poll->wait.private) +- refcount_dec(&req->refs); +- poll->head = NULL; +- spin_unlock(&head->lock); +- } +-} +- +-static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- +- io_poll_remove_double(req); +- req->poll.done = true; +- io_cqring_fill_event(req, error ? error : mangle_poll(mask)); +- io_commit_cqring(ctx); +-} +- +-static void io_poll_task_func(struct callback_head *cb) +-{ +- struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); +- struct io_ring_ctx *ctx = req->ctx; +- struct io_kiocb *nxt; +- +- if (io_poll_rewait(req, &req->poll)) { +- spin_unlock_irq(&ctx->completion_lock); +- } else { +- hash_del(&req->hash_node); +- io_poll_complete(req, req->result, 0); +- spin_unlock_irq(&ctx->completion_lock); +- +- nxt = io_put_req_find_next(req); +- io_cqring_ev_posted(ctx); +- if (nxt) +- __io_req_task_submit(nxt); +- } +- +- percpu_ref_put(&ctx->refs); +-} +- +-static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode, +- int sync, void *key) +-{ +- struct io_kiocb *req = wait->private; +- struct io_poll_iocb *poll = io_poll_get_single(req); +- __poll_t mask = key_to_poll(key); +- +- /* for instances that support it check for an event match first: */ +- if (mask && !(mask & poll->events)) +- return 0; +- +- list_del_init(&wait->entry); +- +- if (poll && poll->head) { +- bool done; +- +- spin_lock(&poll->head->lock); +- done = list_empty(&poll->wait.entry); +- if (!done) +- list_del_init(&poll->wait.entry); +- /* make sure double remove sees this as being gone */ +- wait->private = NULL; +- spin_unlock(&poll->head->lock); +- if (!done) { +- /* use wait func handler, so it matches the rq type */ +- poll->wait.func(&poll->wait, mode, sync, key); +- } +- } +- refcount_dec(&req->refs); +- return 1; +-} +- +-static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events, +- wait_queue_func_t wake_func) +-{ +- poll->head = NULL; +- poll->done = false; +- poll->canceled = false; +- poll->events = events; +- INIT_LIST_HEAD(&poll->wait.entry); +- init_waitqueue_func_entry(&poll->wait, wake_func); +-} +- +-static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, +- struct wait_queue_head *head, +- struct io_poll_iocb **poll_ptr) +-{ +- struct io_kiocb *req = pt->req; +- +- /* +- * The file being polled uses multiple waitqueues for poll handling +- * (e.g. one for read, one for write). Setup a separate io_poll_iocb +- * if this happens. +- */ +- if (unlikely(pt->nr_entries)) { +- struct io_poll_iocb *poll_one = poll; +- +- /* already have a 2nd entry, fail a third attempt */ +- if (*poll_ptr) { +- pt->error = -EINVAL; +- return; +- } +- /* double add on the same waitqueue head, ignore */ +- if (poll->head == head) +- return; +- poll = kmalloc(sizeof(*poll), GFP_ATOMIC); +- if (!poll) { +- pt->error = -ENOMEM; +- return; +- } +- io_init_poll_iocb(poll, poll_one->events, io_poll_double_wake); +- refcount_inc(&req->refs); +- poll->wait.private = req; +- *poll_ptr = poll; +- } +- +- pt->nr_entries++; +- poll->head = head; +- +- if (poll->events & EPOLLEXCLUSIVE) +- add_wait_queue_exclusive(head, &poll->wait); +- else +- add_wait_queue(head, &poll->wait); +-} +- +-static void io_async_queue_proc(struct file *file, struct wait_queue_head *head, +- struct poll_table_struct *p) +-{ +- struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); +- struct async_poll *apoll = pt->req->apoll; +- +- __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll); +-} +- +-static void io_async_task_func(struct callback_head *cb) +-{ +- struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); +- struct async_poll *apoll = req->apoll; +- struct io_ring_ctx *ctx = req->ctx; +- +- trace_io_uring_task_run(req->ctx, req->opcode, req->user_data); +- +- if (io_poll_rewait(req, &apoll->poll)) { +- spin_unlock_irq(&ctx->completion_lock); +- percpu_ref_put(&ctx->refs); +- return; +- } +- +- /* If req is still hashed, it cannot have been canceled. Don't check. */ +- if (hash_hashed(&req->hash_node)) +- hash_del(&req->hash_node); +- +- io_poll_remove_double(req); +- spin_unlock_irq(&ctx->completion_lock); +- +- if (!READ_ONCE(apoll->poll.canceled)) +- __io_req_task_submit(req); +- else +- __io_req_task_cancel(req, -ECANCELED); +- +- percpu_ref_put(&ctx->refs); +- kfree(apoll->double_poll); +- kfree(apoll); +-} +- +-static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync, +- void *key) +-{ +- struct io_kiocb *req = wait->private; +- struct io_poll_iocb *poll = &req->apoll->poll; +- +- trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data, +- key_to_poll(key)); +- +- return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func); +-} +- +-static void io_poll_req_insert(struct io_kiocb *req) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- struct hlist_head *list; +- +- list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)]; +- hlist_add_head(&req->hash_node, list); +-} +- +-static __poll_t __io_arm_poll_handler(struct io_kiocb *req, +- struct io_poll_iocb *poll, +- struct io_poll_table *ipt, __poll_t mask, +- wait_queue_func_t wake_func) +- __acquires(&ctx->completion_lock) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- bool cancel = false; +- +- if (req->file->f_op->may_pollfree) { +- spin_lock_irq(&ctx->completion_lock); +- return -EOPNOTSUPP; +- } +- +- INIT_HLIST_NODE(&req->hash_node); +- io_init_poll_iocb(poll, mask, wake_func); +- poll->file = req->file; +- poll->wait.private = req; +- +- ipt->pt._key = mask; +- ipt->req = req; +- ipt->error = 0; +- ipt->nr_entries = 0; +- +- mask = vfs_poll(req->file, &ipt->pt) & poll->events; +- if (unlikely(!ipt->nr_entries) && !ipt->error) +- ipt->error = -EINVAL; +- +- spin_lock_irq(&ctx->completion_lock); +- if (ipt->error) +- io_poll_remove_double(req); +- if (likely(poll->head)) { +- spin_lock(&poll->head->lock); +- if (unlikely(list_empty(&poll->wait.entry))) { +- if (ipt->error) +- cancel = true; +- ipt->error = 0; +- mask = 0; +- } +- if (mask || ipt->error) +- list_del_init(&poll->wait.entry); +- else if (cancel) +- WRITE_ONCE(poll->canceled, true); +- else if (!poll->done) /* actually waiting for an event */ +- io_poll_req_insert(req); +- spin_unlock(&poll->head->lock); +- } +- +- return mask; +-} +- +-static bool io_arm_poll_handler(struct io_kiocb *req) +-{ +- const struct io_op_def *def = &io_op_defs[req->opcode]; +- struct io_ring_ctx *ctx = req->ctx; +- struct async_poll *apoll; +- struct io_poll_table ipt; +- __poll_t mask, ret; +- int rw; +- +- if (!req->file || !file_can_poll(req->file)) +- return false; +- if (req->flags & REQ_F_POLLED) +- return false; +- if (def->pollin) +- rw = READ; +- else if (def->pollout) +- rw = WRITE; +- else +- return false; +- /* if we can't nonblock try, then no point in arming a poll handler */ +- if (!io_file_supports_async(req->file, rw)) +- return false; +- +- apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC); +- if (unlikely(!apoll)) +- return false; +- apoll->double_poll = NULL; +- +- req->flags |= REQ_F_POLLED; +- req->apoll = apoll; +- +- mask = 0; +- if (def->pollin) +- mask |= POLLIN | POLLRDNORM; +- if (def->pollout) +- mask |= POLLOUT | POLLWRNORM; +- +- /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */ +- if ((req->opcode == IORING_OP_RECVMSG) && +- (req->sr_msg.msg_flags & MSG_ERRQUEUE)) +- mask &= ~POLLIN; +- +- mask |= POLLERR | POLLPRI; +- +- ipt.pt._qproc = io_async_queue_proc; +- +- ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, +- io_async_wake); +- if (ret || ipt.error) { +- io_poll_remove_double(req); +- spin_unlock_irq(&ctx->completion_lock); +- kfree(apoll->double_poll); +- kfree(apoll); +- return false; +- } +- spin_unlock_irq(&ctx->completion_lock); +- trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask, +- apoll->poll.events); +- return true; +-} +- +-static bool __io_poll_remove_one(struct io_kiocb *req, +- struct io_poll_iocb *poll) +-{ +- bool do_complete = false; +- +- spin_lock(&poll->head->lock); +- WRITE_ONCE(poll->canceled, true); +- if (!list_empty(&poll->wait.entry)) { +- list_del_init(&poll->wait.entry); +- do_complete = true; +- } +- spin_unlock(&poll->head->lock); +- hash_del(&req->hash_node); +- return do_complete; +-} +- +-static bool io_poll_remove_one(struct io_kiocb *req) +-{ +- bool do_complete; +- +- io_poll_remove_double(req); +- +- if (req->opcode == IORING_OP_POLL_ADD) { +- do_complete = __io_poll_remove_one(req, &req->poll); +- } else { +- struct async_poll *apoll = req->apoll; +- +- /* non-poll requests have submit ref still */ +- do_complete = __io_poll_remove_one(req, &apoll->poll); +- if (do_complete) { +- io_put_req(req); +- kfree(apoll->double_poll); +- kfree(apoll); +- } +- } +- +- if (do_complete) { +- io_cqring_fill_event(req, -ECANCELED); +- io_commit_cqring(req->ctx); +- req_set_fail_links(req); +- io_put_req_deferred(req, 1); +- } +- +- return do_complete; +-} +- +-/* +- * Returns true if we found and killed one or more poll requests +- */ +-static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, +- struct files_struct *files) +-{ +- struct hlist_node *tmp; +- struct io_kiocb *req; +- int posted = 0, i; +- +- spin_lock_irq(&ctx->completion_lock); +- for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { +- struct hlist_head *list; +- +- list = &ctx->cancel_hash[i]; +- hlist_for_each_entry_safe(req, tmp, list, hash_node) { +- if (io_match_task(req, tsk, files)) +- posted += io_poll_remove_one(req); +- } +- } +- spin_unlock_irq(&ctx->completion_lock); +- +- if (posted) +- io_cqring_ev_posted(ctx); +- +- return posted != 0; +-} +- +-static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr) +-{ +- struct hlist_head *list; +- struct io_kiocb *req; +- +- list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)]; +- hlist_for_each_entry(req, list, hash_node) { +- if (sqe_addr != req->user_data) +- continue; +- if (io_poll_remove_one(req)) +- return 0; +- return -EALREADY; +- } +- +- return -ENOENT; +-} +- +-static int io_poll_remove_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index || +- sqe->poll_events) +- return -EINVAL; +- +- req->poll.addr = READ_ONCE(sqe->addr); +- return 0; +-} +- +-/* +- * Find a running poll command that matches one specified in sqe->addr, +- * and remove it if found. +- */ +-static int io_poll_remove(struct io_kiocb *req) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- u64 addr; +- int ret; +- +- addr = req->poll.addr; +- spin_lock_irq(&ctx->completion_lock); +- ret = io_poll_cancel(ctx, addr); +- spin_unlock_irq(&ctx->completion_lock); +- +- if (ret < 0) +- req_set_fail_links(req); +- io_req_complete(req, ret); +- return 0; +-} +- +-static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, +- void *key) +-{ +- struct io_kiocb *req = wait->private; +- struct io_poll_iocb *poll = &req->poll; +- +- return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func); +-} +- +-static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head, +- struct poll_table_struct *p) +-{ +- struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); +- +- __io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->async_data); +-} +- +-static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- struct io_poll_iocb *poll = &req->poll; +- u32 events; +- +- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index) +- return -EINVAL; +- +- events = READ_ONCE(sqe->poll32_events); +-#ifdef __BIG_ENDIAN +- events = swahw32(events); +-#endif +- poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP | +- (events & EPOLLEXCLUSIVE); +- return 0; +-} +- +-static int io_poll_add(struct io_kiocb *req) +-{ +- struct io_poll_iocb *poll = &req->poll; +- struct io_ring_ctx *ctx = req->ctx; +- struct io_poll_table ipt; +- __poll_t mask; +- +- ipt.pt._qproc = io_poll_queue_proc; +- +- mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events, +- io_poll_wake); +- +- if (mask) { /* no async, we'd stolen it */ +- ipt.error = 0; +- io_poll_complete(req, mask, 0); +- } +- spin_unlock_irq(&ctx->completion_lock); +- +- if (mask) { +- io_cqring_ev_posted(ctx); +- io_put_req(req); +- } +- return ipt.error; +-} +- +-static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) +-{ +- struct io_timeout_data *data = container_of(timer, +- struct io_timeout_data, timer); +- struct io_kiocb *req = data->req; +- struct io_ring_ctx *ctx = req->ctx; +- unsigned long flags; +- +- spin_lock_irqsave(&ctx->completion_lock, flags); +- list_del_init(&req->timeout.list); +- atomic_set(&req->ctx->cq_timeouts, +- atomic_read(&req->ctx->cq_timeouts) + 1); +- +- io_cqring_fill_event(req, -ETIME); +- io_commit_cqring(ctx); +- spin_unlock_irqrestore(&ctx->completion_lock, flags); +- +- io_cqring_ev_posted(ctx); +- req_set_fail_links(req); +- io_put_req(req); +- return HRTIMER_NORESTART; +-} +- +-static int __io_timeout_cancel(struct io_kiocb *req) +-{ +- struct io_timeout_data *io = req->async_data; +- int ret; +- +- ret = hrtimer_try_to_cancel(&io->timer); +- if (ret == -1) +- return -EALREADY; +- list_del_init(&req->timeout.list); +- +- req_set_fail_links(req); +- io_cqring_fill_event(req, -ECANCELED); +- io_put_req_deferred(req, 1); +- return 0; +-} +- +-static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data) +-{ +- struct io_kiocb *req; +- int ret = -ENOENT; +- +- list_for_each_entry(req, &ctx->timeout_list, timeout.list) { +- if (user_data == req->user_data) { +- ret = 0; +- break; +- } +- } +- +- if (ret == -ENOENT) +- return ret; +- +- return __io_timeout_cancel(req); +-} +- +-static int io_timeout_remove_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) +- return -EINVAL; +- if (sqe->ioprio || sqe->buf_index || sqe->len || sqe->timeout_flags || +- sqe->splice_fd_in) +- return -EINVAL; +- +- req->timeout_rem.addr = READ_ONCE(sqe->addr); +- return 0; +-} +- +-/* +- * Remove or update an existing timeout command +- */ +-static int io_timeout_remove(struct io_kiocb *req) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- int ret; +- +- spin_lock_irq(&ctx->completion_lock); +- ret = io_timeout_cancel(ctx, req->timeout_rem.addr); +- +- io_cqring_fill_event(req, ret); +- io_commit_cqring(ctx); +- spin_unlock_irq(&ctx->completion_lock); +- io_cqring_ev_posted(ctx); +- if (ret < 0) +- req_set_fail_links(req); +- io_put_req(req); +- return 0; +-} +- +-static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, +- bool is_timeout_link) +-{ +- struct io_timeout_data *data; +- unsigned flags; +- u32 off = READ_ONCE(sqe->off); +- +- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- if (sqe->ioprio || sqe->buf_index || sqe->len != 1 || +- sqe->splice_fd_in) +- return -EINVAL; +- if (off && is_timeout_link) +- return -EINVAL; +- flags = READ_ONCE(sqe->timeout_flags); +- if (flags & ~IORING_TIMEOUT_ABS) +- return -EINVAL; +- +- req->timeout.off = off; +- +- if (!req->async_data && io_alloc_async_data(req)) +- return -ENOMEM; +- +- data = req->async_data; +- data->req = req; +- +- if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr))) +- return -EFAULT; +- +- if (flags & IORING_TIMEOUT_ABS) +- data->mode = HRTIMER_MODE_ABS; +- else +- data->mode = HRTIMER_MODE_REL; +- +- INIT_LIST_HEAD(&req->timeout.list); +- hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode); +- return 0; +-} +- +-static int io_timeout(struct io_kiocb *req) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- struct io_timeout_data *data = req->async_data; +- struct list_head *entry; +- u32 tail, off = req->timeout.off; +- +- spin_lock_irq(&ctx->completion_lock); +- +- /* +- * sqe->off holds how many events that need to occur for this +- * timeout event to be satisfied. If it isn't set, then this is +- * a pure timeout request, sequence isn't used. +- */ +- if (io_is_timeout_noseq(req)) { +- entry = ctx->timeout_list.prev; +- goto add; +- } +- +- tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); +- req->timeout.target_seq = tail + off; +- +- /* Update the last seq here in case io_flush_timeouts() hasn't. +- * This is safe because ->completion_lock is held, and submissions +- * and completions are never mixed in the same ->completion_lock section. +- */ +- ctx->cq_last_tm_flush = tail; +- +- /* +- * Insertion sort, ensuring the first entry in the list is always +- * the one we need first. +- */ +- list_for_each_prev(entry, &ctx->timeout_list) { +- struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, +- timeout.list); +- +- if (io_is_timeout_noseq(nxt)) +- continue; +- /* nxt.seq is behind @tail, otherwise would've been completed */ +- if (off >= nxt->timeout.target_seq - tail) +- break; +- } +-add: +- list_add(&req->timeout.list, entry); +- data->timer.function = io_timeout_fn; +- hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); +- spin_unlock_irq(&ctx->completion_lock); +- return 0; +-} +- +-static bool io_cancel_cb(struct io_wq_work *work, void *data) +-{ +- struct io_kiocb *req = container_of(work, struct io_kiocb, work); +- +- return req->user_data == (unsigned long) data; +-} +- +-static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr) +-{ +- enum io_wq_cancel cancel_ret; +- int ret = 0; +- +- cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr, false); +- switch (cancel_ret) { +- case IO_WQ_CANCEL_OK: +- ret = 0; +- break; +- case IO_WQ_CANCEL_RUNNING: +- ret = -EALREADY; +- break; +- case IO_WQ_CANCEL_NOTFOUND: +- ret = -ENOENT; +- break; +- } +- +- return ret; +-} +- +-static void io_async_find_and_cancel(struct io_ring_ctx *ctx, +- struct io_kiocb *req, __u64 sqe_addr, +- int success_ret) +-{ +- unsigned long flags; +- int ret; +- +- ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr); +- if (ret != -ENOENT) { +- spin_lock_irqsave(&ctx->completion_lock, flags); +- goto done; +- } +- +- spin_lock_irqsave(&ctx->completion_lock, flags); +- ret = io_timeout_cancel(ctx, sqe_addr); +- if (ret != -ENOENT) +- goto done; +- ret = io_poll_cancel(ctx, sqe_addr); +-done: +- if (!ret) +- ret = success_ret; +- io_cqring_fill_event(req, ret); +- io_commit_cqring(ctx); +- spin_unlock_irqrestore(&ctx->completion_lock, flags); +- io_cqring_ev_posted(ctx); +- +- if (ret < 0) +- req_set_fail_links(req); +- io_put_req(req); +-} +- +-static int io_async_cancel_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +- if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) +- return -EINVAL; +- if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) +- return -EINVAL; +- if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags || +- sqe->splice_fd_in) +- return -EINVAL; +- +- req->cancel.addr = READ_ONCE(sqe->addr); +- return 0; +-} +- +-static int io_async_cancel(struct io_kiocb *req) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- +- io_async_find_and_cancel(ctx, req, req->cancel.addr, 0); +- return 0; +-} +- +-static int io_files_update_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +- if (unlikely(req->ctx->flags & IORING_SETUP_SQPOLL)) +- return -EINVAL; +- if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) +- return -EINVAL; +- if (sqe->ioprio || sqe->rw_flags) +- return -EINVAL; +- +- req->files_update.offset = READ_ONCE(sqe->off); +- req->files_update.nr_args = READ_ONCE(sqe->len); +- if (!req->files_update.nr_args) +- return -EINVAL; +- req->files_update.arg = READ_ONCE(sqe->addr); +- return 0; +-} +- +-static int io_files_update(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- struct io_uring_files_update up; +- int ret; +- +- if (force_nonblock) +- return -EAGAIN; +- +- up.offset = req->files_update.offset; +- up.fds = req->files_update.arg; +- +- mutex_lock(&ctx->uring_lock); +- ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args); +- mutex_unlock(&ctx->uring_lock); +- +- if (ret < 0) +- req_set_fail_links(req); +- __io_req_complete(req, ret, 0, cs); +- return 0; +-} +- +-static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- switch (req->opcode) { +- case IORING_OP_NOP: +- return 0; +- case IORING_OP_READV: +- case IORING_OP_READ_FIXED: +- case IORING_OP_READ: +- return io_read_prep(req, sqe); +- case IORING_OP_WRITEV: +- case IORING_OP_WRITE_FIXED: +- case IORING_OP_WRITE: +- return io_write_prep(req, sqe); +- case IORING_OP_POLL_ADD: +- return io_poll_add_prep(req, sqe); +- case IORING_OP_POLL_REMOVE: +- return io_poll_remove_prep(req, sqe); +- case IORING_OP_FSYNC: +- return io_prep_fsync(req, sqe); +- case IORING_OP_SYNC_FILE_RANGE: +- return io_prep_sfr(req, sqe); +- case IORING_OP_SENDMSG: +- case IORING_OP_SEND: +- return io_sendmsg_prep(req, sqe); +- case IORING_OP_RECVMSG: +- case IORING_OP_RECV: +- return io_recvmsg_prep(req, sqe); +- case IORING_OP_CONNECT: +- return io_connect_prep(req, sqe); +- case IORING_OP_TIMEOUT: +- return io_timeout_prep(req, sqe, false); +- case IORING_OP_TIMEOUT_REMOVE: +- return io_timeout_remove_prep(req, sqe); +- case IORING_OP_ASYNC_CANCEL: +- return io_async_cancel_prep(req, sqe); +- case IORING_OP_LINK_TIMEOUT: +- return io_timeout_prep(req, sqe, true); +- case IORING_OP_ACCEPT: +- return io_accept_prep(req, sqe); +- case IORING_OP_FALLOCATE: +- return io_fallocate_prep(req, sqe); +- case IORING_OP_OPENAT: +- return io_openat_prep(req, sqe); +- case IORING_OP_CLOSE: +- return io_close_prep(req, sqe); +- case IORING_OP_FILES_UPDATE: +- return io_files_update_prep(req, sqe); +- case IORING_OP_STATX: +- return io_statx_prep(req, sqe); +- case IORING_OP_FADVISE: +- return io_fadvise_prep(req, sqe); +- case IORING_OP_MADVISE: +- return io_madvise_prep(req, sqe); +- case IORING_OP_OPENAT2: +- return io_openat2_prep(req, sqe); +- case IORING_OP_EPOLL_CTL: +- return io_epoll_ctl_prep(req, sqe); +- case IORING_OP_SPLICE: +- return io_splice_prep(req, sqe); +- case IORING_OP_PROVIDE_BUFFERS: +- return io_provide_buffers_prep(req, sqe); +- case IORING_OP_REMOVE_BUFFERS: +- return io_remove_buffers_prep(req, sqe); +- case IORING_OP_TEE: +- return io_tee_prep(req, sqe); +- } +- +- printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n", +- req->opcode); +- return-EINVAL; +-} +- +-static int io_req_defer_prep(struct io_kiocb *req, +- const struct io_uring_sqe *sqe) +-{ +- if (!sqe) +- return 0; +- if (io_alloc_async_data(req)) +- return -EAGAIN; +- return io_req_prep(req, sqe); +-} +- +-static u32 io_get_sequence(struct io_kiocb *req) +-{ +- struct io_kiocb *pos; +- struct io_ring_ctx *ctx = req->ctx; +- u32 total_submitted, nr_reqs = 1; +- +- if (req->flags & REQ_F_LINK_HEAD) +- list_for_each_entry(pos, &req->link_list, link_list) +- nr_reqs++; +- +- total_submitted = ctx->cached_sq_head - ctx->cached_sq_dropped; +- return total_submitted - nr_reqs; +-} +- +-static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- struct io_defer_entry *de; +- int ret; +- u32 seq; +- +- /* Still need defer if there is pending req in defer list. */ +- if (likely(list_empty_careful(&ctx->defer_list) && +- !(req->flags & REQ_F_IO_DRAIN))) +- return 0; +- +- seq = io_get_sequence(req); +- /* Still a chance to pass the sequence check */ +- if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) +- return 0; +- +- if (!req->async_data) { +- ret = io_req_defer_prep(req, sqe); +- if (ret) +- return ret; +- } +- io_prep_async_link(req); +- de = kmalloc(sizeof(*de), GFP_KERNEL); +- if (!de) +- return -ENOMEM; +- +- spin_lock_irq(&ctx->completion_lock); +- if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) { +- spin_unlock_irq(&ctx->completion_lock); +- kfree(de); +- io_queue_async_work(req); +- return -EIOCBQUEUED; +- } +- +- trace_io_uring_defer(ctx, req, req->user_data); +- de->req = req; +- de->seq = seq; +- list_add_tail(&de->list, &ctx->defer_list); +- spin_unlock_irq(&ctx->completion_lock); +- return -EIOCBQUEUED; +-} +- +-static void io_req_drop_files(struct io_kiocb *req) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- struct io_uring_task *tctx = req->task->io_uring; +- unsigned long flags; +- +- if (req->work.flags & IO_WQ_WORK_FILES) { +- put_files_struct(req->work.identity->files); +- put_nsproxy(req->work.identity->nsproxy); +- } +- spin_lock_irqsave(&ctx->inflight_lock, flags); +- list_del(&req->inflight_entry); +- spin_unlock_irqrestore(&ctx->inflight_lock, flags); +- req->flags &= ~REQ_F_INFLIGHT; +- req->work.flags &= ~IO_WQ_WORK_FILES; +- if (atomic_read(&tctx->in_idle)) +- wake_up(&tctx->wait); +-} +- +-static void __io_clean_op(struct io_kiocb *req) +-{ +- if (req->flags & REQ_F_BUFFER_SELECTED) { +- switch (req->opcode) { +- case IORING_OP_READV: +- case IORING_OP_READ_FIXED: +- case IORING_OP_READ: +- kfree((void *)(unsigned long)req->rw.addr); +- break; +- case IORING_OP_RECVMSG: +- case IORING_OP_RECV: +- kfree(req->sr_msg.kbuf); +- break; +- } +- req->flags &= ~REQ_F_BUFFER_SELECTED; +- } +- +- if (req->flags & REQ_F_NEED_CLEANUP) { +- switch (req->opcode) { +- case IORING_OP_READV: +- case IORING_OP_READ_FIXED: +- case IORING_OP_READ: +- case IORING_OP_WRITEV: +- case IORING_OP_WRITE_FIXED: +- case IORING_OP_WRITE: { +- struct io_async_rw *io = req->async_data; +- if (io->free_iovec) +- kfree(io->free_iovec); +- break; +- } +- case IORING_OP_RECVMSG: +- case IORING_OP_SENDMSG: { +- struct io_async_msghdr *io = req->async_data; +- if (io->iov != io->fast_iov) +- kfree(io->iov); +- break; +- } +- case IORING_OP_SPLICE: +- case IORING_OP_TEE: +- io_put_file(req, req->splice.file_in, +- (req->splice.flags & SPLICE_F_FD_IN_FIXED)); +- break; +- case IORING_OP_OPENAT: +- case IORING_OP_OPENAT2: +- if (req->open.filename) +- putname(req->open.filename); +- break; +- } +- req->flags &= ~REQ_F_NEED_CLEANUP; +- } +-} +- +-static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, +- struct io_comp_state *cs) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- int ret; +- +- switch (req->opcode) { +- case IORING_OP_NOP: +- ret = io_nop(req, cs); +- break; +- case IORING_OP_READV: +- case IORING_OP_READ_FIXED: +- case IORING_OP_READ: +- ret = io_read(req, force_nonblock, cs); +- break; +- case IORING_OP_WRITEV: +- case IORING_OP_WRITE_FIXED: +- case IORING_OP_WRITE: +- ret = io_write(req, force_nonblock, cs); +- break; +- case IORING_OP_FSYNC: +- ret = io_fsync(req, force_nonblock); +- break; +- case IORING_OP_POLL_ADD: +- ret = io_poll_add(req); +- break; +- case IORING_OP_POLL_REMOVE: +- ret = io_poll_remove(req); +- break; +- case IORING_OP_SYNC_FILE_RANGE: +- ret = io_sync_file_range(req, force_nonblock); +- break; +- case IORING_OP_SENDMSG: +- ret = io_sendmsg(req, force_nonblock, cs); +- break; +- case IORING_OP_SEND: +- ret = io_send(req, force_nonblock, cs); +- break; +- case IORING_OP_RECVMSG: +- ret = io_recvmsg(req, force_nonblock, cs); +- break; +- case IORING_OP_RECV: +- ret = io_recv(req, force_nonblock, cs); +- break; +- case IORING_OP_TIMEOUT: +- ret = io_timeout(req); +- break; +- case IORING_OP_TIMEOUT_REMOVE: +- ret = io_timeout_remove(req); +- break; +- case IORING_OP_ACCEPT: +- ret = io_accept(req, force_nonblock, cs); +- break; +- case IORING_OP_CONNECT: +- ret = io_connect(req, force_nonblock, cs); +- break; +- case IORING_OP_ASYNC_CANCEL: +- ret = io_async_cancel(req); +- break; +- case IORING_OP_FALLOCATE: +- ret = io_fallocate(req, force_nonblock); +- break; +- case IORING_OP_OPENAT: +- ret = io_openat(req, force_nonblock); +- break; +- case IORING_OP_CLOSE: +- ret = io_close(req, force_nonblock, cs); +- break; +- case IORING_OP_FILES_UPDATE: +- ret = io_files_update(req, force_nonblock, cs); +- break; +- case IORING_OP_STATX: +- ret = io_statx(req, force_nonblock); +- break; +- case IORING_OP_FADVISE: +- ret = io_fadvise(req, force_nonblock); +- break; +- case IORING_OP_MADVISE: +- ret = io_madvise(req, force_nonblock); +- break; +- case IORING_OP_OPENAT2: +- ret = io_openat2(req, force_nonblock); +- break; +- case IORING_OP_EPOLL_CTL: +- ret = io_epoll_ctl(req, force_nonblock, cs); +- break; +- case IORING_OP_SPLICE: +- ret = io_splice(req, force_nonblock); +- break; +- case IORING_OP_PROVIDE_BUFFERS: +- ret = io_provide_buffers(req, force_nonblock, cs); +- break; +- case IORING_OP_REMOVE_BUFFERS: +- ret = io_remove_buffers(req, force_nonblock, cs); +- break; +- case IORING_OP_TEE: +- ret = io_tee(req, force_nonblock); +- break; +- default: +- ret = -EINVAL; +- break; +- } +- +- if (ret) +- return ret; +- +- /* If the op doesn't have a file, we're not polling for it */ +- if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) { +- const bool in_async = io_wq_current_is_worker(); +- +- /* workqueue context doesn't hold uring_lock, grab it now */ +- if (in_async) +- mutex_lock(&ctx->uring_lock); +- +- io_iopoll_req_issued(req); +- +- if (in_async) +- mutex_unlock(&ctx->uring_lock); +- } +- +- return 0; +-} +- +-static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work) +-{ +- struct io_kiocb *req = container_of(work, struct io_kiocb, work); +- struct io_kiocb *timeout; +- int ret = 0; +- +- timeout = io_prep_linked_timeout(req); +- if (timeout) +- io_queue_linked_timeout(timeout); +- +- /* if NO_CANCEL is set, we must still run the work */ +- if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) == +- IO_WQ_WORK_CANCEL) { +- ret = -ECANCELED; +- } +- +- if (!ret) { +- do { +- ret = io_issue_sqe(req, false, NULL); +- /* +- * We can get EAGAIN for polled IO even though we're +- * forcing a sync submission from here, since we can't +- * wait for request slots on the block side. +- */ +- if (ret != -EAGAIN) +- break; +- cond_resched(); +- } while (1); +- } +- +- if (ret) { +- struct io_ring_ctx *lock_ctx = NULL; +- +- if (req->ctx->flags & IORING_SETUP_IOPOLL) +- lock_ctx = req->ctx; +- +- /* +- * io_iopoll_complete() does not hold completion_lock to +- * complete polled io, so here for polled io, we can not call +- * io_req_complete() directly, otherwise there maybe concurrent +- * access to cqring, defer_list, etc, which is not safe. Given +- * that io_iopoll_complete() is always called under uring_lock, +- * so here for polled io, we also get uring_lock to complete +- * it. +- */ +- if (lock_ctx) +- mutex_lock(&lock_ctx->uring_lock); +- +- req_set_fail_links(req); +- io_req_complete(req, ret); +- +- if (lock_ctx) +- mutex_unlock(&lock_ctx->uring_lock); +- } +- +- return io_steal_work(req); +-} +- +-static inline struct file *io_file_from_index(struct io_ring_ctx *ctx, +- int index) +-{ +- struct fixed_file_table *table; +- +- table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT]; +- return table->files[index & IORING_FILE_TABLE_MASK]; +-} +- +-static struct file *io_file_get(struct io_submit_state *state, +- struct io_kiocb *req, int fd, bool fixed) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- struct file *file; +- +- if (fixed) { +- if (unlikely((unsigned int)fd >= ctx->nr_user_files)) +- return NULL; +- fd = array_index_nospec(fd, ctx->nr_user_files); +- file = io_file_from_index(ctx, fd); +- if (file) { +- req->fixed_file_refs = &ctx->file_data->node->refs; +- percpu_ref_get(req->fixed_file_refs); +- } +- } else { +- trace_io_uring_file_get(ctx, fd); +- file = __io_file_get(state, fd); +- } +- +- if (file && file->f_op == &io_uring_fops && +- !(req->flags & REQ_F_INFLIGHT)) { +- io_req_init_async(req); +- req->flags |= REQ_F_INFLIGHT; +- +- spin_lock_irq(&ctx->inflight_lock); +- list_add(&req->inflight_entry, &ctx->inflight_list); +- spin_unlock_irq(&ctx->inflight_lock); +- } +- +- return file; +-} +- +-static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req, +- int fd) +-{ +- bool fixed; +- +- fixed = (req->flags & REQ_F_FIXED_FILE) != 0; +- if (unlikely(!fixed && io_async_submit(req->ctx))) +- return -EBADF; +- +- req->file = io_file_get(state, req, fd, fixed); +- if (req->file || io_op_defs[req->opcode].needs_file_no_error) +- return 0; +- return -EBADF; +-} +- +-static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) +-{ +- struct io_timeout_data *data = container_of(timer, +- struct io_timeout_data, timer); +- struct io_kiocb *req = data->req; +- struct io_ring_ctx *ctx = req->ctx; +- struct io_kiocb *prev = NULL; +- unsigned long flags; +- +- spin_lock_irqsave(&ctx->completion_lock, flags); +- +- /* +- * We don't expect the list to be empty, that will only happen if we +- * race with the completion of the linked work. +- */ +- if (!list_empty(&req->link_list)) { +- prev = list_entry(req->link_list.prev, struct io_kiocb, +- link_list); +- list_del_init(&req->link_list); +- if (!refcount_inc_not_zero(&prev->refs)) +- prev = NULL; +- } +- +- list_del(&req->timeout.list); +- spin_unlock_irqrestore(&ctx->completion_lock, flags); +- +- if (prev) { +- io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME); +- io_put_req_deferred(prev, 1); +- } else { +- io_cqring_add_event(req, -ETIME, 0); +- io_put_req_deferred(req, 1); +- } +- return HRTIMER_NORESTART; +-} +- +-static void __io_queue_linked_timeout(struct io_kiocb *req) +-{ +- /* +- * If the list is now empty, then our linked request finished before +- * we got a chance to setup the timer +- */ +- if (!list_empty(&req->link_list)) { +- struct io_timeout_data *data = req->async_data; +- +- data->timer.function = io_link_timeout_fn; +- hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), +- data->mode); +- } +-} +- +-static void io_queue_linked_timeout(struct io_kiocb *req) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- +- spin_lock_irq(&ctx->completion_lock); +- __io_queue_linked_timeout(req); +- spin_unlock_irq(&ctx->completion_lock); +- +- /* drop submission reference */ +- io_put_req(req); +-} +- +-static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) +-{ +- struct io_kiocb *nxt; +- +- if (!(req->flags & REQ_F_LINK_HEAD)) +- return NULL; +- if (req->flags & REQ_F_LINK_TIMEOUT) +- return NULL; +- +- nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, +- link_list); +- if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT) +- return NULL; +- +- nxt->flags |= REQ_F_LTIMEOUT_ACTIVE; +- req->flags |= REQ_F_LINK_TIMEOUT; +- return nxt; +-} +- +-static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) +-{ +- struct io_kiocb *linked_timeout; +- const struct cred *old_creds = NULL; +- int ret; +- +-again: +- linked_timeout = io_prep_linked_timeout(req); +- +- if ((req->flags & REQ_F_WORK_INITIALIZED) && +- (req->work.flags & IO_WQ_WORK_CREDS) && +- req->work.identity->creds != current_cred()) { +- if (old_creds) +- revert_creds(old_creds); +- if (old_creds == req->work.identity->creds) +- old_creds = NULL; /* restored original creds */ +- else +- old_creds = override_creds(req->work.identity->creds); +- } +- +- ret = io_issue_sqe(req, true, cs); +- +- /* +- * We async punt it if the file wasn't marked NOWAIT, or if the file +- * doesn't support non-blocking read/write attempts +- */ +- if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) { +- if (!io_arm_poll_handler(req)) { +- /* +- * Queued up for async execution, worker will release +- * submit reference when the iocb is actually submitted. +- */ +- io_queue_async_work(req); +- } +- +- if (linked_timeout) +- io_queue_linked_timeout(linked_timeout); +- } else if (likely(!ret)) { +- /* drop submission reference */ +- req = io_put_req_find_next(req); +- if (linked_timeout) +- io_queue_linked_timeout(linked_timeout); +- +- if (req) { +- if (!(req->flags & REQ_F_FORCE_ASYNC)) +- goto again; +- io_queue_async_work(req); +- } +- } else { +- /* un-prep timeout, so it'll be killed as any other linked */ +- req->flags &= ~REQ_F_LINK_TIMEOUT; +- req_set_fail_links(req); +- io_put_req(req); +- io_req_complete(req, ret); +- } +- +- if (old_creds) +- revert_creds(old_creds); +-} +- +-static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, +- struct io_comp_state *cs) +-{ +- int ret; +- +- ret = io_req_defer(req, sqe); +- if (ret) { +- if (ret != -EIOCBQUEUED) { +-fail_req: +- req_set_fail_links(req); +- io_put_req(req); +- io_req_complete(req, ret); +- } +- } else if (req->flags & REQ_F_FORCE_ASYNC) { +- if (!req->async_data) { +- ret = io_req_defer_prep(req, sqe); +- if (unlikely(ret)) +- goto fail_req; +- } +- io_queue_async_work(req); +- } else { +- if (sqe) { +- ret = io_req_prep(req, sqe); +- if (unlikely(ret)) +- goto fail_req; +- } +- __io_queue_sqe(req, cs); +- } +-} +- +-static inline void io_queue_link_head(struct io_kiocb *req, +- struct io_comp_state *cs) +-{ +- if (unlikely(req->flags & REQ_F_FAIL_LINK)) { +- io_put_req(req); +- io_req_complete(req, -ECANCELED); +- } else +- io_queue_sqe(req, NULL, cs); +-} +- +-static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, +- struct io_kiocb **link, struct io_comp_state *cs) +-{ +- struct io_ring_ctx *ctx = req->ctx; +- int ret; +- +- /* +- * If we already have a head request, queue this one for async +- * submittal once the head completes. If we don't have a head but +- * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be +- * submitted sync once the chain is complete. If none of those +- * conditions are true (normal request), then just queue it. +- */ +- if (*link) { +- struct io_kiocb *head = *link; +- +- /* +- * Taking sequential execution of a link, draining both sides +- * of the link also fullfils IOSQE_IO_DRAIN semantics for all +- * requests in the link. So, it drains the head and the +- * next after the link request. The last one is done via +- * drain_next flag to persist the effect across calls. +- */ +- if (req->flags & REQ_F_IO_DRAIN) { +- head->flags |= REQ_F_IO_DRAIN; +- ctx->drain_next = 1; +- } +- ret = io_req_defer_prep(req, sqe); +- if (unlikely(ret)) { +- /* fail even hard links since we don't submit */ +- head->flags |= REQ_F_FAIL_LINK; +- return ret; +- } +- trace_io_uring_link(ctx, req, head); +- list_add_tail(&req->link_list, &head->link_list); +- +- /* last request of a link, enqueue the link */ +- if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) { +- io_queue_link_head(head, cs); +- *link = NULL; +- } +- } else { +- if (unlikely(ctx->drain_next)) { +- req->flags |= REQ_F_IO_DRAIN; +- ctx->drain_next = 0; +- } +- if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { +- req->flags |= REQ_F_LINK_HEAD; +- INIT_LIST_HEAD(&req->link_list); +- +- ret = io_req_defer_prep(req, sqe); +- if (unlikely(ret)) +- req->flags |= REQ_F_FAIL_LINK; +- *link = req; +- } else { +- io_queue_sqe(req, sqe, cs); +- } +- } +- +- return 0; +-} +- +-/* +- * Batched submission is done, ensure local IO is flushed out. +- */ +-static void io_submit_state_end(struct io_submit_state *state) +-{ +- if (!list_empty(&state->comp.list)) +- io_submit_flush_completions(&state->comp); +- blk_finish_plug(&state->plug); +- io_state_file_put(state); +- if (state->free_reqs) +- kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs); +-} +- +-/* +- * Start submission side cache. +- */ +-static void io_submit_state_start(struct io_submit_state *state, +- struct io_ring_ctx *ctx, unsigned int max_ios) +-{ +- blk_start_plug(&state->plug); +- state->comp.nr = 0; +- INIT_LIST_HEAD(&state->comp.list); +- state->comp.ctx = ctx; +- state->free_reqs = 0; +- state->file = NULL; +- state->ios_left = max_ios; +-} +- +-static void io_commit_sqring(struct io_ring_ctx *ctx) +-{ +- struct io_rings *rings = ctx->rings; +- +- /* +- * Ensure any loads from the SQEs are done at this point, +- * since once we write the new head, the application could +- * write new data to them. +- */ +- smp_store_release(&rings->sq.head, ctx->cached_sq_head); +-} +- +-/* +- * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory +- * that is mapped by userspace. This means that care needs to be taken to +- * ensure that reads are stable, as we cannot rely on userspace always +- * being a good citizen. If members of the sqe are validated and then later +- * used, it's important that those reads are done through READ_ONCE() to +- * prevent a re-load down the line. +- */ +-static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) +-{ +- u32 *sq_array = ctx->sq_array; +- unsigned head; +- +- /* +- * The cached sq head (or cq tail) serves two purposes: +- * +- * 1) allows us to batch the cost of updating the user visible +- * head updates. +- * 2) allows the kernel side to track the head on its own, even +- * though the application is the one updating it. +- */ +- head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]); +- if (likely(head < ctx->sq_entries)) +- return &ctx->sq_sqes[head]; +- +- /* drop invalid entries */ +- ctx->cached_sq_dropped++; +- WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped); +- return NULL; +-} +- +-static inline void io_consume_sqe(struct io_ring_ctx *ctx) +-{ +- ctx->cached_sq_head++; +-} +- +-/* +- * Check SQE restrictions (opcode and flags). +- * +- * Returns 'true' if SQE is allowed, 'false' otherwise. +- */ +-static inline bool io_check_restriction(struct io_ring_ctx *ctx, +- struct io_kiocb *req, +- unsigned int sqe_flags) +-{ +- if (!ctx->restricted) +- return true; +- +- if (!test_bit(req->opcode, ctx->restrictions.sqe_op)) +- return false; +- +- if ((sqe_flags & ctx->restrictions.sqe_flags_required) != +- ctx->restrictions.sqe_flags_required) +- return false; +- +- if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed | +- ctx->restrictions.sqe_flags_required)) +- return false; +- +- return true; +-} +- +-#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ +- IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ +- IOSQE_BUFFER_SELECT) +- +-static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, +- const struct io_uring_sqe *sqe, +- struct io_submit_state *state) +-{ +- unsigned int sqe_flags; +- int id, ret; +- +- req->opcode = READ_ONCE(sqe->opcode); +- req->user_data = READ_ONCE(sqe->user_data); +- req->async_data = NULL; +- req->file = NULL; +- req->ctx = ctx; +- req->flags = 0; +- /* one is dropped after submission, the other at completion */ +- refcount_set(&req->refs, 2); +- req->task = current; +- req->result = 0; +- +- if (unlikely(req->opcode >= IORING_OP_LAST)) +- return -EINVAL; +- +- if (unlikely(io_sq_thread_acquire_mm(ctx, req))) +- return -EFAULT; +- +- sqe_flags = READ_ONCE(sqe->flags); +- /* enforce forwards compatibility on users */ +- if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) +- return -EINVAL; +- +- if (unlikely(!io_check_restriction(ctx, req, sqe_flags))) +- return -EACCES; +- +- if ((sqe_flags & IOSQE_BUFFER_SELECT) && +- !io_op_defs[req->opcode].buffer_select) +- return -EOPNOTSUPP; +- +- id = READ_ONCE(sqe->personality); +- if (id) { +- struct io_identity *iod; +- +- iod = xa_load(&ctx->personalities, id); +- if (unlikely(!iod)) +- return -EINVAL; +- refcount_inc(&iod->count); +- +- __io_req_init_async(req); +- get_cred(iod->creds); +- req->work.identity = iod; +- req->work.flags |= IO_WQ_WORK_CREDS; +- } +- +- /* same numerical values with corresponding REQ_F_*, safe to copy */ +- req->flags |= sqe_flags; +- +- if (!io_op_defs[req->opcode].needs_file) +- return 0; +- +- ret = io_req_set_file(state, req, READ_ONCE(sqe->fd)); +- state->ios_left--; +- return ret; +-} +- +-static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) +-{ +- struct io_submit_state state; +- struct io_kiocb *link = NULL; +- int i, submitted = 0; +- +- /* if we have a backlog and couldn't flush it all, return BUSY */ +- if (test_bit(0, &ctx->sq_check_overflow)) { +- if (!__io_cqring_overflow_flush(ctx, false, NULL, NULL)) +- return -EBUSY; +- } +- +- /* make sure SQ entry isn't read before tail */ +- nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx)); +- +- if (!percpu_ref_tryget_many(&ctx->refs, nr)) +- return -EAGAIN; +- +- percpu_counter_add(¤t->io_uring->inflight, nr); +- refcount_add(nr, ¤t->usage); +- +- io_submit_state_start(&state, ctx, nr); +- +- for (i = 0; i < nr; i++) { +- const struct io_uring_sqe *sqe; +- struct io_kiocb *req; +- int err; +- +- sqe = io_get_sqe(ctx); +- if (unlikely(!sqe)) { +- io_consume_sqe(ctx); +- break; +- } +- req = io_alloc_req(ctx, &state); +- if (unlikely(!req)) { +- if (!submitted) +- submitted = -EAGAIN; +- break; +- } +- io_consume_sqe(ctx); +- /* will complete beyond this point, count as submitted */ +- submitted++; +- +- err = io_init_req(ctx, req, sqe, &state); +- if (unlikely(err)) { +-fail_req: +- io_put_req(req); +- io_req_complete(req, err); +- break; +- } +- +- trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, +- true, io_async_submit(ctx)); +- err = io_submit_sqe(req, sqe, &link, &state.comp); +- if (err) +- goto fail_req; +- } +- +- if (unlikely(submitted != nr)) { +- int ref_used = (submitted == -EAGAIN) ? 0 : submitted; +- struct io_uring_task *tctx = current->io_uring; +- int unused = nr - ref_used; +- +- percpu_ref_put_many(&ctx->refs, unused); +- percpu_counter_sub(&tctx->inflight, unused); +- put_task_struct_many(current, unused); +- } +- if (link) +- io_queue_link_head(link, &state.comp); +- io_submit_state_end(&state); +- +- /* Commit SQ ring head once we've consumed and submitted all SQEs */ +- io_commit_sqring(ctx); +- +- return submitted; +-} +- +-static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx) +-{ +- /* Tell userspace we may need a wakeup call */ +- spin_lock_irq(&ctx->completion_lock); +- ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP; +- spin_unlock_irq(&ctx->completion_lock); +-} +- +-static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx) +-{ +- spin_lock_irq(&ctx->completion_lock); +- ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP; +- spin_unlock_irq(&ctx->completion_lock); +-} +- +-static int io_sq_wake_function(struct wait_queue_entry *wqe, unsigned mode, +- int sync, void *key) +-{ +- struct io_ring_ctx *ctx = container_of(wqe, struct io_ring_ctx, sqo_wait_entry); +- int ret; +- +- ret = autoremove_wake_function(wqe, mode, sync, key); +- if (ret) { +- unsigned long flags; +- +- spin_lock_irqsave(&ctx->completion_lock, flags); +- ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP; +- spin_unlock_irqrestore(&ctx->completion_lock, flags); +- } +- return ret; +-} +- +-enum sq_ret { +- SQT_IDLE = 1, +- SQT_SPIN = 2, +- SQT_DID_WORK = 4, +-}; +- +-static enum sq_ret __io_sq_thread(struct io_ring_ctx *ctx, +- unsigned long start_jiffies, bool cap_entries) +-{ +- unsigned long timeout = start_jiffies + ctx->sq_thread_idle; +- struct io_sq_data *sqd = ctx->sq_data; +- unsigned int to_submit; +- int ret = 0; +- +-again: +- if (!list_empty(&ctx->iopoll_list)) { +- unsigned nr_events = 0; +- +- mutex_lock(&ctx->uring_lock); +- if (!list_empty(&ctx->iopoll_list) && !need_resched()) +- io_do_iopoll(ctx, &nr_events, 0); +- mutex_unlock(&ctx->uring_lock); +- } +- +- to_submit = io_sqring_entries(ctx); +- +- /* +- * If submit got -EBUSY, flag us as needing the application +- * to enter the kernel to reap and flush events. +- */ +- if (!to_submit || ret == -EBUSY || need_resched()) { +- /* +- * Drop cur_mm before scheduling, we can't hold it for +- * long periods (or over schedule()). Do this before +- * adding ourselves to the waitqueue, as the unuse/drop +- * may sleep. +- */ +- io_sq_thread_drop_mm(); +- +- /* +- * We're polling. If we're within the defined idle +- * period, then let us spin without work before going +- * to sleep. The exception is if we got EBUSY doing +- * more IO, we should wait for the application to +- * reap events and wake us up. +- */ +- if (!list_empty(&ctx->iopoll_list) || need_resched() || +- (!time_after(jiffies, timeout) && ret != -EBUSY && +- !percpu_ref_is_dying(&ctx->refs))) +- return SQT_SPIN; +- +- prepare_to_wait(&sqd->wait, &ctx->sqo_wait_entry, +- TASK_INTERRUPTIBLE); +- +- /* +- * While doing polled IO, before going to sleep, we need +- * to check if there are new reqs added to iopoll_list, +- * it is because reqs may have been punted to io worker +- * and will be added to iopoll_list later, hence check +- * the iopoll_list again. +- */ +- if ((ctx->flags & IORING_SETUP_IOPOLL) && +- !list_empty_careful(&ctx->iopoll_list)) { +- finish_wait(&sqd->wait, &ctx->sqo_wait_entry); +- goto again; +- } +- +- to_submit = io_sqring_entries(ctx); +- if (!to_submit || ret == -EBUSY) +- return SQT_IDLE; +- } +- +- finish_wait(&sqd->wait, &ctx->sqo_wait_entry); +- io_ring_clear_wakeup_flag(ctx); +- +- /* if we're handling multiple rings, cap submit size for fairness */ +- if (cap_entries && to_submit > 8) +- to_submit = 8; +- +- mutex_lock(&ctx->uring_lock); +- if (likely(!percpu_ref_is_dying(&ctx->refs) && !ctx->sqo_dead)) +- ret = io_submit_sqes(ctx, to_submit); +- mutex_unlock(&ctx->uring_lock); +- +- if (!io_sqring_full(ctx) && wq_has_sleeper(&ctx->sqo_sq_wait)) +- wake_up(&ctx->sqo_sq_wait); +- +- return SQT_DID_WORK; +-} +- +-static void io_sqd_init_new(struct io_sq_data *sqd) +-{ +- struct io_ring_ctx *ctx; +- +- while (!list_empty(&sqd->ctx_new_list)) { +- ctx = list_first_entry(&sqd->ctx_new_list, struct io_ring_ctx, sqd_list); +- init_wait(&ctx->sqo_wait_entry); +- ctx->sqo_wait_entry.func = io_sq_wake_function; +- list_move_tail(&ctx->sqd_list, &sqd->ctx_list); +- complete(&ctx->sq_thread_comp); +- } +-} +- +-static int io_sq_thread(void *data) +-{ +- struct cgroup_subsys_state *cur_css = NULL; +- const struct cred *old_cred = NULL; +- struct io_sq_data *sqd = data; +- struct io_ring_ctx *ctx; +- unsigned long start_jiffies; +- +- start_jiffies = jiffies; +- while (!kthread_should_stop()) { +- enum sq_ret ret = 0; +- bool cap_entries; +- +- /* +- * Any changes to the sqd lists are synchronized through the +- * kthread parking. This synchronizes the thread vs users, +- * the users are synchronized on the sqd->ctx_lock. +- */ +- if (kthread_should_park()) { +- kthread_parkme(); +- /* +- * When sq thread is unparked, in case the previous park operation +- * comes from io_put_sq_data(), which means that sq thread is going +- * to be stopped, so here needs to have a check. +- */ +- if (kthread_should_stop()) +- break; +- } +- +- if (unlikely(!list_empty(&sqd->ctx_new_list))) +- io_sqd_init_new(sqd); +- +- cap_entries = !list_is_singular(&sqd->ctx_list); +- +- list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { +- if (current->cred != ctx->creds) { +- if (old_cred) +- revert_creds(old_cred); +- old_cred = override_creds(ctx->creds); +- } +- io_sq_thread_associate_blkcg(ctx, &cur_css); +-#ifdef CONFIG_AUDIT +- current->loginuid = ctx->loginuid; +- current->sessionid = ctx->sessionid; +-#endif +- +- ret |= __io_sq_thread(ctx, start_jiffies, cap_entries); +- +- io_sq_thread_drop_mm(); +- } +- +- if (ret & SQT_SPIN) { +- io_run_task_work(); +- io_sq_thread_drop_mm(); +- cond_resched(); +- } else if (ret == SQT_IDLE) { +- if (kthread_should_park()) +- continue; +- list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) +- io_ring_set_wakeup_flag(ctx); +- schedule(); +- start_jiffies = jiffies; +- list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) +- io_ring_clear_wakeup_flag(ctx); +- } +- } +- +- io_run_task_work(); +- io_sq_thread_drop_mm(); +- +- if (cur_css) +- io_sq_thread_unassociate_blkcg(); +- if (old_cred) +- revert_creds(old_cred); +- +- kthread_parkme(); +- +- return 0; +-} +- +-struct io_wait_queue { +- struct wait_queue_entry wq; +- struct io_ring_ctx *ctx; +- unsigned to_wait; +- unsigned nr_timeouts; +-}; +- +-static inline bool io_should_wake(struct io_wait_queue *iowq) +-{ +- struct io_ring_ctx *ctx = iowq->ctx; +- +- /* +- * Wake up if we have enough events, or if a timeout occurred since we +- * started waiting. For timeouts, we always want to return to userspace, +- * regardless of event count. +- */ +- return io_cqring_events(ctx) >= iowq->to_wait || +- atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; +-} +- +-static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, +- int wake_flags, void *key) +-{ +- struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue, +- wq); +- +- /* +- * Cannot safely flush overflowed CQEs from here, ensure we wake up +- * the task, and the next invocation will do it. +- */ +- if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->cq_check_overflow)) +- return autoremove_wake_function(curr, mode, wake_flags, key); +- return -1; +-} +- +-static int io_run_task_work_sig(void) +-{ +- if (io_run_task_work()) +- return 1; +- if (!signal_pending(current)) +- return 0; +- if (current->jobctl & JOBCTL_TASK_WORK) { +- spin_lock_irq(¤t->sighand->siglock); +- current->jobctl &= ~JOBCTL_TASK_WORK; +- recalc_sigpending(); +- spin_unlock_irq(¤t->sighand->siglock); +- return 1; +- } +- return -EINTR; +-} +- +-/* +- * Wait until events become available, if we don't already have some. The +- * application must reap them itself, as they reside on the shared cq ring. +- */ +-static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, +- const sigset_t __user *sig, size_t sigsz) +-{ +- struct io_wait_queue iowq = { +- .wq = { +- .private = current, +- .func = io_wake_function, +- .entry = LIST_HEAD_INIT(iowq.wq.entry), +- }, +- .ctx = ctx, +- .to_wait = min_events, +- }; +- struct io_rings *rings = ctx->rings; +- int ret = 0; +- +- do { +- io_cqring_overflow_flush(ctx, false, NULL, NULL); +- if (io_cqring_events(ctx) >= min_events) +- return 0; +- if (!io_run_task_work()) +- break; +- } while (1); +- +- if (sig) { +-#ifdef CONFIG_COMPAT +- if (in_compat_syscall()) +- ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig, +- sigsz); +- else +-#endif +- ret = set_user_sigmask(sig, sigsz); +- +- if (ret) +- return ret; +- } +- +- iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts); +- trace_io_uring_cqring_wait(ctx, min_events); +- do { +- io_cqring_overflow_flush(ctx, false, NULL, NULL); +- prepare_to_wait_exclusive(&ctx->wait, &iowq.wq, +- TASK_INTERRUPTIBLE); +- /* make sure we run task_work before checking for signals */ +- ret = io_run_task_work_sig(); +- if (ret > 0) { +- finish_wait(&ctx->wait, &iowq.wq); +- continue; +- } +- else if (ret < 0) +- break; +- if (io_should_wake(&iowq)) +- break; +- if (test_bit(0, &ctx->cq_check_overflow)) { +- finish_wait(&ctx->wait, &iowq.wq); +- continue; +- } +- schedule(); +- } while (1); +- finish_wait(&ctx->wait, &iowq.wq); +- +- restore_saved_sigmask_unless(ret == -EINTR); +- +- return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0; +-} +- +-static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) +-{ +-#if defined(CONFIG_UNIX) +- if (ctx->ring_sock) { +- struct sock *sock = ctx->ring_sock->sk; +- struct sk_buff *skb; +- +- while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL) +- kfree_skb(skb); +- } +-#else +- int i; +- +- for (i = 0; i < ctx->nr_user_files; i++) { +- struct file *file; +- +- file = io_file_from_index(ctx, i); +- if (file) +- fput(file); +- } +-#endif +-} +- +-static void io_file_ref_kill(struct percpu_ref *ref) +-{ +- struct fixed_file_data *data; +- +- data = container_of(ref, struct fixed_file_data, refs); +- complete(&data->done); +-} +- +-static void io_sqe_files_set_node(struct fixed_file_data *file_data, +- struct fixed_file_ref_node *ref_node) +-{ +- spin_lock_bh(&file_data->lock); +- file_data->node = ref_node; +- list_add_tail(&ref_node->node, &file_data->ref_list); +- spin_unlock_bh(&file_data->lock); +- percpu_ref_get(&file_data->refs); +-} +- +- +-static void io_sqe_files_kill_node(struct fixed_file_data *data) +-{ +- struct fixed_file_ref_node *ref_node = NULL; +- +- spin_lock_bh(&data->lock); +- ref_node = data->node; +- spin_unlock_bh(&data->lock); +- if (ref_node) +- percpu_ref_kill(&ref_node->refs); +-} +- +-static int io_file_ref_quiesce(struct fixed_file_data *data, +- struct io_ring_ctx *ctx) +-{ +- int ret; +- struct fixed_file_ref_node *backup_node; +- +- if (data->quiesce) +- return -ENXIO; +- +- data->quiesce = true; +- do { +- backup_node = alloc_fixed_file_ref_node(ctx); +- if (!backup_node) +- break; +- +- io_sqe_files_kill_node(data); +- percpu_ref_kill(&data->refs); +- flush_delayed_work(&ctx->file_put_work); +- +- ret = wait_for_completion_interruptible(&data->done); +- if (!ret) +- break; +- +- percpu_ref_resurrect(&data->refs); +- io_sqe_files_set_node(data, backup_node); +- backup_node = NULL; +- reinit_completion(&data->done); +- mutex_unlock(&ctx->uring_lock); +- ret = io_run_task_work_sig(); +- mutex_lock(&ctx->uring_lock); +- +- if (ret < 0) +- break; +- backup_node = alloc_fixed_file_ref_node(ctx); +- ret = -ENOMEM; +- if (!backup_node) +- break; +- } while (1); +- data->quiesce = false; +- +- if (backup_node) +- destroy_fixed_file_ref_node(backup_node); +- return ret; +-} +- +-static int io_sqe_files_unregister(struct io_ring_ctx *ctx) +-{ +- struct fixed_file_data *data = ctx->file_data; +- unsigned nr_tables, i; +- int ret; +- +- /* +- * percpu_ref_is_dying() is to stop parallel files unregister +- * Since we possibly drop uring lock later in this function to +- * run task work. +- */ +- if (!data || percpu_ref_is_dying(&data->refs)) +- return -ENXIO; +- ret = io_file_ref_quiesce(data, ctx); +- if (ret) +- return ret; +- +- __io_sqe_files_unregister(ctx); +- nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE); +- for (i = 0; i < nr_tables; i++) +- kfree(data->table[i].files); +- kfree(data->table); +- percpu_ref_exit(&data->refs); +- kfree(data); +- ctx->file_data = NULL; +- ctx->nr_user_files = 0; +- return 0; +-} +- +-static void io_put_sq_data(struct io_sq_data *sqd) +-{ +- if (refcount_dec_and_test(&sqd->refs)) { +- /* +- * The park is a bit of a work-around, without it we get +- * warning spews on shutdown with SQPOLL set and affinity +- * set to a single CPU. +- */ +- if (sqd->thread) { +- kthread_park(sqd->thread); +- kthread_stop(sqd->thread); +- } +- +- kfree(sqd); +- } +-} +- +-static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p) +-{ +- struct io_ring_ctx *ctx_attach; +- struct io_sq_data *sqd; +- struct fd f; +- +- f = fdget(p->wq_fd); +- if (!f.file) +- return ERR_PTR(-ENXIO); +- if (f.file->f_op != &io_uring_fops) { +- fdput(f); +- return ERR_PTR(-EINVAL); +- } +- +- ctx_attach = f.file->private_data; +- sqd = ctx_attach->sq_data; +- if (!sqd) { +- fdput(f); +- return ERR_PTR(-EINVAL); +- } +- +- refcount_inc(&sqd->refs); +- fdput(f); +- return sqd; +-} +- +-static struct io_sq_data *io_get_sq_data(struct io_uring_params *p) +-{ +- struct io_sq_data *sqd; +- +- if (p->flags & IORING_SETUP_ATTACH_WQ) +- return io_attach_sq_data(p); +- +- sqd = kzalloc(sizeof(*sqd), GFP_KERNEL); +- if (!sqd) +- return ERR_PTR(-ENOMEM); +- +- refcount_set(&sqd->refs, 1); +- INIT_LIST_HEAD(&sqd->ctx_list); +- INIT_LIST_HEAD(&sqd->ctx_new_list); +- mutex_init(&sqd->ctx_lock); +- mutex_init(&sqd->lock); +- init_waitqueue_head(&sqd->wait); +- return sqd; +-} +- +-static void io_sq_thread_unpark(struct io_sq_data *sqd) +- __releases(&sqd->lock) +-{ +- if (!sqd->thread) +- return; +- kthread_unpark(sqd->thread); +- mutex_unlock(&sqd->lock); +-} +- +-static void io_sq_thread_park(struct io_sq_data *sqd) +- __acquires(&sqd->lock) +-{ +- if (!sqd->thread) +- return; +- mutex_lock(&sqd->lock); +- kthread_park(sqd->thread); +-} +- +-static void io_sq_thread_stop(struct io_ring_ctx *ctx) +-{ +- struct io_sq_data *sqd = ctx->sq_data; +- +- if (sqd) { +- if (sqd->thread) { +- /* +- * We may arrive here from the error branch in +- * io_sq_offload_create() where the kthread is created +- * without being waked up, thus wake it up now to make +- * sure the wait will complete. +- */ +- wake_up_process(sqd->thread); +- wait_for_completion(&ctx->sq_thread_comp); +- +- io_sq_thread_park(sqd); +- } +- +- mutex_lock(&sqd->ctx_lock); +- list_del(&ctx->sqd_list); +- mutex_unlock(&sqd->ctx_lock); +- +- if (sqd->thread) { +- finish_wait(&sqd->wait, &ctx->sqo_wait_entry); +- io_sq_thread_unpark(sqd); +- } +- +- io_put_sq_data(sqd); +- ctx->sq_data = NULL; +- } +-} +- +-static void io_finish_async(struct io_ring_ctx *ctx) +-{ +- io_sq_thread_stop(ctx); +- +- if (ctx->io_wq) { +- io_wq_destroy(ctx->io_wq); +- ctx->io_wq = NULL; +- } +-} +- +-#if defined(CONFIG_UNIX) +-/* +- * Ensure the UNIX gc is aware of our file set, so we are certain that +- * the io_uring can be safely unregistered on process exit, even if we have +- * loops in the file referencing. +- */ +-static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset) +-{ +- struct sock *sk = ctx->ring_sock->sk; +- struct scm_fp_list *fpl; +- struct sk_buff *skb; +- int i, nr_files; +- +- fpl = kzalloc(sizeof(*fpl), GFP_KERNEL); +- if (!fpl) +- return -ENOMEM; +- +- skb = alloc_skb(0, GFP_KERNEL); +- if (!skb) { +- kfree(fpl); +- return -ENOMEM; +- } +- +- skb->sk = sk; +- skb->scm_io_uring = 1; +- +- nr_files = 0; +- fpl->user = get_uid(ctx->user); +- for (i = 0; i < nr; i++) { +- struct file *file = io_file_from_index(ctx, i + offset); +- +- if (!file) +- continue; +- fpl->fp[nr_files] = get_file(file); +- unix_inflight(fpl->user, fpl->fp[nr_files]); +- nr_files++; +- } +- +- if (nr_files) { +- fpl->max = SCM_MAX_FD; +- fpl->count = nr_files; +- UNIXCB(skb).fp = fpl; +- skb->destructor = unix_destruct_scm; +- refcount_add(skb->truesize, &sk->sk_wmem_alloc); +- skb_queue_head(&sk->sk_receive_queue, skb); +- +- for (i = 0; i < nr; i++) { +- struct file *file = io_file_from_index(ctx, i + offset); +- +- if (file) +- fput(file); +- } +- } else { +- kfree_skb(skb); +- free_uid(fpl->user); +- kfree(fpl); +- } +- +- return 0; +-} +- +-/* +- * If UNIX sockets are enabled, fd passing can cause a reference cycle which +- * causes regular reference counting to break down. We rely on the UNIX +- * garbage collection to take care of this problem for us. +- */ +-static int io_sqe_files_scm(struct io_ring_ctx *ctx) +-{ +- unsigned left, total; +- int ret = 0; +- +- total = 0; +- left = ctx->nr_user_files; +- while (left) { +- unsigned this_files = min_t(unsigned, left, SCM_MAX_FD); +- +- ret = __io_sqe_files_scm(ctx, this_files, total); +- if (ret) +- break; +- left -= this_files; +- total += this_files; +- } +- +- if (!ret) +- return 0; +- +- while (total < ctx->nr_user_files) { +- struct file *file = io_file_from_index(ctx, total); +- +- if (file) +- fput(file); +- total++; +- } +- +- return ret; +-} +-#else +-static int io_sqe_files_scm(struct io_ring_ctx *ctx) +-{ +- return 0; +-} +-#endif +- +-static int io_sqe_alloc_file_tables(struct fixed_file_data *file_data, +- unsigned nr_tables, unsigned nr_files) +-{ +- int i; +- +- for (i = 0; i < nr_tables; i++) { +- struct fixed_file_table *table = &file_data->table[i]; +- unsigned this_files; +- +- this_files = min(nr_files, IORING_MAX_FILES_TABLE); +- table->files = kcalloc(this_files, sizeof(struct file *), +- GFP_KERNEL_ACCOUNT); +- if (!table->files) +- break; +- nr_files -= this_files; +- } +- +- if (i == nr_tables) +- return 0; +- +- for (i = 0; i < nr_tables; i++) { +- struct fixed_file_table *table = &file_data->table[i]; +- kfree(table->files); +- } +- return 1; +-} +- +-static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file) +-{ +-#if defined(CONFIG_UNIX) +- struct sock *sock = ctx->ring_sock->sk; +- struct sk_buff_head list, *head = &sock->sk_receive_queue; +- struct sk_buff *skb; +- int i; +- +- __skb_queue_head_init(&list); +- +- /* +- * Find the skb that holds this file in its SCM_RIGHTS. When found, +- * remove this entry and rearrange the file array. +- */ +- skb = skb_dequeue(head); +- while (skb) { +- struct scm_fp_list *fp; +- +- fp = UNIXCB(skb).fp; +- for (i = 0; i < fp->count; i++) { +- int left; +- +- if (fp->fp[i] != file) +- continue; +- +- unix_notinflight(fp->user, fp->fp[i]); +- left = fp->count - 1 - i; +- if (left) { +- memmove(&fp->fp[i], &fp->fp[i + 1], +- left * sizeof(struct file *)); +- } +- fp->count--; +- if (!fp->count) { +- kfree_skb(skb); +- skb = NULL; +- } else { +- __skb_queue_tail(&list, skb); +- } +- fput(file); +- file = NULL; +- break; +- } +- +- if (!file) +- break; +- +- __skb_queue_tail(&list, skb); +- +- skb = skb_dequeue(head); +- } +- +- if (skb_peek(&list)) { +- spin_lock_irq(&head->lock); +- while ((skb = __skb_dequeue(&list)) != NULL) +- __skb_queue_tail(head, skb); +- spin_unlock_irq(&head->lock); +- } +-#else +- fput(file); +-#endif +-} +- +-struct io_file_put { +- struct list_head list; +- struct file *file; +-}; +- +-static void __io_file_put_work(struct fixed_file_ref_node *ref_node) +-{ +- struct fixed_file_data *file_data = ref_node->file_data; +- struct io_ring_ctx *ctx = file_data->ctx; +- struct io_file_put *pfile, *tmp; +- +- list_for_each_entry_safe(pfile, tmp, &ref_node->file_list, list) { +- list_del(&pfile->list); +- io_ring_file_put(ctx, pfile->file); +- kfree(pfile); +- } +- +- percpu_ref_exit(&ref_node->refs); +- kfree(ref_node); +- percpu_ref_put(&file_data->refs); +-} +- +-static void io_file_put_work(struct work_struct *work) +-{ +- struct io_ring_ctx *ctx; +- struct llist_node *node; +- +- ctx = container_of(work, struct io_ring_ctx, file_put_work.work); +- node = llist_del_all(&ctx->file_put_llist); +- +- while (node) { +- struct fixed_file_ref_node *ref_node; +- struct llist_node *next = node->next; +- +- ref_node = llist_entry(node, struct fixed_file_ref_node, llist); +- __io_file_put_work(ref_node); +- node = next; +- } +-} +- +-static void io_file_data_ref_zero(struct percpu_ref *ref) +-{ +- struct fixed_file_ref_node *ref_node; +- struct fixed_file_data *data; +- struct io_ring_ctx *ctx; +- bool first_add = false; +- int delay = HZ; +- +- ref_node = container_of(ref, struct fixed_file_ref_node, refs); +- data = ref_node->file_data; +- ctx = data->ctx; +- +- spin_lock_bh(&data->lock); +- ref_node->done = true; +- +- while (!list_empty(&data->ref_list)) { +- ref_node = list_first_entry(&data->ref_list, +- struct fixed_file_ref_node, node); +- /* recycle ref nodes in order */ +- if (!ref_node->done) +- break; +- list_del(&ref_node->node); +- first_add |= llist_add(&ref_node->llist, &ctx->file_put_llist); +- } +- spin_unlock_bh(&data->lock); +- +- if (percpu_ref_is_dying(&data->refs)) +- delay = 0; +- +- if (!delay) +- mod_delayed_work(system_wq, &ctx->file_put_work, 0); +- else if (first_add) +- queue_delayed_work(system_wq, &ctx->file_put_work, delay); +-} +- +-static struct fixed_file_ref_node *alloc_fixed_file_ref_node( +- struct io_ring_ctx *ctx) +-{ +- struct fixed_file_ref_node *ref_node; +- +- ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL); +- if (!ref_node) +- return NULL; +- +- if (percpu_ref_init(&ref_node->refs, io_file_data_ref_zero, +- 0, GFP_KERNEL)) { +- kfree(ref_node); +- return NULL; +- } +- INIT_LIST_HEAD(&ref_node->node); +- INIT_LIST_HEAD(&ref_node->file_list); +- ref_node->file_data = ctx->file_data; +- ref_node->done = false; +- return ref_node; +-} +- +-static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node) +-{ +- percpu_ref_exit(&ref_node->refs); +- kfree(ref_node); +-} +- +-static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, +- unsigned nr_args) +-{ +- __s32 __user *fds = (__s32 __user *) arg; +- unsigned nr_tables, i; +- struct file *file; +- int fd, ret = -ENOMEM; +- struct fixed_file_ref_node *ref_node; +- struct fixed_file_data *file_data; +- +- if (ctx->file_data) +- return -EBUSY; +- if (!nr_args) +- return -EINVAL; +- if (nr_args > IORING_MAX_FIXED_FILES) +- return -EMFILE; +- if (nr_args > rlimit(RLIMIT_NOFILE)) +- return -EMFILE; +- +- file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL_ACCOUNT); +- if (!file_data) +- return -ENOMEM; +- file_data->ctx = ctx; +- init_completion(&file_data->done); +- INIT_LIST_HEAD(&file_data->ref_list); +- spin_lock_init(&file_data->lock); +- +- nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE); +- file_data->table = kcalloc(nr_tables, sizeof(*file_data->table), +- GFP_KERNEL_ACCOUNT); +- if (!file_data->table) +- goto out_free; +- +- if (percpu_ref_init(&file_data->refs, io_file_ref_kill, +- PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) +- goto out_free; +- +- if (io_sqe_alloc_file_tables(file_data, nr_tables, nr_args)) +- goto out_ref; +- ctx->file_data = file_data; +- +- for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { +- struct fixed_file_table *table; +- unsigned index; +- +- if (copy_from_user(&fd, &fds[i], sizeof(fd))) { +- ret = -EFAULT; +- goto out_fput; +- } +- /* allow sparse sets */ +- if (fd == -1) +- continue; +- +- file = fget(fd); +- ret = -EBADF; +- if (!file) +- goto out_fput; +- +- /* +- * Don't allow io_uring instances to be registered. If UNIX +- * isn't enabled, then this causes a reference cycle and this +- * instance can never get freed. If UNIX is enabled we'll +- * handle it just fine, but there's still no point in allowing +- * a ring fd as it doesn't support regular read/write anyway. +- */ +- if (file->f_op == &io_uring_fops) { +- fput(file); +- goto out_fput; +- } +- table = &file_data->table[i >> IORING_FILE_TABLE_SHIFT]; +- index = i & IORING_FILE_TABLE_MASK; +- table->files[index] = file; +- } +- +- ret = io_sqe_files_scm(ctx); +- if (ret) { +- io_sqe_files_unregister(ctx); +- return ret; +- } +- +- ref_node = alloc_fixed_file_ref_node(ctx); +- if (!ref_node) { +- io_sqe_files_unregister(ctx); +- return -ENOMEM; +- } +- +- io_sqe_files_set_node(file_data, ref_node); +- return ret; +-out_fput: +- for (i = 0; i < ctx->nr_user_files; i++) { +- file = io_file_from_index(ctx, i); +- if (file) +- fput(file); +- } +- for (i = 0; i < nr_tables; i++) +- kfree(file_data->table[i].files); +- ctx->nr_user_files = 0; +-out_ref: +- percpu_ref_exit(&file_data->refs); +-out_free: +- kfree(file_data->table); +- kfree(file_data); +- ctx->file_data = NULL; +- return ret; +-} +- +-static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file, +- int index) +-{ +-#if defined(CONFIG_UNIX) +- struct sock *sock = ctx->ring_sock->sk; +- struct sk_buff_head *head = &sock->sk_receive_queue; +- struct sk_buff *skb; +- +- /* +- * See if we can merge this file into an existing skb SCM_RIGHTS +- * file set. If there's no room, fall back to allocating a new skb +- * and filling it in. +- */ +- spin_lock_irq(&head->lock); +- skb = skb_peek(head); +- if (skb) { +- struct scm_fp_list *fpl = UNIXCB(skb).fp; +- +- if (fpl->count < SCM_MAX_FD) { +- __skb_unlink(skb, head); +- spin_unlock_irq(&head->lock); +- fpl->fp[fpl->count] = get_file(file); +- unix_inflight(fpl->user, fpl->fp[fpl->count]); +- fpl->count++; +- spin_lock_irq(&head->lock); +- __skb_queue_head(head, skb); +- } else { +- skb = NULL; +- } +- } +- spin_unlock_irq(&head->lock); +- +- if (skb) { +- fput(file); +- return 0; +- } +- +- return __io_sqe_files_scm(ctx, 1, index); +-#else +- return 0; +-#endif +-} +- +-static int io_queue_file_removal(struct fixed_file_data *data, +- struct file *file) +-{ +- struct io_file_put *pfile; +- struct fixed_file_ref_node *ref_node = data->node; +- +- pfile = kzalloc(sizeof(*pfile), GFP_KERNEL); +- if (!pfile) +- return -ENOMEM; +- +- pfile->file = file; +- list_add(&pfile->list, &ref_node->file_list); +- +- return 0; +-} +- +-static int __io_sqe_files_update(struct io_ring_ctx *ctx, +- struct io_uring_files_update *up, +- unsigned nr_args) +-{ +- struct fixed_file_data *data = ctx->file_data; +- struct fixed_file_ref_node *ref_node; +- struct file *file; +- __s32 __user *fds; +- int fd, i, err; +- __u32 done; +- bool needs_switch = false; +- +- if (check_add_overflow(up->offset, nr_args, &done)) +- return -EOVERFLOW; +- if (done > ctx->nr_user_files) +- return -EINVAL; +- +- ref_node = alloc_fixed_file_ref_node(ctx); +- if (!ref_node) +- return -ENOMEM; +- +- done = 0; +- fds = u64_to_user_ptr(up->fds); +- while (nr_args) { +- struct fixed_file_table *table; +- unsigned index; +- +- err = 0; +- if (copy_from_user(&fd, &fds[done], sizeof(fd))) { +- err = -EFAULT; +- break; +- } +- i = array_index_nospec(up->offset, ctx->nr_user_files); +- table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; +- index = i & IORING_FILE_TABLE_MASK; +- if (table->files[index]) { +- file = table->files[index]; +- err = io_queue_file_removal(data, file); +- if (err) +- break; +- table->files[index] = NULL; +- needs_switch = true; +- } +- if (fd != -1) { +- file = fget(fd); +- if (!file) { +- err = -EBADF; +- break; +- } +- /* +- * Don't allow io_uring instances to be registered. If +- * UNIX isn't enabled, then this causes a reference +- * cycle and this instance can never get freed. If UNIX +- * is enabled we'll handle it just fine, but there's +- * still no point in allowing a ring fd as it doesn't +- * support regular read/write anyway. +- */ +- if (file->f_op == &io_uring_fops) { +- fput(file); +- err = -EBADF; +- break; +- } +- table->files[index] = file; +- err = io_sqe_file_register(ctx, file, i); +- if (err) { +- table->files[index] = NULL; +- fput(file); +- break; +- } +- } +- nr_args--; +- done++; +- up->offset++; +- } +- +- if (needs_switch) { +- percpu_ref_kill(&data->node->refs); +- io_sqe_files_set_node(data, ref_node); +- } else +- destroy_fixed_file_ref_node(ref_node); +- +- return done ? done : err; +-} +- +-static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg, +- unsigned nr_args) +-{ +- struct io_uring_files_update up; +- +- if (!ctx->file_data) +- return -ENXIO; +- if (!nr_args) +- return -EINVAL; +- if (copy_from_user(&up, arg, sizeof(up))) +- return -EFAULT; +- if (up.resv) +- return -EINVAL; +- +- return __io_sqe_files_update(ctx, &up, nr_args); +-} +- +-static void io_free_work(struct io_wq_work *work) +-{ +- struct io_kiocb *req = container_of(work, struct io_kiocb, work); +- +- /* Consider that io_steal_work() relies on this ref */ +- io_put_req(req); +-} +- +-static int io_init_wq_offload(struct io_ring_ctx *ctx, +- struct io_uring_params *p) +-{ +- struct io_wq_data data; +- struct fd f; +- struct io_ring_ctx *ctx_attach; +- unsigned int concurrency; +- int ret = 0; +- +- data.user = ctx->user; +- data.free_work = io_free_work; +- data.do_work = io_wq_submit_work; +- +- if (!(p->flags & IORING_SETUP_ATTACH_WQ)) { +- /* Do QD, or 4 * CPUS, whatever is smallest */ +- concurrency = min(ctx->sq_entries, 4 * num_online_cpus()); +- +- ctx->io_wq = io_wq_create(concurrency, &data); +- if (IS_ERR(ctx->io_wq)) { +- ret = PTR_ERR(ctx->io_wq); +- ctx->io_wq = NULL; +- } +- return ret; +- } +- +- f = fdget(p->wq_fd); +- if (!f.file) +- return -EBADF; +- +- if (f.file->f_op != &io_uring_fops) { +- ret = -EINVAL; +- goto out_fput; +- } +- +- ctx_attach = f.file->private_data; +- /* @io_wq is protected by holding the fd */ +- if (!io_wq_get(ctx_attach->io_wq, &data)) { +- ret = -EINVAL; +- goto out_fput; +- } +- +- ctx->io_wq = ctx_attach->io_wq; +-out_fput: +- fdput(f); +- return ret; +-} +- +-static int io_uring_alloc_task_context(struct task_struct *task) +-{ +- struct io_uring_task *tctx; +- int ret; +- +- tctx = kmalloc(sizeof(*tctx), GFP_KERNEL); +- if (unlikely(!tctx)) +- return -ENOMEM; +- +- ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL); +- if (unlikely(ret)) { +- kfree(tctx); +- return ret; +- } +- +- xa_init(&tctx->xa); +- init_waitqueue_head(&tctx->wait); +- tctx->last = NULL; +- atomic_set(&tctx->in_idle, 0); +- tctx->sqpoll = false; +- io_init_identity(&tctx->__identity); +- tctx->identity = &tctx->__identity; +- task->io_uring = tctx; +- return 0; +-} +- +-void __io_uring_free(struct task_struct *tsk) +-{ +- struct io_uring_task *tctx = tsk->io_uring; +- +- WARN_ON_ONCE(!xa_empty(&tctx->xa)); +- WARN_ON_ONCE(refcount_read(&tctx->identity->count) != 1); +- if (tctx->identity != &tctx->__identity) +- kfree(tctx->identity); +- percpu_counter_destroy(&tctx->inflight); +- kfree(tctx); +- tsk->io_uring = NULL; +-} +- +-static int io_sq_offload_create(struct io_ring_ctx *ctx, +- struct io_uring_params *p) +-{ +- int ret; +- +- if (ctx->flags & IORING_SETUP_SQPOLL) { +- struct io_sq_data *sqd; +- +- ret = -EPERM; +- if (!capable(CAP_SYS_ADMIN)) +- goto err; +- +- sqd = io_get_sq_data(p); +- if (IS_ERR(sqd)) { +- ret = PTR_ERR(sqd); +- goto err; +- } +- +- ctx->sq_data = sqd; +- io_sq_thread_park(sqd); +- mutex_lock(&sqd->ctx_lock); +- list_add(&ctx->sqd_list, &sqd->ctx_new_list); +- mutex_unlock(&sqd->ctx_lock); +- io_sq_thread_unpark(sqd); +- +- ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle); +- if (!ctx->sq_thread_idle) +- ctx->sq_thread_idle = HZ; +- +- if (sqd->thread) +- goto done; +- +- if (p->flags & IORING_SETUP_SQ_AFF) { +- int cpu = p->sq_thread_cpu; +- +- ret = -EINVAL; +- if (cpu >= nr_cpu_ids) +- goto err; +- if (!cpu_online(cpu)) +- goto err; +- +- sqd->thread = kthread_create_on_cpu(io_sq_thread, sqd, +- cpu, "io_uring-sq"); +- } else { +- sqd->thread = kthread_create(io_sq_thread, sqd, +- "io_uring-sq"); +- } +- if (IS_ERR(sqd->thread)) { +- ret = PTR_ERR(sqd->thread); +- sqd->thread = NULL; +- goto err; +- } +- ret = io_uring_alloc_task_context(sqd->thread); +- if (ret) +- goto err; +- } else if (p->flags & IORING_SETUP_SQ_AFF) { +- /* Can't have SQ_AFF without SQPOLL */ +- ret = -EINVAL; +- goto err; +- } +- +-done: +- ret = io_init_wq_offload(ctx, p); +- if (ret) +- goto err; +- +- return 0; +-err: +- io_finish_async(ctx); +- return ret; +-} +- +-static void io_sq_offload_start(struct io_ring_ctx *ctx) +-{ +- struct io_sq_data *sqd = ctx->sq_data; +- +- ctx->flags &= ~IORING_SETUP_R_DISABLED; +- if ((ctx->flags & IORING_SETUP_SQPOLL) && sqd && sqd->thread) +- wake_up_process(sqd->thread); +-} +- +-static inline void __io_unaccount_mem(struct user_struct *user, +- unsigned long nr_pages) +-{ +- atomic_long_sub(nr_pages, &user->locked_vm); +-} +- +-static inline int __io_account_mem(struct user_struct *user, +- unsigned long nr_pages) +-{ +- unsigned long page_limit, cur_pages, new_pages; +- +- /* Don't allow more pages than we can safely lock */ +- page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; +- +- do { +- cur_pages = atomic_long_read(&user->locked_vm); +- new_pages = cur_pages + nr_pages; +- if (new_pages > page_limit) +- return -ENOMEM; +- } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages, +- new_pages) != cur_pages); +- +- return 0; +-} +- +-static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages, +- enum io_mem_account acct) +-{ +- if (ctx->limit_mem) +- __io_unaccount_mem(ctx->user, nr_pages); +- +- if (ctx->mm_account) { +- if (acct == ACCT_LOCKED) +- ctx->mm_account->locked_vm -= nr_pages; +- else if (acct == ACCT_PINNED) +- atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm); +- } +-} +- +-static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages, +- enum io_mem_account acct) +-{ +- int ret; +- +- if (ctx->limit_mem) { +- ret = __io_account_mem(ctx->user, nr_pages); +- if (ret) +- return ret; +- } +- +- if (ctx->mm_account) { +- if (acct == ACCT_LOCKED) +- ctx->mm_account->locked_vm += nr_pages; +- else if (acct == ACCT_PINNED) +- atomic64_add(nr_pages, &ctx->mm_account->pinned_vm); +- } +- +- return 0; +-} +- +-static void io_mem_free(void *ptr) +-{ +- struct page *page; +- +- if (!ptr) +- return; +- +- page = virt_to_head_page(ptr); +- if (put_page_testzero(page)) +- free_compound_page(page); +-} +- +-static void *io_mem_alloc(size_t size) +-{ +- gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP | +- __GFP_NORETRY; +- +- return (void *) __get_free_pages(gfp_flags, get_order(size)); +-} +- +-static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries, +- size_t *sq_offset) +-{ +- struct io_rings *rings; +- size_t off, sq_array_size; +- +- off = struct_size(rings, cqes, cq_entries); +- if (off == SIZE_MAX) +- return SIZE_MAX; +- +-#ifdef CONFIG_SMP +- off = ALIGN(off, SMP_CACHE_BYTES); +- if (off == 0) +- return SIZE_MAX; +-#endif +- +- if (sq_offset) +- *sq_offset = off; +- +- sq_array_size = array_size(sizeof(u32), sq_entries); +- if (sq_array_size == SIZE_MAX) +- return SIZE_MAX; +- +- if (check_add_overflow(off, sq_array_size, &off)) +- return SIZE_MAX; +- +- return off; +-} +- +-static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries) +-{ +- size_t pages; +- +- pages = (size_t)1 << get_order( +- rings_size(sq_entries, cq_entries, NULL)); +- pages += (size_t)1 << get_order( +- array_size(sizeof(struct io_uring_sqe), sq_entries)); +- +- return pages; +-} +- +-static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx) +-{ +- int i, j; +- +- if (!ctx->user_bufs) +- return -ENXIO; +- +- for (i = 0; i < ctx->nr_user_bufs; i++) { +- struct io_mapped_ubuf *imu = &ctx->user_bufs[i]; +- +- for (j = 0; j < imu->nr_bvecs; j++) +- unpin_user_page(imu->bvec[j].bv_page); +- +- if (imu->acct_pages) +- io_unaccount_mem(ctx, imu->acct_pages, ACCT_PINNED); +- kvfree(imu->bvec); +- imu->nr_bvecs = 0; +- } +- +- kfree(ctx->user_bufs); +- ctx->user_bufs = NULL; +- ctx->nr_user_bufs = 0; +- return 0; +-} +- +-static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst, +- void __user *arg, unsigned index) +-{ +- struct iovec __user *src; +- +-#ifdef CONFIG_COMPAT +- if (ctx->compat) { +- struct compat_iovec __user *ciovs; +- struct compat_iovec ciov; +- +- ciovs = (struct compat_iovec __user *) arg; +- if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov))) +- return -EFAULT; +- +- dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base); +- dst->iov_len = ciov.iov_len; +- return 0; +- } +-#endif +- src = (struct iovec __user *) arg; +- if (copy_from_user(dst, &src[index], sizeof(*dst))) +- return -EFAULT; +- return 0; +-} +- +-/* +- * Not super efficient, but this is just a registration time. And we do cache +- * the last compound head, so generally we'll only do a full search if we don't +- * match that one. +- * +- * We check if the given compound head page has already been accounted, to +- * avoid double accounting it. This allows us to account the full size of the +- * page, not just the constituent pages of a huge page. +- */ +-static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages, +- int nr_pages, struct page *hpage) +-{ +- int i, j; +- +- /* check current page array */ +- for (i = 0; i < nr_pages; i++) { +- if (!PageCompound(pages[i])) +- continue; +- if (compound_head(pages[i]) == hpage) +- return true; +- } +- +- /* check previously registered pages */ +- for (i = 0; i < ctx->nr_user_bufs; i++) { +- struct io_mapped_ubuf *imu = &ctx->user_bufs[i]; +- +- for (j = 0; j < imu->nr_bvecs; j++) { +- if (!PageCompound(imu->bvec[j].bv_page)) +- continue; +- if (compound_head(imu->bvec[j].bv_page) == hpage) +- return true; +- } +- } +- +- return false; +-} +- +-static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages, +- int nr_pages, struct io_mapped_ubuf *imu, +- struct page **last_hpage) +-{ +- int i, ret; +- +- for (i = 0; i < nr_pages; i++) { +- if (!PageCompound(pages[i])) { +- imu->acct_pages++; +- } else { +- struct page *hpage; +- +- hpage = compound_head(pages[i]); +- if (hpage == *last_hpage) +- continue; +- *last_hpage = hpage; +- if (headpage_already_acct(ctx, pages, i, hpage)) +- continue; +- imu->acct_pages += page_size(hpage) >> PAGE_SHIFT; +- } +- } +- +- if (!imu->acct_pages) +- return 0; +- +- ret = io_account_mem(ctx, imu->acct_pages, ACCT_PINNED); +- if (ret) +- imu->acct_pages = 0; +- return ret; +-} +- +-static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg, +- unsigned nr_args) +-{ +- struct vm_area_struct **vmas = NULL; +- struct page **pages = NULL; +- struct page *last_hpage = NULL; +- int i, j, got_pages = 0; +- int ret = -EINVAL; +- +- if (ctx->user_bufs) +- return -EBUSY; +- if (!nr_args || nr_args > UIO_MAXIOV) +- return -EINVAL; +- +- ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf), +- GFP_KERNEL); +- if (!ctx->user_bufs) +- return -ENOMEM; +- +- for (i = 0; i < nr_args; i++) { +- struct io_mapped_ubuf *imu = &ctx->user_bufs[i]; +- unsigned long off, start, end, ubuf; +- int pret, nr_pages; +- struct iovec iov; +- size_t size; +- +- ret = io_copy_iov(ctx, &iov, arg, i); +- if (ret) +- goto err; +- +- /* +- * Don't impose further limits on the size and buffer +- * constraints here, we'll -EINVAL later when IO is +- * submitted if they are wrong. +- */ +- ret = -EFAULT; +- if (!iov.iov_base || !iov.iov_len) +- goto err; +- +- /* arbitrary limit, but we need something */ +- if (iov.iov_len > SZ_1G) +- goto err; +- +- ubuf = (unsigned long) iov.iov_base; +- end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT; +- start = ubuf >> PAGE_SHIFT; +- nr_pages = end - start; +- +- ret = 0; +- if (!pages || nr_pages > got_pages) { +- kvfree(vmas); +- kvfree(pages); +- pages = kvmalloc_array(nr_pages, sizeof(struct page *), +- GFP_KERNEL); +- vmas = kvmalloc_array(nr_pages, +- sizeof(struct vm_area_struct *), +- GFP_KERNEL); +- if (!pages || !vmas) { +- ret = -ENOMEM; +- goto err; +- } +- got_pages = nr_pages; +- } +- +- imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec), +- GFP_KERNEL); +- ret = -ENOMEM; +- if (!imu->bvec) +- goto err; +- +- ret = 0; +- mmap_read_lock(current->mm); +- pret = pin_user_pages(ubuf, nr_pages, +- FOLL_WRITE | FOLL_LONGTERM, +- pages, vmas); +- if (pret == nr_pages) { +- /* don't support file backed memory */ +- for (j = 0; j < nr_pages; j++) { +- struct vm_area_struct *vma = vmas[j]; +- +- if (vma->vm_file && +- !is_file_hugepages(vma->vm_file)) { +- ret = -EOPNOTSUPP; +- break; +- } +- } +- } else { +- ret = pret < 0 ? pret : -EFAULT; +- } +- mmap_read_unlock(current->mm); +- if (ret) { +- /* +- * if we did partial map, or found file backed vmas, +- * release any pages we did get +- */ +- if (pret > 0) +- unpin_user_pages(pages, pret); +- kvfree(imu->bvec); +- goto err; +- } +- +- ret = io_buffer_account_pin(ctx, pages, pret, imu, &last_hpage); +- if (ret) { +- unpin_user_pages(pages, pret); +- kvfree(imu->bvec); +- goto err; +- } +- +- off = ubuf & ~PAGE_MASK; +- size = iov.iov_len; +- for (j = 0; j < nr_pages; j++) { +- size_t vec_len; +- +- vec_len = min_t(size_t, size, PAGE_SIZE - off); +- imu->bvec[j].bv_page = pages[j]; +- imu->bvec[j].bv_len = vec_len; +- imu->bvec[j].bv_offset = off; +- off = 0; +- size -= vec_len; +- } +- /* store original address for later verification */ +- imu->ubuf = ubuf; +- imu->len = iov.iov_len; +- imu->nr_bvecs = nr_pages; +- +- ctx->nr_user_bufs++; +- } +- kvfree(pages); +- kvfree(vmas); +- return 0; +-err: +- kvfree(pages); +- kvfree(vmas); +- io_sqe_buffer_unregister(ctx); +- return ret; +-} +- +-static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg) +-{ +- __s32 __user *fds = arg; +- int fd; +- +- if (ctx->cq_ev_fd) +- return -EBUSY; +- +- if (copy_from_user(&fd, fds, sizeof(*fds))) +- return -EFAULT; +- +- ctx->cq_ev_fd = eventfd_ctx_fdget(fd); +- if (IS_ERR(ctx->cq_ev_fd)) { +- int ret = PTR_ERR(ctx->cq_ev_fd); +- ctx->cq_ev_fd = NULL; +- return ret; +- } +- +- return 0; +-} +- +-static int io_eventfd_unregister(struct io_ring_ctx *ctx) +-{ +- if (ctx->cq_ev_fd) { +- eventfd_ctx_put(ctx->cq_ev_fd); +- ctx->cq_ev_fd = NULL; +- return 0; +- } +- +- return -ENXIO; +-} +- +-static void io_destroy_buffers(struct io_ring_ctx *ctx) +-{ +- struct io_buffer *buf; +- unsigned long index; +- +- xa_for_each(&ctx->io_buffers, index, buf) +- __io_remove_buffers(ctx, buf, index, -1U); +-} +- +-static void io_ring_ctx_free(struct io_ring_ctx *ctx) +-{ +- io_finish_async(ctx); +- io_sqe_buffer_unregister(ctx); +- +- if (ctx->sqo_task) { +- put_task_struct(ctx->sqo_task); +- ctx->sqo_task = NULL; +- } +- +-#ifdef CONFIG_BLK_CGROUP +- if (ctx->sqo_blkcg_css) +- css_put(ctx->sqo_blkcg_css); +-#endif +- +- mutex_lock(&ctx->uring_lock); +- io_sqe_files_unregister(ctx); +- mutex_unlock(&ctx->uring_lock); +- io_eventfd_unregister(ctx); +- io_destroy_buffers(ctx); +- +-#if defined(CONFIG_UNIX) +- if (ctx->ring_sock) { +- ctx->ring_sock->file = NULL; /* so that iput() is called */ +- sock_release(ctx->ring_sock); +- } +-#endif +- +- if (ctx->mm_account) { +- mmdrop(ctx->mm_account); +- ctx->mm_account = NULL; +- } +- +- io_mem_free(ctx->rings); +- io_mem_free(ctx->sq_sqes); +- +- percpu_ref_exit(&ctx->refs); +- free_uid(ctx->user); +- put_cred(ctx->creds); +- kfree(ctx->cancel_hash); +- kmem_cache_free(req_cachep, ctx->fallback_req); +- kfree(ctx); +-} +- +-static __poll_t io_uring_poll(struct file *file, poll_table *wait) +-{ +- struct io_ring_ctx *ctx = file->private_data; +- __poll_t mask = 0; +- +- poll_wait(file, &ctx->cq_wait, wait); +- /* +- * synchronizes with barrier from wq_has_sleeper call in +- * io_commit_cqring +- */ +- smp_rmb(); +- if (!io_sqring_full(ctx)) +- mask |= EPOLLOUT | EPOLLWRNORM; +- +- /* +- * Don't flush cqring overflow list here, just do a simple check. +- * Otherwise there could possible be ABBA deadlock: +- * CPU0 CPU1 +- * ---- ---- +- * lock(&ctx->uring_lock); +- * lock(&ep->mtx); +- * lock(&ctx->uring_lock); +- * lock(&ep->mtx); +- * +- * Users may get EPOLLIN meanwhile seeing nothing in cqring, this +- * pushs them to do the flush. +- */ +- if (io_cqring_events(ctx) || test_bit(0, &ctx->cq_check_overflow)) +- mask |= EPOLLIN | EPOLLRDNORM; +- +- return mask; +-} +- +-static int io_uring_fasync(int fd, struct file *file, int on) +-{ +- struct io_ring_ctx *ctx = file->private_data; +- +- return fasync_helper(fd, file, on, &ctx->cq_fasync); +-} +- +-static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) +-{ +- struct io_identity *iod; +- +- iod = xa_erase(&ctx->personalities, id); +- if (iod) { +- put_cred(iod->creds); +- if (refcount_dec_and_test(&iod->count)) +- kfree(iod); +- return 0; +- } +- +- return -EINVAL; +-} +- +-static void io_ring_exit_work(struct work_struct *work) +-{ +- struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, +- exit_work); +- +- /* +- * If we're doing polled IO and end up having requests being +- * submitted async (out-of-line), then completions can come in while +- * we're waiting for refs to drop. We need to reap these manually, +- * as nobody else will be looking for them. +- */ +- do { +- io_iopoll_try_reap_events(ctx); +- } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)); +- io_ring_ctx_free(ctx); +-} +- +-static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data) +-{ +- struct io_kiocb *req = container_of(work, struct io_kiocb, work); +- +- return req->ctx == data; +-} +- +-static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) +-{ +- unsigned long index; +- struct io_identify *iod; +- +- mutex_lock(&ctx->uring_lock); +- percpu_ref_kill(&ctx->refs); +- /* if force is set, the ring is going away. always drop after that */ +- +- if (WARN_ON_ONCE((ctx->flags & IORING_SETUP_SQPOLL) && !ctx->sqo_dead)) +- ctx->sqo_dead = 1; +- +- ctx->cq_overflow_flushed = 1; +- if (ctx->rings) +- __io_cqring_overflow_flush(ctx, true, NULL, NULL); +- mutex_unlock(&ctx->uring_lock); +- +- io_kill_timeouts(ctx, NULL, NULL); +- io_poll_remove_all(ctx, NULL, NULL); +- +- if (ctx->io_wq) +- io_wq_cancel_cb(ctx->io_wq, io_cancel_ctx_cb, ctx, true); +- +- /* if we failed setting up the ctx, we might not have any rings */ +- io_iopoll_try_reap_events(ctx); +- xa_for_each(&ctx->personalities, index, iod) +- io_unregister_personality(ctx, index); +- +- /* +- * Do this upfront, so we won't have a grace period where the ring +- * is closed but resources aren't reaped yet. This can cause +- * spurious failure in setting up a new ring. +- */ +- io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries), +- ACCT_LOCKED); +- +- INIT_WORK(&ctx->exit_work, io_ring_exit_work); +- /* +- * Use system_unbound_wq to avoid spawning tons of event kworkers +- * if we're exiting a ton of rings at the same time. It just adds +- * noise and overhead, there's no discernable change in runtime +- * over using system_wq. +- */ +- queue_work(system_unbound_wq, &ctx->exit_work); +-} +- +-static int io_uring_release(struct inode *inode, struct file *file) +-{ +- struct io_ring_ctx *ctx = file->private_data; +- +- file->private_data = NULL; +- io_ring_ctx_wait_and_kill(ctx); +- return 0; +-} +- +-struct io_task_cancel { +- struct task_struct *task; +- struct files_struct *files; +-}; +- +-static bool io_cancel_task_cb(struct io_wq_work *work, void *data) +-{ +- struct io_kiocb *req = container_of(work, struct io_kiocb, work); +- struct io_task_cancel *cancel = data; +- bool ret; +- +- if (cancel->files && (req->flags & REQ_F_LINK_TIMEOUT)) { +- unsigned long flags; +- struct io_ring_ctx *ctx = req->ctx; +- +- /* protect against races with linked timeouts */ +- spin_lock_irqsave(&ctx->completion_lock, flags); +- ret = io_match_task(req, cancel->task, cancel->files); +- spin_unlock_irqrestore(&ctx->completion_lock, flags); +- } else { +- ret = io_match_task(req, cancel->task, cancel->files); +- } +- return ret; +-} +- +-static void io_cancel_defer_files(struct io_ring_ctx *ctx, +- struct task_struct *task, +- struct files_struct *files) +-{ +- struct io_defer_entry *de = NULL; +- LIST_HEAD(list); +- +- spin_lock_irq(&ctx->completion_lock); +- list_for_each_entry_reverse(de, &ctx->defer_list, list) { +- if (io_match_task(de->req, task, files)) { +- list_cut_position(&list, &ctx->defer_list, &de->list); +- break; +- } +- } +- spin_unlock_irq(&ctx->completion_lock); +- +- while (!list_empty(&list)) { +- de = list_first_entry(&list, struct io_defer_entry, list); +- list_del_init(&de->list); +- req_set_fail_links(de->req); +- io_put_req(de->req); +- io_req_complete(de->req, -ECANCELED); +- kfree(de); +- } +-} +- +-static int io_uring_count_inflight(struct io_ring_ctx *ctx, +- struct task_struct *task, +- struct files_struct *files) +-{ +- struct io_kiocb *req; +- int cnt = 0; +- +- spin_lock_irq(&ctx->inflight_lock); +- list_for_each_entry(req, &ctx->inflight_list, inflight_entry) +- cnt += io_match_task(req, task, files); +- spin_unlock_irq(&ctx->inflight_lock); +- return cnt; +-} +- +-static void io_uring_cancel_files(struct io_ring_ctx *ctx, +- struct task_struct *task, +- struct files_struct *files) +-{ +- while (!list_empty_careful(&ctx->inflight_list)) { +- struct io_task_cancel cancel = { .task = task, .files = files }; +- DEFINE_WAIT(wait); +- int inflight; +- +- inflight = io_uring_count_inflight(ctx, task, files); +- if (!inflight) +- break; +- +- io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, &cancel, true); +- io_poll_remove_all(ctx, task, files); +- io_kill_timeouts(ctx, task, files); +- /* cancellations _may_ trigger task work */ +- io_run_task_work(); +- +- prepare_to_wait(&task->io_uring->wait, &wait, +- TASK_UNINTERRUPTIBLE); +- if (inflight == io_uring_count_inflight(ctx, task, files)) +- schedule(); +- finish_wait(&task->io_uring->wait, &wait); +- } +-} +- +-static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, +- struct task_struct *task) +-{ +- while (1) { +- struct io_task_cancel cancel = { .task = task, .files = NULL, }; +- enum io_wq_cancel cret; +- bool ret = false; +- +- cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, &cancel, true); +- if (cret != IO_WQ_CANCEL_NOTFOUND) +- ret = true; +- +- /* SQPOLL thread does its own polling */ +- if (!(ctx->flags & IORING_SETUP_SQPOLL)) { +- while (!list_empty_careful(&ctx->iopoll_list)) { +- io_iopoll_try_reap_events(ctx); +- ret = true; +- } +- } +- +- ret |= io_poll_remove_all(ctx, task, NULL); +- ret |= io_kill_timeouts(ctx, task, NULL); +- if (!ret) +- break; +- io_run_task_work(); +- cond_resched(); +- } +-} +- +-static void io_disable_sqo_submit(struct io_ring_ctx *ctx) +-{ +- mutex_lock(&ctx->uring_lock); +- ctx->sqo_dead = 1; +- if (ctx->flags & IORING_SETUP_R_DISABLED) +- io_sq_offload_start(ctx); +- mutex_unlock(&ctx->uring_lock); +- +- /* make sure callers enter the ring to get error */ +- if (ctx->rings) +- io_ring_set_wakeup_flag(ctx); +-} +- +-/* +- * We need to iteratively cancel requests, in case a request has dependent +- * hard links. These persist even for failure of cancelations, hence keep +- * looping until none are found. +- */ +-static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, +- struct files_struct *files) +-{ +- struct task_struct *task = current; +- +- if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { +- io_disable_sqo_submit(ctx); +- task = ctx->sq_data->thread; +- atomic_inc(&task->io_uring->in_idle); +- io_sq_thread_park(ctx->sq_data); +- } +- +- io_cancel_defer_files(ctx, task, files); +- io_cqring_overflow_flush(ctx, true, task, files); +- +- if (!files) +- __io_uring_cancel_task_requests(ctx, task); +- else +- io_uring_cancel_files(ctx, task, files); +- +- if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { +- atomic_dec(&task->io_uring->in_idle); +- io_sq_thread_unpark(ctx->sq_data); +- } +-} +- +-/* +- * Note that this task has used io_uring. We use it for cancelation purposes. +- */ +-static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) +-{ +- struct io_uring_task *tctx = current->io_uring; +- int ret; +- +- if (unlikely(!tctx)) { +- ret = io_uring_alloc_task_context(current); +- if (unlikely(ret)) +- return ret; +- tctx = current->io_uring; +- } +- if (tctx->last != file) { +- void *old = xa_load(&tctx->xa, (unsigned long)file); +- +- if (!old) { +- get_file(file); +- ret = xa_err(xa_store(&tctx->xa, (unsigned long)file, +- file, GFP_KERNEL)); +- if (ret) { +- fput(file); +- return ret; +- } +- } +- tctx->last = file; +- } +- +- /* +- * This is race safe in that the task itself is doing this, hence it +- * cannot be going through the exit/cancel paths at the same time. +- * This cannot be modified while exit/cancel is running. +- */ +- if (!tctx->sqpoll && (ctx->flags & IORING_SETUP_SQPOLL)) +- tctx->sqpoll = true; +- +- return 0; +-} +- +-/* +- * Remove this io_uring_file -> task mapping. +- */ +-static void io_uring_del_task_file(struct file *file) +-{ +- struct io_uring_task *tctx = current->io_uring; +- +- if (tctx->last == file) +- tctx->last = NULL; +- file = xa_erase(&tctx->xa, (unsigned long)file); +- if (file) +- fput(file); +-} +- +-static void io_uring_remove_task_files(struct io_uring_task *tctx) +-{ +- struct file *file; +- unsigned long index; +- +- xa_for_each(&tctx->xa, index, file) +- io_uring_del_task_file(file); +-} +- +-void __io_uring_files_cancel(struct files_struct *files) +-{ +- struct io_uring_task *tctx = current->io_uring; +- struct file *file; +- unsigned long index; +- +- /* make sure overflow events are dropped */ +- atomic_inc(&tctx->in_idle); +- xa_for_each(&tctx->xa, index, file) +- io_uring_cancel_task_requests(file->private_data, files); +- atomic_dec(&tctx->in_idle); +- +- if (files) +- io_uring_remove_task_files(tctx); +-} +- +-static s64 tctx_inflight(struct io_uring_task *tctx) +-{ +- unsigned long index; +- struct file *file; +- s64 inflight; +- +- inflight = percpu_counter_sum(&tctx->inflight); +- if (!tctx->sqpoll) +- return inflight; +- +- /* +- * If we have SQPOLL rings, then we need to iterate and find them, and +- * add the pending count for those. +- */ +- xa_for_each(&tctx->xa, index, file) { +- struct io_ring_ctx *ctx = file->private_data; +- +- if (ctx->flags & IORING_SETUP_SQPOLL) { +- struct io_uring_task *__tctx = ctx->sqo_task->io_uring; +- +- inflight += percpu_counter_sum(&__tctx->inflight); +- } +- } +- +- return inflight; +-} +- +-/* +- * Find any io_uring fd that this task has registered or done IO on, and cancel +- * requests. +- */ +-void __io_uring_task_cancel(void) +-{ +- struct io_uring_task *tctx = current->io_uring; +- DEFINE_WAIT(wait); +- s64 inflight; +- +- /* make sure overflow events are dropped */ +- atomic_inc(&tctx->in_idle); +- +- /* trigger io_disable_sqo_submit() */ +- if (tctx->sqpoll) +- __io_uring_files_cancel(NULL); +- +- do { +- /* read completions before cancelations */ +- inflight = tctx_inflight(tctx); +- if (!inflight) +- break; +- __io_uring_files_cancel(NULL); +- +- prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE); +- +- /* +- * If we've seen completions, retry without waiting. This +- * avoids a race where a completion comes in before we did +- * prepare_to_wait(). +- */ +- if (inflight == tctx_inflight(tctx)) +- schedule(); +- finish_wait(&tctx->wait, &wait); +- } while (1); +- +- atomic_dec(&tctx->in_idle); +- +- io_uring_remove_task_files(tctx); +-} +- +-static int io_uring_flush(struct file *file, void *data) +-{ +- struct io_uring_task *tctx = current->io_uring; +- struct io_ring_ctx *ctx = file->private_data; +- +- if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) +- io_uring_cancel_task_requests(ctx, NULL); +- +- if (!tctx) +- return 0; +- +- /* we should have cancelled and erased it before PF_EXITING */ +- WARN_ON_ONCE((current->flags & PF_EXITING) && +- xa_load(&tctx->xa, (unsigned long)file)); +- +- /* +- * fput() is pending, will be 2 if the only other ref is our potential +- * task file note. If the task is exiting, drop regardless of count. +- */ +- if (atomic_long_read(&file->f_count) != 2) +- return 0; +- +- if (ctx->flags & IORING_SETUP_SQPOLL) { +- /* there is only one file note, which is owned by sqo_task */ +- WARN_ON_ONCE(ctx->sqo_task != current && +- xa_load(&tctx->xa, (unsigned long)file)); +- /* sqo_dead check is for when this happens after cancellation */ +- WARN_ON_ONCE(ctx->sqo_task == current && !ctx->sqo_dead && +- !xa_load(&tctx->xa, (unsigned long)file)); +- +- io_disable_sqo_submit(ctx); +- } +- +- if (!(ctx->flags & IORING_SETUP_SQPOLL) || ctx->sqo_task == current) +- io_uring_del_task_file(file); +- return 0; +-} +- +-static void *io_uring_validate_mmap_request(struct file *file, +- loff_t pgoff, size_t sz) +-{ +- struct io_ring_ctx *ctx = file->private_data; +- loff_t offset = pgoff << PAGE_SHIFT; +- struct page *page; +- void *ptr; +- +- switch (offset) { +- case IORING_OFF_SQ_RING: +- case IORING_OFF_CQ_RING: +- ptr = ctx->rings; +- break; +- case IORING_OFF_SQES: +- ptr = ctx->sq_sqes; +- break; +- default: +- return ERR_PTR(-EINVAL); +- } +- +- page = virt_to_head_page(ptr); +- if (sz > page_size(page)) +- return ERR_PTR(-EINVAL); +- +- return ptr; +-} +- +-#ifdef CONFIG_MMU +- +-static int io_uring_mmap(struct file *file, struct vm_area_struct *vma) +-{ +- size_t sz = vma->vm_end - vma->vm_start; +- unsigned long pfn; +- void *ptr; +- +- ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz); +- if (IS_ERR(ptr)) +- return PTR_ERR(ptr); +- +- pfn = virt_to_phys(ptr) >> PAGE_SHIFT; +- return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot); +-} +- +-#else /* !CONFIG_MMU */ +- +-static int io_uring_mmap(struct file *file, struct vm_area_struct *vma) +-{ +- return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL; +-} +- +-static unsigned int io_uring_nommu_mmap_capabilities(struct file *file) +-{ +- return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE; +-} +- +-static unsigned long io_uring_nommu_get_unmapped_area(struct file *file, +- unsigned long addr, unsigned long len, +- unsigned long pgoff, unsigned long flags) +-{ +- void *ptr; +- +- ptr = io_uring_validate_mmap_request(file, pgoff, len); +- if (IS_ERR(ptr)) +- return PTR_ERR(ptr); +- +- return (unsigned long) ptr; +-} +- +-#endif /* !CONFIG_MMU */ +- +-static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx) +-{ +- int ret = 0; +- DEFINE_WAIT(wait); +- +- do { +- if (!io_sqring_full(ctx)) +- break; +- +- prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE); +- +- if (unlikely(ctx->sqo_dead)) { +- ret = -EOWNERDEAD; +- break; +- } +- +- if (!io_sqring_full(ctx)) +- break; +- +- schedule(); +- } while (!signal_pending(current)); +- +- finish_wait(&ctx->sqo_sq_wait, &wait); +- return ret; +-} +- +-SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, +- u32, min_complete, u32, flags, const sigset_t __user *, sig, +- size_t, sigsz) +-{ +- struct io_ring_ctx *ctx; +- long ret = -EBADF; +- int submitted = 0; +- struct fd f; +- +- io_run_task_work(); +- +- if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP | +- IORING_ENTER_SQ_WAIT)) +- return -EINVAL; +- +- f = fdget(fd); +- if (!f.file) +- return -EBADF; +- +- ret = -EOPNOTSUPP; +- if (f.file->f_op != &io_uring_fops) +- goto out_fput; +- +- ret = -ENXIO; +- ctx = f.file->private_data; +- if (!percpu_ref_tryget(&ctx->refs)) +- goto out_fput; +- +- ret = -EBADFD; +- if (ctx->flags & IORING_SETUP_R_DISABLED) +- goto out; +- +- /* +- * For SQ polling, the thread will do all submissions and completions. +- * Just return the requested submit count, and wake the thread if +- * we were asked to. +- */ +- ret = 0; +- if (ctx->flags & IORING_SETUP_SQPOLL) { +- io_cqring_overflow_flush(ctx, false, NULL, NULL); +- +- if (unlikely(ctx->sqo_dead)) { +- ret = -EOWNERDEAD; +- goto out; +- } +- if (flags & IORING_ENTER_SQ_WAKEUP) +- wake_up(&ctx->sq_data->wait); +- if (flags & IORING_ENTER_SQ_WAIT) { +- ret = io_sqpoll_wait_sq(ctx); +- if (ret) +- goto out; +- } +- submitted = to_submit; +- } else if (to_submit) { +- ret = io_uring_add_task_file(ctx, f.file); +- if (unlikely(ret)) +- goto out; +- mutex_lock(&ctx->uring_lock); +- submitted = io_submit_sqes(ctx, to_submit); +- mutex_unlock(&ctx->uring_lock); +- +- if (submitted != to_submit) +- goto out; +- } +- if (flags & IORING_ENTER_GETEVENTS) { +- min_complete = min(min_complete, ctx->cq_entries); +- +- /* +- * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user +- * space applications don't need to do io completion events +- * polling again, they can rely on io_sq_thread to do polling +- * work, which can reduce cpu usage and uring_lock contention. +- */ +- if (ctx->flags & IORING_SETUP_IOPOLL && +- !(ctx->flags & IORING_SETUP_SQPOLL)) { +- ret = io_iopoll_check(ctx, min_complete); +- } else { +- ret = io_cqring_wait(ctx, min_complete, sig, sigsz); +- } +- } +- +-out: +- percpu_ref_put(&ctx->refs); +-out_fput: +- fdput(f); +- return submitted ? submitted : ret; +-} +- +-#ifdef CONFIG_PROC_FS +-static int io_uring_show_cred(struct seq_file *m, unsigned int id, +- const struct io_identity *iod) +-{ +- const struct cred *cred = iod->creds; +- struct user_namespace *uns = seq_user_ns(m); +- struct group_info *gi; +- kernel_cap_t cap; +- unsigned __capi; +- int g; +- +- seq_printf(m, "%5d\n", id); +- seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid)); +- seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid)); +- seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid)); +- seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid)); +- seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid)); +- seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid)); +- seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid)); +- seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid)); +- seq_puts(m, "\n\tGroups:\t"); +- gi = cred->group_info; +- for (g = 0; g < gi->ngroups; g++) { +- seq_put_decimal_ull(m, g ? " " : "", +- from_kgid_munged(uns, gi->gid[g])); +- } +- seq_puts(m, "\n\tCapEff:\t"); +- cap = cred->cap_effective; +- CAP_FOR_EACH_U32(__capi) +- seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8); +- seq_putc(m, '\n'); +- return 0; +-} +- +-static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) +-{ +- struct io_sq_data *sq = NULL; +- bool has_lock; +- int i; +- +- /* +- * Avoid ABBA deadlock between the seq lock and the io_uring mutex, +- * since fdinfo case grabs it in the opposite direction of normal use +- * cases. If we fail to get the lock, we just don't iterate any +- * structures that could be going away outside the io_uring mutex. +- */ +- has_lock = mutex_trylock(&ctx->uring_lock); +- +- if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) +- sq = ctx->sq_data; +- +- seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1); +- seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1); +- seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files); +- for (i = 0; has_lock && i < ctx->nr_user_files; i++) { +- struct fixed_file_table *table; +- struct file *f; +- +- table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; +- f = table->files[i & IORING_FILE_TABLE_MASK]; +- if (f) +- seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname); +- else +- seq_printf(m, "%5u: \n", i); +- } +- seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs); +- for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) { +- struct io_mapped_ubuf *buf = &ctx->user_bufs[i]; +- +- seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, +- (unsigned int) buf->len); +- } +- if (has_lock && !xa_empty(&ctx->personalities)) { +- unsigned long index; +- const struct io_identity *iod; +- +- seq_printf(m, "Personalities:\n"); +- xa_for_each(&ctx->personalities, index, iod) +- io_uring_show_cred(m, index, iod); +- } +- seq_printf(m, "PollList:\n"); +- spin_lock_irq(&ctx->completion_lock); +- for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { +- struct hlist_head *list = &ctx->cancel_hash[i]; +- struct io_kiocb *req; +- +- hlist_for_each_entry(req, list, hash_node) +- seq_printf(m, " op=%d, task_works=%d\n", req->opcode, +- req->task->task_works != NULL); +- } +- spin_unlock_irq(&ctx->completion_lock); +- if (has_lock) +- mutex_unlock(&ctx->uring_lock); +-} +- +-static void io_uring_show_fdinfo(struct seq_file *m, struct file *f) +-{ +- struct io_ring_ctx *ctx = f->private_data; +- +- if (percpu_ref_tryget(&ctx->refs)) { +- __io_uring_show_fdinfo(ctx, m); +- percpu_ref_put(&ctx->refs); +- } +-} +-#endif +- +-static const struct file_operations io_uring_fops = { +- .release = io_uring_release, +- .flush = io_uring_flush, +- .mmap = io_uring_mmap, +-#ifndef CONFIG_MMU +- .get_unmapped_area = io_uring_nommu_get_unmapped_area, +- .mmap_capabilities = io_uring_nommu_mmap_capabilities, +-#endif +- .poll = io_uring_poll, +- .fasync = io_uring_fasync, +-#ifdef CONFIG_PROC_FS +- .show_fdinfo = io_uring_show_fdinfo, +-#endif +-}; +- +-static int io_allocate_scq_urings(struct io_ring_ctx *ctx, +- struct io_uring_params *p) +-{ +- struct io_rings *rings; +- size_t size, sq_array_offset; +- +- /* make sure these are sane, as we already accounted them */ +- ctx->sq_entries = p->sq_entries; +- ctx->cq_entries = p->cq_entries; +- +- size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset); +- if (size == SIZE_MAX) +- return -EOVERFLOW; +- +- rings = io_mem_alloc(size); +- if (!rings) +- return -ENOMEM; +- +- ctx->rings = rings; +- ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); +- rings->sq_ring_mask = p->sq_entries - 1; +- rings->cq_ring_mask = p->cq_entries - 1; +- rings->sq_ring_entries = p->sq_entries; +- rings->cq_ring_entries = p->cq_entries; +- ctx->sq_mask = rings->sq_ring_mask; +- ctx->cq_mask = rings->cq_ring_mask; +- +- size = array_size(sizeof(struct io_uring_sqe), p->sq_entries); +- if (size == SIZE_MAX) { +- io_mem_free(ctx->rings); +- ctx->rings = NULL; +- return -EOVERFLOW; +- } +- +- ctx->sq_sqes = io_mem_alloc(size); +- if (!ctx->sq_sqes) { +- io_mem_free(ctx->rings); +- ctx->rings = NULL; +- return -ENOMEM; +- } +- +- return 0; +-} +- +-static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file) +-{ +- int ret, fd; +- +- fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); +- if (fd < 0) +- return fd; +- +- ret = io_uring_add_task_file(ctx, file); +- if (ret) { +- put_unused_fd(fd); +- return ret; +- } +- fd_install(fd, file); +- return fd; +-} +- +-/* +- * Allocate an anonymous fd, this is what constitutes the application +- * visible backing of an io_uring instance. The application mmaps this +- * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled, +- * we have to tie this fd to a socket for file garbage collection purposes. +- */ +-static struct file *io_uring_get_file(struct io_ring_ctx *ctx) +-{ +- struct file *file; +-#if defined(CONFIG_UNIX) +- int ret; +- +- ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP, +- &ctx->ring_sock); +- if (ret) +- return ERR_PTR(ret); +-#endif +- +- file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx, +- O_RDWR | O_CLOEXEC); +-#if defined(CONFIG_UNIX) +- if (IS_ERR(file)) { +- sock_release(ctx->ring_sock); +- ctx->ring_sock = NULL; +- } else { +- ctx->ring_sock->file = file; +- } +-#endif +- return file; +-} +- +-static int io_uring_create(unsigned entries, struct io_uring_params *p, +- struct io_uring_params __user *params) +-{ +- struct user_struct *user = NULL; +- struct io_ring_ctx *ctx; +- struct file *file; +- bool limit_mem; +- int ret; +- +- if (!entries) +- return -EINVAL; +- if (entries > IORING_MAX_ENTRIES) { +- if (!(p->flags & IORING_SETUP_CLAMP)) +- return -EINVAL; +- entries = IORING_MAX_ENTRIES; +- } +- +- /* +- * Use twice as many entries for the CQ ring. It's possible for the +- * application to drive a higher depth than the size of the SQ ring, +- * since the sqes are only used at submission time. This allows for +- * some flexibility in overcommitting a bit. If the application has +- * set IORING_SETUP_CQSIZE, it will have passed in the desired number +- * of CQ ring entries manually. +- */ +- p->sq_entries = roundup_pow_of_two(entries); +- if (p->flags & IORING_SETUP_CQSIZE) { +- /* +- * If IORING_SETUP_CQSIZE is set, we do the same roundup +- * to a power-of-two, if it isn't already. We do NOT impose +- * any cq vs sq ring sizing. +- */ +- if (!p->cq_entries) +- return -EINVAL; +- if (p->cq_entries > IORING_MAX_CQ_ENTRIES) { +- if (!(p->flags & IORING_SETUP_CLAMP)) +- return -EINVAL; +- p->cq_entries = IORING_MAX_CQ_ENTRIES; +- } +- p->cq_entries = roundup_pow_of_two(p->cq_entries); +- if (p->cq_entries < p->sq_entries) +- return -EINVAL; +- } else { +- p->cq_entries = 2 * p->sq_entries; +- } +- +- user = get_uid(current_user()); +- limit_mem = !capable(CAP_IPC_LOCK); +- +- if (limit_mem) { +- ret = __io_account_mem(user, +- ring_pages(p->sq_entries, p->cq_entries)); +- if (ret) { +- free_uid(user); +- return ret; +- } +- } +- +- ctx = io_ring_ctx_alloc(p); +- if (!ctx) { +- if (limit_mem) +- __io_unaccount_mem(user, ring_pages(p->sq_entries, +- p->cq_entries)); +- free_uid(user); +- return -ENOMEM; +- } +- ctx->compat = in_compat_syscall(); +- ctx->user = user; +- ctx->creds = get_current_cred(); +-#ifdef CONFIG_AUDIT +- ctx->loginuid = current->loginuid; +- ctx->sessionid = current->sessionid; +-#endif +- ctx->sqo_task = get_task_struct(current); +- +- /* +- * This is just grabbed for accounting purposes. When a process exits, +- * the mm is exited and dropped before the files, hence we need to hang +- * on to this mm purely for the purposes of being able to unaccount +- * memory (locked/pinned vm). It's not used for anything else. +- */ +- mmgrab(current->mm); +- ctx->mm_account = current->mm; +- +-#ifdef CONFIG_BLK_CGROUP +- /* +- * The sq thread will belong to the original cgroup it was inited in. +- * If the cgroup goes offline (e.g. disabling the io controller), then +- * issued bios will be associated with the closest cgroup later in the +- * block layer. +- */ +- rcu_read_lock(); +- ctx->sqo_blkcg_css = blkcg_css(); +- ret = css_tryget_online(ctx->sqo_blkcg_css); +- rcu_read_unlock(); +- if (!ret) { +- /* don't init against a dying cgroup, have the user try again */ +- ctx->sqo_blkcg_css = NULL; +- ret = -ENODEV; +- goto err; +- } +-#endif +- +- /* +- * Account memory _before_ installing the file descriptor. Once +- * the descriptor is installed, it can get closed at any time. Also +- * do this before hitting the general error path, as ring freeing +- * will un-account as well. +- */ +- io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries), +- ACCT_LOCKED); +- ctx->limit_mem = limit_mem; +- +- ret = io_allocate_scq_urings(ctx, p); +- if (ret) +- goto err; +- +- ret = io_sq_offload_create(ctx, p); +- if (ret) +- goto err; +- +- if (!(p->flags & IORING_SETUP_R_DISABLED)) +- io_sq_offload_start(ctx); +- +- memset(&p->sq_off, 0, sizeof(p->sq_off)); +- p->sq_off.head = offsetof(struct io_rings, sq.head); +- p->sq_off.tail = offsetof(struct io_rings, sq.tail); +- p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask); +- p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries); +- p->sq_off.flags = offsetof(struct io_rings, sq_flags); +- p->sq_off.dropped = offsetof(struct io_rings, sq_dropped); +- p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; +- +- memset(&p->cq_off, 0, sizeof(p->cq_off)); +- p->cq_off.head = offsetof(struct io_rings, cq.head); +- p->cq_off.tail = offsetof(struct io_rings, cq.tail); +- p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask); +- p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries); +- p->cq_off.overflow = offsetof(struct io_rings, cq_overflow); +- p->cq_off.cqes = offsetof(struct io_rings, cqes); +- p->cq_off.flags = offsetof(struct io_rings, cq_flags); +- +- p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP | +- IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS | +- IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL | +- IORING_FEAT_POLL_32BITS; +- +- if (copy_to_user(params, p, sizeof(*p))) { +- ret = -EFAULT; +- goto err; +- } +- +- file = io_uring_get_file(ctx); +- if (IS_ERR(file)) { +- ret = PTR_ERR(file); +- goto err; +- } +- +- /* +- * Install ring fd as the very last thing, so we don't risk someone +- * having closed it before we finish setup +- */ +- ret = io_uring_install_fd(ctx, file); +- if (ret < 0) { +- io_disable_sqo_submit(ctx); +- /* fput will clean it up */ +- fput(file); +- return ret; +- } +- +- trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags); +- return ret; +-err: +- io_disable_sqo_submit(ctx); +- io_ring_ctx_wait_and_kill(ctx); +- return ret; +-} +- +-/* +- * Sets up an aio uring context, and returns the fd. Applications asks for a +- * ring size, we return the actual sq/cq ring sizes (among other things) in the +- * params structure passed in. +- */ +-static long io_uring_setup(u32 entries, struct io_uring_params __user *params) +-{ +- struct io_uring_params p; +- int i; +- +- if (copy_from_user(&p, params, sizeof(p))) +- return -EFAULT; +- for (i = 0; i < ARRAY_SIZE(p.resv); i++) { +- if (p.resv[i]) +- return -EINVAL; +- } +- +- if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL | +- IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE | +- IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ | +- IORING_SETUP_R_DISABLED)) +- return -EINVAL; +- +- return io_uring_create(entries, &p, params); +-} +- +-SYSCALL_DEFINE2(io_uring_setup, u32, entries, +- struct io_uring_params __user *, params) +-{ +- return io_uring_setup(entries, params); +-} +- +-static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) +-{ +- struct io_uring_probe *p; +- size_t size; +- int i, ret; +- +- size = struct_size(p, ops, nr_args); +- if (size == SIZE_MAX) +- return -EOVERFLOW; +- p = kzalloc(size, GFP_KERNEL); +- if (!p) +- return -ENOMEM; +- +- ret = -EFAULT; +- if (copy_from_user(p, arg, size)) +- goto out; +- ret = -EINVAL; +- if (memchr_inv(p, 0, size)) +- goto out; +- +- p->last_op = IORING_OP_LAST - 1; +- if (nr_args > IORING_OP_LAST) +- nr_args = IORING_OP_LAST; +- +- for (i = 0; i < nr_args; i++) { +- p->ops[i].op = i; +- if (!io_op_defs[i].not_supported) +- p->ops[i].flags = IO_URING_OP_SUPPORTED; +- } +- p->ops_len = i; +- +- ret = 0; +- if (copy_to_user(arg, p, size)) +- ret = -EFAULT; +-out: +- kfree(p); +- return ret; +-} +- +-static int io_register_personality(struct io_ring_ctx *ctx) +-{ +- struct io_identity *iod; +- u32 id; +- int ret; +- +- iod = kmalloc(sizeof(*iod), GFP_KERNEL); +- if (unlikely(!iod)) +- return -ENOMEM; +- +- io_init_identity(iod); +- iod->creds = get_current_cred(); +- +- ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)iod, +- XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL); +- if (ret < 0) { +- put_cred(iod->creds); +- kfree(iod); +- return ret; +- } +- return id; +-} +- +-static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg, +- unsigned int nr_args) +-{ +- struct io_uring_restriction *res; +- size_t size; +- int i, ret; +- +- /* Restrictions allowed only if rings started disabled */ +- if (!(ctx->flags & IORING_SETUP_R_DISABLED)) +- return -EBADFD; +- +- /* We allow only a single restrictions registration */ +- if (ctx->restrictions.registered) +- return -EBUSY; +- +- if (!arg || nr_args > IORING_MAX_RESTRICTIONS) +- return -EINVAL; +- +- size = array_size(nr_args, sizeof(*res)); +- if (size == SIZE_MAX) +- return -EOVERFLOW; +- +- res = memdup_user(arg, size); +- if (IS_ERR(res)) +- return PTR_ERR(res); +- +- ret = 0; +- +- for (i = 0; i < nr_args; i++) { +- switch (res[i].opcode) { +- case IORING_RESTRICTION_REGISTER_OP: +- if (res[i].register_op >= IORING_REGISTER_LAST) { +- ret = -EINVAL; +- goto out; +- } +- +- __set_bit(res[i].register_op, +- ctx->restrictions.register_op); +- break; +- case IORING_RESTRICTION_SQE_OP: +- if (res[i].sqe_op >= IORING_OP_LAST) { +- ret = -EINVAL; +- goto out; +- } +- +- __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op); +- break; +- case IORING_RESTRICTION_SQE_FLAGS_ALLOWED: +- ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags; +- break; +- case IORING_RESTRICTION_SQE_FLAGS_REQUIRED: +- ctx->restrictions.sqe_flags_required = res[i].sqe_flags; +- break; +- default: +- ret = -EINVAL; +- goto out; +- } +- } +- +-out: +- /* Reset all restrictions if an error happened */ +- if (ret != 0) +- memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); +- else +- ctx->restrictions.registered = true; +- +- kfree(res); +- return ret; +-} +- +-static int io_register_enable_rings(struct io_ring_ctx *ctx) +-{ +- if (!(ctx->flags & IORING_SETUP_R_DISABLED)) +- return -EBADFD; +- +- if (ctx->restrictions.registered) +- ctx->restricted = 1; +- +- io_sq_offload_start(ctx); +- return 0; +-} +- +-static bool io_register_op_must_quiesce(int op) +-{ +- switch (op) { +- case IORING_UNREGISTER_FILES: +- case IORING_REGISTER_FILES_UPDATE: +- case IORING_REGISTER_PROBE: +- case IORING_REGISTER_PERSONALITY: +- case IORING_UNREGISTER_PERSONALITY: +- return false; +- default: +- return true; +- } +-} +- +-static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, +- void __user *arg, unsigned nr_args) +- __releases(ctx->uring_lock) +- __acquires(ctx->uring_lock) +-{ +- int ret; +- +- /* +- * We're inside the ring mutex, if the ref is already dying, then +- * someone else killed the ctx or is already going through +- * io_uring_register(). +- */ +- if (percpu_ref_is_dying(&ctx->refs)) +- return -ENXIO; +- +- if (io_register_op_must_quiesce(opcode)) { +- percpu_ref_kill(&ctx->refs); +- +- /* +- * Drop uring mutex before waiting for references to exit. If +- * another thread is currently inside io_uring_enter() it might +- * need to grab the uring_lock to make progress. If we hold it +- * here across the drain wait, then we can deadlock. It's safe +- * to drop the mutex here, since no new references will come in +- * after we've killed the percpu ref. +- */ +- mutex_unlock(&ctx->uring_lock); +- do { +- ret = wait_for_completion_interruptible(&ctx->ref_comp); +- if (!ret) +- break; +- ret = io_run_task_work_sig(); +- if (ret < 0) +- break; +- } while (1); +- mutex_lock(&ctx->uring_lock); +- +- if (ret) { +- io_refs_resurrect(&ctx->refs, &ctx->ref_comp); +- return ret; +- } +- } +- +- if (ctx->restricted) { +- if (opcode >= IORING_REGISTER_LAST) { +- ret = -EINVAL; +- goto out; +- } +- +- if (!test_bit(opcode, ctx->restrictions.register_op)) { +- ret = -EACCES; +- goto out; +- } +- } +- +- switch (opcode) { +- case IORING_REGISTER_BUFFERS: +- ret = io_sqe_buffer_register(ctx, arg, nr_args); +- break; +- case IORING_UNREGISTER_BUFFERS: +- ret = -EINVAL; +- if (arg || nr_args) +- break; +- ret = io_sqe_buffer_unregister(ctx); +- break; +- case IORING_REGISTER_FILES: +- ret = io_sqe_files_register(ctx, arg, nr_args); +- break; +- case IORING_UNREGISTER_FILES: +- ret = -EINVAL; +- if (arg || nr_args) +- break; +- ret = io_sqe_files_unregister(ctx); +- break; +- case IORING_REGISTER_FILES_UPDATE: +- ret = io_sqe_files_update(ctx, arg, nr_args); +- break; +- case IORING_REGISTER_EVENTFD: +- case IORING_REGISTER_EVENTFD_ASYNC: +- ret = -EINVAL; +- if (nr_args != 1) +- break; +- ret = io_eventfd_register(ctx, arg); +- if (ret) +- break; +- if (opcode == IORING_REGISTER_EVENTFD_ASYNC) +- ctx->eventfd_async = 1; +- else +- ctx->eventfd_async = 0; +- break; +- case IORING_UNREGISTER_EVENTFD: +- ret = -EINVAL; +- if (arg || nr_args) +- break; +- ret = io_eventfd_unregister(ctx); +- break; +- case IORING_REGISTER_PROBE: +- ret = -EINVAL; +- if (!arg || nr_args > 256) +- break; +- ret = io_probe(ctx, arg, nr_args); +- break; +- case IORING_REGISTER_PERSONALITY: +- ret = -EINVAL; +- if (arg || nr_args) +- break; +- ret = io_register_personality(ctx); +- break; +- case IORING_UNREGISTER_PERSONALITY: +- ret = -EINVAL; +- if (arg) +- break; +- ret = io_unregister_personality(ctx, nr_args); +- break; +- case IORING_REGISTER_ENABLE_RINGS: +- ret = -EINVAL; +- if (arg || nr_args) +- break; +- ret = io_register_enable_rings(ctx); +- break; +- case IORING_REGISTER_RESTRICTIONS: +- ret = io_register_restrictions(ctx, arg, nr_args); +- break; +- default: +- ret = -EINVAL; +- break; +- } +- +-out: +- if (io_register_op_must_quiesce(opcode)) { +- /* bring the ctx back to life */ +- percpu_ref_reinit(&ctx->refs); +- reinit_completion(&ctx->ref_comp); +- } +- return ret; +-} +- +-SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, +- void __user *, arg, unsigned int, nr_args) +-{ +- struct io_ring_ctx *ctx; +- long ret = -EBADF; +- struct fd f; +- +- f = fdget(fd); +- if (!f.file) +- return -EBADF; +- +- ret = -EOPNOTSUPP; +- if (f.file->f_op != &io_uring_fops) +- goto out_fput; +- +- ctx = f.file->private_data; +- +- mutex_lock(&ctx->uring_lock); +- ret = __io_uring_register(ctx, opcode, arg, nr_args); +- mutex_unlock(&ctx->uring_lock); +- trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, +- ctx->cq_ev_fd != NULL, ret); +-out_fput: +- fdput(f); +- return ret; +-} +- +-static int __init io_uring_init(void) +-{ +-#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \ +- BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \ +- BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \ +-} while (0) +- +-#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \ +- __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename) +- BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64); +- BUILD_BUG_SQE_ELEM(0, __u8, opcode); +- BUILD_BUG_SQE_ELEM(1, __u8, flags); +- BUILD_BUG_SQE_ELEM(2, __u16, ioprio); +- BUILD_BUG_SQE_ELEM(4, __s32, fd); +- BUILD_BUG_SQE_ELEM(8, __u64, off); +- BUILD_BUG_SQE_ELEM(8, __u64, addr2); +- BUILD_BUG_SQE_ELEM(16, __u64, addr); +- BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in); +- BUILD_BUG_SQE_ELEM(24, __u32, len); +- BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags); +- BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags); +- BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags); +- BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags); +- BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events); +- BUILD_BUG_SQE_ELEM(28, __u32, poll32_events); +- BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags); +- BUILD_BUG_SQE_ELEM(28, __u32, msg_flags); +- BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags); +- BUILD_BUG_SQE_ELEM(28, __u32, accept_flags); +- BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags); +- BUILD_BUG_SQE_ELEM(28, __u32, open_flags); +- BUILD_BUG_SQE_ELEM(28, __u32, statx_flags); +- BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice); +- BUILD_BUG_SQE_ELEM(28, __u32, splice_flags); +- BUILD_BUG_SQE_ELEM(32, __u64, user_data); +- BUILD_BUG_SQE_ELEM(40, __u16, buf_index); +- BUILD_BUG_SQE_ELEM(42, __u16, personality); +- BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in); +- +- BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); +- BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int)); +- req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC); +- return 0; +-}; +-__initcall(io_uring_init); +--- a/include/linux/io_uring.h ++++ b/include/linux/io_uring.h +@@ -5,50 +5,20 @@ + #include + #include + +-struct io_identity { +- struct files_struct *files; +- struct mm_struct *mm; +-#ifdef CONFIG_BLK_CGROUP +- struct cgroup_subsys_state *blkcg_css; +-#endif +- const struct cred *creds; +- struct nsproxy *nsproxy; +- struct fs_struct *fs; +- unsigned long fsize; +-#ifdef CONFIG_AUDIT +- kuid_t loginuid; +- unsigned int sessionid; +-#endif +- refcount_t count; +-}; +- +-struct io_uring_task { +- /* submission side */ +- struct xarray xa; +- struct wait_queue_head wait; +- struct file *last; +- struct percpu_counter inflight; +- struct io_identity __identity; +- struct io_identity *identity; +- atomic_t in_idle; +- bool sqpoll; +-}; +- + #if defined(CONFIG_IO_URING) + struct sock *io_uring_get_socket(struct file *file); +-void __io_uring_task_cancel(void); +-void __io_uring_files_cancel(struct files_struct *files); ++void __io_uring_cancel(bool cancel_all); + void __io_uring_free(struct task_struct *tsk); + +-static inline void io_uring_task_cancel(void) ++static inline void io_uring_files_cancel(void) + { +- if (current->io_uring && !xa_empty(¤t->io_uring->xa)) +- __io_uring_task_cancel(); ++ if (current->io_uring) ++ __io_uring_cancel(false); + } +-static inline void io_uring_files_cancel(struct files_struct *files) ++static inline void io_uring_task_cancel(void) + { +- if (current->io_uring && !xa_empty(¤t->io_uring->xa)) +- __io_uring_files_cancel(files); ++ if (current->io_uring) ++ __io_uring_cancel(true); + } + static inline void io_uring_free(struct task_struct *tsk) + { +@@ -63,7 +33,7 @@ static inline struct sock *io_uring_get_ + static inline void io_uring_task_cancel(void) + { + } +-static inline void io_uring_files_cancel(struct files_struct *files) ++static inline void io_uring_files_cancel(void) + { + } + static inline void io_uring_free(struct task_struct *tsk) +--- a/include/linux/sched.h ++++ b/include/linux/sched.h +@@ -885,6 +885,9 @@ struct task_struct { + /* CLONE_CHILD_CLEARTID: */ + int __user *clear_child_tid; + ++ /* PF_IO_WORKER */ ++ void *pf_io_worker; ++ + u64 utime; + u64 stime; + #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME +--- a/include/linux/syscalls.h ++++ b/include/linux/syscalls.h +@@ -341,7 +341,7 @@ asmlinkage long sys_io_uring_setup(u32 e + struct io_uring_params __user *p); + asmlinkage long sys_io_uring_enter(unsigned int fd, u32 to_submit, + u32 min_complete, u32 flags, +- const sigset_t __user *sig, size_t sigsz); ++ const void __user *argp, size_t argsz); + asmlinkage long sys_io_uring_register(unsigned int fd, unsigned int op, + void __user *arg, unsigned int nr_args); + +--- a/include/trace/events/io_uring.h ++++ b/include/trace/events/io_uring.h +@@ -12,11 +12,11 @@ struct io_wq_work; + /** + * io_uring_create - called after a new io_uring context was prepared + * +- * @fd: corresponding file descriptor +- * @ctx: pointer to a ring context structure ++ * @fd: corresponding file descriptor ++ * @ctx: pointer to a ring context structure + * @sq_entries: actual SQ size + * @cq_entries: actual CQ size +- * @flags: SQ ring flags, provided to io_uring_setup(2) ++ * @flags: SQ ring flags, provided to io_uring_setup(2) + * + * Allows to trace io_uring creation and provide pointer to a context, that can + * be used later to find correlated events. +@@ -49,15 +49,15 @@ TRACE_EVENT(io_uring_create, + ); + + /** +- * io_uring_register - called after a buffer/file/eventfd was succesfully ++ * io_uring_register - called after a buffer/file/eventfd was successfully + * registered for a ring + * +- * @ctx: pointer to a ring context structure +- * @opcode: describes which operation to perform ++ * @ctx: pointer to a ring context structure ++ * @opcode: describes which operation to perform + * @nr_user_files: number of registered files + * @nr_user_bufs: number of registered buffers + * @cq_ev_fd: whether eventfs registered or not +- * @ret: return code ++ * @ret: return code + * + * Allows to trace fixed files/buffers/eventfds, that could be registered to + * avoid an overhead of getting references to them for every operation. This +@@ -142,16 +142,16 @@ TRACE_EVENT(io_uring_queue_async_work, + TP_ARGS(ctx, rw, req, work, flags), + + TP_STRUCT__entry ( +- __field( void *, ctx ) +- __field( int, rw ) +- __field( void *, req ) ++ __field( void *, ctx ) ++ __field( int, rw ) ++ __field( void *, req ) + __field( struct io_wq_work *, work ) + __field( unsigned int, flags ) + ), + + TP_fast_assign( + __entry->ctx = ctx; +- __entry->rw = rw; ++ __entry->rw = rw; + __entry->req = req; + __entry->work = work; + __entry->flags = flags; +@@ -196,10 +196,10 @@ TRACE_EVENT(io_uring_defer, + + /** + * io_uring_link - called before the io_uring request added into link_list of +- * another request ++ * another request + * +- * @ctx: pointer to a ring context structure +- * @req: pointer to a linked request ++ * @ctx: pointer to a ring context structure ++ * @req: pointer to a linked request + * @target_req: pointer to a previous request, that would contain @req + * + * Allows to track linked requests, to understand dependencies between requests +@@ -212,8 +212,8 @@ TRACE_EVENT(io_uring_link, + TP_ARGS(ctx, req, target_req), + + TP_STRUCT__entry ( +- __field( void *, ctx ) +- __field( void *, req ) ++ __field( void *, ctx ) ++ __field( void *, req ) + __field( void *, target_req ) + ), + +@@ -244,7 +244,7 @@ TRACE_EVENT(io_uring_cqring_wait, + TP_ARGS(ctx, min_events), + + TP_STRUCT__entry ( +- __field( void *, ctx ) ++ __field( void *, ctx ) + __field( int, min_events ) + ), + +@@ -272,7 +272,7 @@ TRACE_EVENT(io_uring_fail_link, + TP_ARGS(req, link), + + TP_STRUCT__entry ( +- __field( void *, req ) ++ __field( void *, req ) + __field( void *, link ) + ), + +@@ -290,38 +290,42 @@ TRACE_EVENT(io_uring_fail_link, + * @ctx: pointer to a ring context structure + * @user_data: user data associated with the request + * @res: result of the request ++ * @cflags: completion flags + * + */ + TRACE_EVENT(io_uring_complete, + +- TP_PROTO(void *ctx, u64 user_data, long res), ++ TP_PROTO(void *ctx, u64 user_data, int res, unsigned cflags), + +- TP_ARGS(ctx, user_data, res), ++ TP_ARGS(ctx, user_data, res, cflags), + + TP_STRUCT__entry ( + __field( void *, ctx ) + __field( u64, user_data ) +- __field( long, res ) ++ __field( int, res ) ++ __field( unsigned, cflags ) + ), + + TP_fast_assign( + __entry->ctx = ctx; + __entry->user_data = user_data; + __entry->res = res; ++ __entry->cflags = cflags; + ), + +- TP_printk("ring %p, user_data 0x%llx, result %ld", ++ TP_printk("ring %p, user_data 0x%llx, result %d, cflags %x", + __entry->ctx, (unsigned long long)__entry->user_data, +- __entry->res) ++ __entry->res, __entry->cflags) + ); + +- + /** + * io_uring_submit_sqe - called before submitting one SQE + * + * @ctx: pointer to a ring context structure ++ * @req: pointer to a submitted request + * @opcode: opcode of request + * @user_data: user data associated with the request ++ * @flags request flags + * @force_nonblock: whether a context blocking or not + * @sq_thread: true if sq_thread has submitted this SQE + * +@@ -330,41 +334,60 @@ TRACE_EVENT(io_uring_complete, + */ + TRACE_EVENT(io_uring_submit_sqe, + +- TP_PROTO(void *ctx, u8 opcode, u64 user_data, bool force_nonblock, +- bool sq_thread), ++ TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data, u32 flags, ++ bool force_nonblock, bool sq_thread), + +- TP_ARGS(ctx, opcode, user_data, force_nonblock, sq_thread), ++ TP_ARGS(ctx, req, opcode, user_data, flags, force_nonblock, sq_thread), + + TP_STRUCT__entry ( + __field( void *, ctx ) ++ __field( void *, req ) + __field( u8, opcode ) + __field( u64, user_data ) ++ __field( u32, flags ) + __field( bool, force_nonblock ) + __field( bool, sq_thread ) + ), + + TP_fast_assign( + __entry->ctx = ctx; ++ __entry->req = req; + __entry->opcode = opcode; + __entry->user_data = user_data; ++ __entry->flags = flags; + __entry->force_nonblock = force_nonblock; + __entry->sq_thread = sq_thread; + ), + +- TP_printk("ring %p, op %d, data 0x%llx, non block %d, sq_thread %d", +- __entry->ctx, __entry->opcode, +- (unsigned long long) __entry->user_data, +- __entry->force_nonblock, __entry->sq_thread) ++ TP_printk("ring %p, req %p, op %d, data 0x%llx, flags %u, " ++ "non block %d, sq_thread %d", __entry->ctx, __entry->req, ++ __entry->opcode, (unsigned long long)__entry->user_data, ++ __entry->flags, __entry->force_nonblock, __entry->sq_thread) + ); + ++/* ++ * io_uring_poll_arm - called after arming a poll wait if successful ++ * ++ * @ctx: pointer to a ring context structure ++ * @req: pointer to the armed request ++ * @opcode: opcode of request ++ * @user_data: user data associated with the request ++ * @mask: request poll events mask ++ * @events: registered events of interest ++ * ++ * Allows to track which fds are waiting for and what are the events of ++ * interest. ++ */ + TRACE_EVENT(io_uring_poll_arm, + +- TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask, int events), ++ TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data, ++ int mask, int events), + +- TP_ARGS(ctx, opcode, user_data, mask, events), ++ TP_ARGS(ctx, req, opcode, user_data, mask, events), + + TP_STRUCT__entry ( + __field( void *, ctx ) ++ __field( void *, req ) + __field( u8, opcode ) + __field( u64, user_data ) + __field( int, mask ) +@@ -373,16 +396,17 @@ TRACE_EVENT(io_uring_poll_arm, + + TP_fast_assign( + __entry->ctx = ctx; ++ __entry->req = req; + __entry->opcode = opcode; + __entry->user_data = user_data; + __entry->mask = mask; + __entry->events = events; + ), + +- TP_printk("ring %p, op %d, data 0x%llx, mask 0x%x, events 0x%x", +- __entry->ctx, __entry->opcode, +- (unsigned long long) __entry->user_data, +- __entry->mask, __entry->events) ++ TP_printk("ring %p, req %p, op %d, data 0x%llx, mask 0x%x, events 0x%x", ++ __entry->ctx, __entry->req, __entry->opcode, ++ (unsigned long long) __entry->user_data, ++ __entry->mask, __entry->events) + ); + + TRACE_EVENT(io_uring_poll_wake, +@@ -437,27 +461,40 @@ TRACE_EVENT(io_uring_task_add, + __entry->mask) + ); + ++/* ++ * io_uring_task_run - called when task_work_run() executes the poll events ++ * notification callbacks ++ * ++ * @ctx: pointer to a ring context structure ++ * @req: pointer to the armed request ++ * @opcode: opcode of request ++ * @user_data: user data associated with the request ++ * ++ * Allows to track when notified poll events are processed ++ */ + TRACE_EVENT(io_uring_task_run, + +- TP_PROTO(void *ctx, u8 opcode, u64 user_data), ++ TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data), + +- TP_ARGS(ctx, opcode, user_data), ++ TP_ARGS(ctx, req, opcode, user_data), + + TP_STRUCT__entry ( + __field( void *, ctx ) ++ __field( void *, req ) + __field( u8, opcode ) + __field( u64, user_data ) + ), + + TP_fast_assign( + __entry->ctx = ctx; ++ __entry->req = req; + __entry->opcode = opcode; + __entry->user_data = user_data; + ), + +- TP_printk("ring %p, op %d, data 0x%llx", +- __entry->ctx, __entry->opcode, +- (unsigned long long) __entry->user_data) ++ TP_printk("ring %p, req %p, op %d, data 0x%llx", ++ __entry->ctx, __entry->req, __entry->opcode, ++ (unsigned long long) __entry->user_data) + ); + + #endif /* _TRACE_IO_URING_H */ +--- a/include/uapi/linux/io_uring.h ++++ b/include/uapi/linux/io_uring.h +@@ -42,23 +42,25 @@ struct io_uring_sqe { + __u32 statx_flags; + __u32 fadvise_advice; + __u32 splice_flags; ++ __u32 rename_flags; ++ __u32 unlink_flags; ++ __u32 hardlink_flags; + }; + __u64 user_data; /* data to be passed back at completion time */ ++ /* pack this to avoid bogus arm OABI complaints */ + union { +- struct { +- /* pack this to avoid bogus arm OABI complaints */ +- union { +- /* index into fixed buffers, if used */ +- __u16 buf_index; +- /* for grouped buffer selection */ +- __u16 buf_group; +- } __attribute__((packed)); +- /* personality to use, if used */ +- __u16 personality; +- __s32 splice_fd_in; +- }; +- __u64 __pad2[3]; ++ /* index into fixed buffers, if used */ ++ __u16 buf_index; ++ /* for grouped buffer selection */ ++ __u16 buf_group; ++ } __attribute__((packed)); ++ /* personality to use, if used */ ++ __u16 personality; ++ union { ++ __s32 splice_fd_in; ++ __u32 file_index; + }; ++ __u64 __pad2[2]; + }; + + enum { +@@ -132,6 +134,9 @@ enum { + IORING_OP_PROVIDE_BUFFERS, + IORING_OP_REMOVE_BUFFERS, + IORING_OP_TEE, ++ IORING_OP_SHUTDOWN, ++ IORING_OP_RENAMEAT, ++ IORING_OP_UNLINKAT, + + /* this goes last, obviously */ + IORING_OP_LAST, +@@ -145,8 +150,13 @@ enum { + /* + * sqe->timeout_flags + */ +-#define IORING_TIMEOUT_ABS (1U << 0) +- ++#define IORING_TIMEOUT_ABS (1U << 0) ++#define IORING_TIMEOUT_UPDATE (1U << 1) ++#define IORING_TIMEOUT_BOOTTIME (1U << 2) ++#define IORING_TIMEOUT_REALTIME (1U << 3) ++#define IORING_LINK_TIMEOUT_UPDATE (1U << 4) ++#define IORING_TIMEOUT_CLOCK_MASK (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME) ++#define IORING_TIMEOUT_UPDATE_MASK (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE) + /* + * sqe->splice_flags + * extends splice(2) flags +@@ -154,6 +164,21 @@ enum { + #define SPLICE_F_FD_IN_FIXED (1U << 31) /* the last bit of __u32 */ + + /* ++ * POLL_ADD flags. Note that since sqe->poll_events is the flag space, the ++ * command flags for POLL_ADD are stored in sqe->len. ++ * ++ * IORING_POLL_ADD_MULTI Multishot poll. Sets IORING_CQE_F_MORE if ++ * the poll handler will continue to report ++ * CQEs on behalf of the same SQE. ++ * ++ * IORING_POLL_UPDATE Update existing poll request, matching ++ * sqe->addr as the old user_data field. ++ */ ++#define IORING_POLL_ADD_MULTI (1U << 0) ++#define IORING_POLL_UPDATE_EVENTS (1U << 1) ++#define IORING_POLL_UPDATE_USER_DATA (1U << 2) ++ ++/* + * IO completion data structure (Completion Queue Entry) + */ + struct io_uring_cqe { +@@ -166,8 +191,10 @@ struct io_uring_cqe { + * cqe->flags + * + * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID ++ * IORING_CQE_F_MORE If set, parent SQE will generate more CQE entries + */ + #define IORING_CQE_F_BUFFER (1U << 0) ++#define IORING_CQE_F_MORE (1U << 1) + + enum { + IORING_CQE_BUFFER_SHIFT = 16, +@@ -226,6 +253,7 @@ struct io_cqring_offsets { + #define IORING_ENTER_GETEVENTS (1U << 0) + #define IORING_ENTER_SQ_WAKEUP (1U << 1) + #define IORING_ENTER_SQ_WAIT (1U << 2) ++#define IORING_ENTER_EXT_ARG (1U << 3) + + /* + * Passed in for io_uring_setup(2). Copied back with updated info on success +@@ -253,6 +281,10 @@ struct io_uring_params { + #define IORING_FEAT_CUR_PERSONALITY (1U << 4) + #define IORING_FEAT_FAST_POLL (1U << 5) + #define IORING_FEAT_POLL_32BITS (1U << 6) ++#define IORING_FEAT_SQPOLL_NONFIXED (1U << 7) ++#define IORING_FEAT_EXT_ARG (1U << 8) ++#define IORING_FEAT_NATIVE_WORKERS (1U << 9) ++#define IORING_FEAT_RSRC_TAGS (1U << 10) + + /* + * io_uring_register(2) opcodes and arguments +@@ -272,16 +304,62 @@ enum { + IORING_REGISTER_RESTRICTIONS = 11, + IORING_REGISTER_ENABLE_RINGS = 12, + ++ /* extended with tagging */ ++ IORING_REGISTER_FILES2 = 13, ++ IORING_REGISTER_FILES_UPDATE2 = 14, ++ IORING_REGISTER_BUFFERS2 = 15, ++ IORING_REGISTER_BUFFERS_UPDATE = 16, ++ ++ /* set/clear io-wq thread affinities */ ++ IORING_REGISTER_IOWQ_AFF = 17, ++ IORING_UNREGISTER_IOWQ_AFF = 18, ++ ++ /* set/get max number of io-wq workers */ ++ IORING_REGISTER_IOWQ_MAX_WORKERS = 19, ++ + /* this goes last */ + IORING_REGISTER_LAST + }; + ++/* io-wq worker categories */ ++enum { ++ IO_WQ_BOUND, ++ IO_WQ_UNBOUND, ++}; ++ ++/* deprecated, see struct io_uring_rsrc_update */ + struct io_uring_files_update { + __u32 offset; + __u32 resv; + __aligned_u64 /* __s32 * */ fds; + }; + ++struct io_uring_rsrc_register { ++ __u32 nr; ++ __u32 resv; ++ __u64 resv2; ++ __aligned_u64 data; ++ __aligned_u64 tags; ++}; ++ ++struct io_uring_rsrc_update { ++ __u32 offset; ++ __u32 resv; ++ __aligned_u64 data; ++}; ++ ++struct io_uring_rsrc_update2 { ++ __u32 offset; ++ __u32 resv; ++ __aligned_u64 data; ++ __aligned_u64 tags; ++ __u32 nr; ++ __u32 resv2; ++}; ++ ++/* Skip updating fd indexes set to this value in the fd table */ ++#define IORING_REGISTER_FILES_SKIP (-2) ++ + #define IO_URING_OP_SUPPORTED (1U << 0) + + struct io_uring_probe_op { +@@ -329,4 +407,11 @@ enum { + IORING_RESTRICTION_LAST + }; + ++struct io_uring_getevents_arg { ++ __u64 sigmask; ++ __u32 sigmask_sz; ++ __u32 pad; ++ __u64 ts; ++}; ++ + #endif +--- /dev/null ++++ b/io_uring/Makefile +@@ -0,0 +1,6 @@ ++# SPDX-License-Identifier: GPL-2.0 ++# ++# Makefile for io_uring ++ ++obj-$(CONFIG_IO_URING) += io_uring.o ++obj-$(CONFIG_IO_WQ) += io-wq.o +--- /dev/null ++++ b/io_uring/io-wq.c +@@ -0,0 +1,1398 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Basic worker thread pool for io_uring ++ * ++ * Copyright (C) 2019 Jens Axboe ++ * ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "io-wq.h" ++ ++#define WORKER_IDLE_TIMEOUT (5 * HZ) ++ ++enum { ++ IO_WORKER_F_UP = 1, /* up and active */ ++ IO_WORKER_F_RUNNING = 2, /* account as running */ ++ IO_WORKER_F_FREE = 4, /* worker on free list */ ++ IO_WORKER_F_BOUND = 8, /* is doing bounded work */ ++}; ++ ++enum { ++ IO_WQ_BIT_EXIT = 0, /* wq exiting */ ++}; ++ ++enum { ++ IO_ACCT_STALLED_BIT = 0, /* stalled on hash */ ++}; ++ ++/* ++ * One for each thread in a wqe pool ++ */ ++struct io_worker { ++ refcount_t ref; ++ unsigned flags; ++ struct hlist_nulls_node nulls_node; ++ struct list_head all_list; ++ struct task_struct *task; ++ struct io_wqe *wqe; ++ ++ struct io_wq_work *cur_work; ++ spinlock_t lock; ++ ++ struct completion ref_done; ++ ++ unsigned long create_state; ++ struct callback_head create_work; ++ int create_index; ++ ++ union { ++ struct rcu_head rcu; ++ struct work_struct work; ++ }; ++}; ++ ++#if BITS_PER_LONG == 64 ++#define IO_WQ_HASH_ORDER 6 ++#else ++#define IO_WQ_HASH_ORDER 5 ++#endif ++ ++#define IO_WQ_NR_HASH_BUCKETS (1u << IO_WQ_HASH_ORDER) ++ ++struct io_wqe_acct { ++ unsigned nr_workers; ++ unsigned max_workers; ++ int index; ++ atomic_t nr_running; ++ struct io_wq_work_list work_list; ++ unsigned long flags; ++}; ++ ++enum { ++ IO_WQ_ACCT_BOUND, ++ IO_WQ_ACCT_UNBOUND, ++ IO_WQ_ACCT_NR, ++}; ++ ++/* ++ * Per-node worker thread pool ++ */ ++struct io_wqe { ++ raw_spinlock_t lock; ++ struct io_wqe_acct acct[2]; ++ ++ int node; ++ ++ struct hlist_nulls_head free_list; ++ struct list_head all_list; ++ ++ struct wait_queue_entry wait; ++ ++ struct io_wq *wq; ++ struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS]; ++ ++ cpumask_var_t cpu_mask; ++}; ++ ++/* ++ * Per io_wq state ++ */ ++struct io_wq { ++ unsigned long state; ++ ++ free_work_fn *free_work; ++ io_wq_work_fn *do_work; ++ ++ struct io_wq_hash *hash; ++ ++ atomic_t worker_refs; ++ struct completion worker_done; ++ ++ struct hlist_node cpuhp_node; ++ ++ struct task_struct *task; ++ ++ struct io_wqe *wqes[]; ++}; ++ ++static enum cpuhp_state io_wq_online; ++ ++struct io_cb_cancel_data { ++ work_cancel_fn *fn; ++ void *data; ++ int nr_running; ++ int nr_pending; ++ bool cancel_all; ++}; ++ ++static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index); ++static void io_wqe_dec_running(struct io_worker *worker); ++static bool io_acct_cancel_pending_work(struct io_wqe *wqe, ++ struct io_wqe_acct *acct, ++ struct io_cb_cancel_data *match); ++static void create_worker_cb(struct callback_head *cb); ++static void io_wq_cancel_tw_create(struct io_wq *wq); ++ ++static bool io_worker_get(struct io_worker *worker) ++{ ++ return refcount_inc_not_zero(&worker->ref); ++} ++ ++static void io_worker_release(struct io_worker *worker) ++{ ++ if (refcount_dec_and_test(&worker->ref)) ++ complete(&worker->ref_done); ++} ++ ++static inline struct io_wqe_acct *io_get_acct(struct io_wqe *wqe, bool bound) ++{ ++ return &wqe->acct[bound ? IO_WQ_ACCT_BOUND : IO_WQ_ACCT_UNBOUND]; ++} ++ ++static inline struct io_wqe_acct *io_work_get_acct(struct io_wqe *wqe, ++ struct io_wq_work *work) ++{ ++ return io_get_acct(wqe, !(work->flags & IO_WQ_WORK_UNBOUND)); ++} ++ ++static inline struct io_wqe_acct *io_wqe_get_acct(struct io_worker *worker) ++{ ++ return io_get_acct(worker->wqe, worker->flags & IO_WORKER_F_BOUND); ++} ++ ++static void io_worker_ref_put(struct io_wq *wq) ++{ ++ if (atomic_dec_and_test(&wq->worker_refs)) ++ complete(&wq->worker_done); ++} ++ ++static void io_worker_cancel_cb(struct io_worker *worker) ++{ ++ struct io_wqe_acct *acct = io_wqe_get_acct(worker); ++ struct io_wqe *wqe = worker->wqe; ++ struct io_wq *wq = wqe->wq; ++ ++ atomic_dec(&acct->nr_running); ++ raw_spin_lock(&worker->wqe->lock); ++ acct->nr_workers--; ++ raw_spin_unlock(&worker->wqe->lock); ++ io_worker_ref_put(wq); ++ clear_bit_unlock(0, &worker->create_state); ++ io_worker_release(worker); ++} ++ ++static bool io_task_worker_match(struct callback_head *cb, void *data) ++{ ++ struct io_worker *worker; ++ ++ if (cb->func != create_worker_cb) ++ return false; ++ worker = container_of(cb, struct io_worker, create_work); ++ return worker == data; ++} ++ ++static void io_worker_exit(struct io_worker *worker) ++{ ++ struct io_wqe *wqe = worker->wqe; ++ struct io_wq *wq = wqe->wq; ++ ++ while (1) { ++ struct callback_head *cb = task_work_cancel_match(wq->task, ++ io_task_worker_match, worker); ++ ++ if (!cb) ++ break; ++ io_worker_cancel_cb(worker); ++ } ++ ++ if (refcount_dec_and_test(&worker->ref)) ++ complete(&worker->ref_done); ++ wait_for_completion(&worker->ref_done); ++ ++ raw_spin_lock(&wqe->lock); ++ if (worker->flags & IO_WORKER_F_FREE) ++ hlist_nulls_del_rcu(&worker->nulls_node); ++ list_del_rcu(&worker->all_list); ++ preempt_disable(); ++ io_wqe_dec_running(worker); ++ worker->flags = 0; ++ current->flags &= ~PF_IO_WORKER; ++ preempt_enable(); ++ raw_spin_unlock(&wqe->lock); ++ ++ kfree_rcu(worker, rcu); ++ io_worker_ref_put(wqe->wq); ++ do_exit(0); ++} ++ ++static inline bool io_acct_run_queue(struct io_wqe_acct *acct) ++{ ++ if (!wq_list_empty(&acct->work_list) && ++ !test_bit(IO_ACCT_STALLED_BIT, &acct->flags)) ++ return true; ++ return false; ++} ++ ++/* ++ * Check head of free list for an available worker. If one isn't available, ++ * caller must create one. ++ */ ++static bool io_wqe_activate_free_worker(struct io_wqe *wqe, ++ struct io_wqe_acct *acct) ++ __must_hold(RCU) ++{ ++ struct hlist_nulls_node *n; ++ struct io_worker *worker; ++ ++ /* ++ * Iterate free_list and see if we can find an idle worker to ++ * activate. If a given worker is on the free_list but in the process ++ * of exiting, keep trying. ++ */ ++ hlist_nulls_for_each_entry_rcu(worker, n, &wqe->free_list, nulls_node) { ++ if (!io_worker_get(worker)) ++ continue; ++ if (io_wqe_get_acct(worker) != acct) { ++ io_worker_release(worker); ++ continue; ++ } ++ if (wake_up_process(worker->task)) { ++ io_worker_release(worker); ++ return true; ++ } ++ io_worker_release(worker); ++ } ++ ++ return false; ++} ++ ++/* ++ * We need a worker. If we find a free one, we're good. If not, and we're ++ * below the max number of workers, create one. ++ */ ++static bool io_wqe_create_worker(struct io_wqe *wqe, struct io_wqe_acct *acct) ++{ ++ /* ++ * Most likely an attempt to queue unbounded work on an io_wq that ++ * wasn't setup with any unbounded workers. ++ */ ++ if (unlikely(!acct->max_workers)) ++ pr_warn_once("io-wq is not configured for unbound workers"); ++ ++ raw_spin_lock(&wqe->lock); ++ if (acct->nr_workers >= acct->max_workers) { ++ raw_spin_unlock(&wqe->lock); ++ return true; ++ } ++ acct->nr_workers++; ++ raw_spin_unlock(&wqe->lock); ++ atomic_inc(&acct->nr_running); ++ atomic_inc(&wqe->wq->worker_refs); ++ return create_io_worker(wqe->wq, wqe, acct->index); ++} ++ ++static void io_wqe_inc_running(struct io_worker *worker) ++{ ++ struct io_wqe_acct *acct = io_wqe_get_acct(worker); ++ ++ atomic_inc(&acct->nr_running); ++} ++ ++static void create_worker_cb(struct callback_head *cb) ++{ ++ struct io_worker *worker; ++ struct io_wq *wq; ++ struct io_wqe *wqe; ++ struct io_wqe_acct *acct; ++ bool do_create = false; ++ ++ worker = container_of(cb, struct io_worker, create_work); ++ wqe = worker->wqe; ++ wq = wqe->wq; ++ acct = &wqe->acct[worker->create_index]; ++ raw_spin_lock(&wqe->lock); ++ if (acct->nr_workers < acct->max_workers) { ++ acct->nr_workers++; ++ do_create = true; ++ } ++ raw_spin_unlock(&wqe->lock); ++ if (do_create) { ++ create_io_worker(wq, wqe, worker->create_index); ++ } else { ++ atomic_dec(&acct->nr_running); ++ io_worker_ref_put(wq); ++ } ++ clear_bit_unlock(0, &worker->create_state); ++ io_worker_release(worker); ++} ++ ++static bool io_queue_worker_create(struct io_worker *worker, ++ struct io_wqe_acct *acct, ++ task_work_func_t func) ++{ ++ struct io_wqe *wqe = worker->wqe; ++ struct io_wq *wq = wqe->wq; ++ ++ /* raced with exit, just ignore create call */ ++ if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) ++ goto fail; ++ if (!io_worker_get(worker)) ++ goto fail; ++ /* ++ * create_state manages ownership of create_work/index. We should ++ * only need one entry per worker, as the worker going to sleep ++ * will trigger the condition, and waking will clear it once it ++ * runs the task_work. ++ */ ++ if (test_bit(0, &worker->create_state) || ++ test_and_set_bit_lock(0, &worker->create_state)) ++ goto fail_release; ++ ++ atomic_inc(&wq->worker_refs); ++ init_task_work(&worker->create_work, func); ++ worker->create_index = acct->index; ++ if (!task_work_add(wq->task, &worker->create_work, TWA_SIGNAL)) { ++ /* ++ * EXIT may have been set after checking it above, check after ++ * adding the task_work and remove any creation item if it is ++ * now set. wq exit does that too, but we can have added this ++ * work item after we canceled in io_wq_exit_workers(). ++ */ ++ if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) ++ io_wq_cancel_tw_create(wq); ++ io_worker_ref_put(wq); ++ return true; ++ } ++ io_worker_ref_put(wq); ++ clear_bit_unlock(0, &worker->create_state); ++fail_release: ++ io_worker_release(worker); ++fail: ++ atomic_dec(&acct->nr_running); ++ io_worker_ref_put(wq); ++ return false; ++} ++ ++static void io_wqe_dec_running(struct io_worker *worker) ++ __must_hold(wqe->lock) ++{ ++ struct io_wqe_acct *acct = io_wqe_get_acct(worker); ++ struct io_wqe *wqe = worker->wqe; ++ ++ if (!(worker->flags & IO_WORKER_F_UP)) ++ return; ++ ++ if (atomic_dec_and_test(&acct->nr_running) && io_acct_run_queue(acct)) { ++ atomic_inc(&acct->nr_running); ++ atomic_inc(&wqe->wq->worker_refs); ++ raw_spin_unlock(&wqe->lock); ++ io_queue_worker_create(worker, acct, create_worker_cb); ++ raw_spin_lock(&wqe->lock); ++ } ++} ++ ++/* ++ * Worker will start processing some work. Move it to the busy list, if ++ * it's currently on the freelist ++ */ ++static void __io_worker_busy(struct io_wqe *wqe, struct io_worker *worker, ++ struct io_wq_work *work) ++ __must_hold(wqe->lock) ++{ ++ if (worker->flags & IO_WORKER_F_FREE) { ++ worker->flags &= ~IO_WORKER_F_FREE; ++ hlist_nulls_del_init_rcu(&worker->nulls_node); ++ } ++} ++ ++/* ++ * No work, worker going to sleep. Move to freelist, and unuse mm if we ++ * have one attached. Dropping the mm may potentially sleep, so we drop ++ * the lock in that case and return success. Since the caller has to ++ * retry the loop in that case (we changed task state), we don't regrab ++ * the lock if we return success. ++ */ ++static void __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker) ++ __must_hold(wqe->lock) ++{ ++ if (!(worker->flags & IO_WORKER_F_FREE)) { ++ worker->flags |= IO_WORKER_F_FREE; ++ hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); ++ } ++} ++ ++static inline unsigned int io_get_work_hash(struct io_wq_work *work) ++{ ++ return work->flags >> IO_WQ_HASH_SHIFT; ++} ++ ++static bool io_wait_on_hash(struct io_wqe *wqe, unsigned int hash) ++{ ++ struct io_wq *wq = wqe->wq; ++ bool ret = false; ++ ++ spin_lock_irq(&wq->hash->wait.lock); ++ if (list_empty(&wqe->wait.entry)) { ++ __add_wait_queue(&wq->hash->wait, &wqe->wait); ++ if (!test_bit(hash, &wq->hash->map)) { ++ __set_current_state(TASK_RUNNING); ++ list_del_init(&wqe->wait.entry); ++ ret = true; ++ } ++ } ++ spin_unlock_irq(&wq->hash->wait.lock); ++ return ret; ++} ++ ++static struct io_wq_work *io_get_next_work(struct io_wqe_acct *acct, ++ struct io_worker *worker) ++ __must_hold(wqe->lock) ++{ ++ struct io_wq_work_node *node, *prev; ++ struct io_wq_work *work, *tail; ++ unsigned int stall_hash = -1U; ++ struct io_wqe *wqe = worker->wqe; ++ ++ wq_list_for_each(node, prev, &acct->work_list) { ++ unsigned int hash; ++ ++ work = container_of(node, struct io_wq_work, list); ++ ++ /* not hashed, can run anytime */ ++ if (!io_wq_is_hashed(work)) { ++ wq_list_del(&acct->work_list, node, prev); ++ return work; ++ } ++ ++ hash = io_get_work_hash(work); ++ /* all items with this hash lie in [work, tail] */ ++ tail = wqe->hash_tail[hash]; ++ ++ /* hashed, can run if not already running */ ++ if (!test_and_set_bit(hash, &wqe->wq->hash->map)) { ++ wqe->hash_tail[hash] = NULL; ++ wq_list_cut(&acct->work_list, &tail->list, prev); ++ return work; ++ } ++ if (stall_hash == -1U) ++ stall_hash = hash; ++ /* fast forward to a next hash, for-each will fix up @prev */ ++ node = &tail->list; ++ } ++ ++ if (stall_hash != -1U) { ++ bool unstalled; ++ ++ /* ++ * Set this before dropping the lock to avoid racing with new ++ * work being added and clearing the stalled bit. ++ */ ++ set_bit(IO_ACCT_STALLED_BIT, &acct->flags); ++ raw_spin_unlock(&wqe->lock); ++ unstalled = io_wait_on_hash(wqe, stall_hash); ++ raw_spin_lock(&wqe->lock); ++ if (unstalled) { ++ clear_bit(IO_ACCT_STALLED_BIT, &acct->flags); ++ if (wq_has_sleeper(&wqe->wq->hash->wait)) ++ wake_up(&wqe->wq->hash->wait); ++ } ++ } ++ ++ return NULL; ++} ++ ++static bool io_flush_signals(void) ++{ ++ if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL))) { ++ __set_current_state(TASK_RUNNING); ++ tracehook_notify_signal(); ++ return true; ++ } ++ return false; ++} ++ ++static void io_assign_current_work(struct io_worker *worker, ++ struct io_wq_work *work) ++{ ++ if (work) { ++ io_flush_signals(); ++ cond_resched(); ++ } ++ ++ spin_lock(&worker->lock); ++ worker->cur_work = work; ++ spin_unlock(&worker->lock); ++} ++ ++static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work); ++ ++static void io_worker_handle_work(struct io_worker *worker) ++ __releases(wqe->lock) ++{ ++ struct io_wqe_acct *acct = io_wqe_get_acct(worker); ++ struct io_wqe *wqe = worker->wqe; ++ struct io_wq *wq = wqe->wq; ++ bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state); ++ ++ do { ++ struct io_wq_work *work; ++get_next: ++ /* ++ * If we got some work, mark us as busy. If we didn't, but ++ * the list isn't empty, it means we stalled on hashed work. ++ * Mark us stalled so we don't keep looking for work when we ++ * can't make progress, any work completion or insertion will ++ * clear the stalled flag. ++ */ ++ work = io_get_next_work(acct, worker); ++ if (work) ++ __io_worker_busy(wqe, worker, work); ++ ++ raw_spin_unlock(&wqe->lock); ++ if (!work) ++ break; ++ io_assign_current_work(worker, work); ++ __set_current_state(TASK_RUNNING); ++ ++ /* handle a whole dependent link */ ++ do { ++ struct io_wq_work *next_hashed, *linked; ++ unsigned int hash = io_get_work_hash(work); ++ ++ next_hashed = wq_next_work(work); ++ ++ if (unlikely(do_kill) && (work->flags & IO_WQ_WORK_UNBOUND)) ++ work->flags |= IO_WQ_WORK_CANCEL; ++ wq->do_work(work); ++ io_assign_current_work(worker, NULL); ++ ++ linked = wq->free_work(work); ++ work = next_hashed; ++ if (!work && linked && !io_wq_is_hashed(linked)) { ++ work = linked; ++ linked = NULL; ++ } ++ io_assign_current_work(worker, work); ++ if (linked) ++ io_wqe_enqueue(wqe, linked); ++ ++ if (hash != -1U && !next_hashed) { ++ /* serialize hash clear with wake_up() */ ++ spin_lock_irq(&wq->hash->wait.lock); ++ clear_bit(hash, &wq->hash->map); ++ clear_bit(IO_ACCT_STALLED_BIT, &acct->flags); ++ spin_unlock_irq(&wq->hash->wait.lock); ++ if (wq_has_sleeper(&wq->hash->wait)) ++ wake_up(&wq->hash->wait); ++ raw_spin_lock(&wqe->lock); ++ /* skip unnecessary unlock-lock wqe->lock */ ++ if (!work) ++ goto get_next; ++ raw_spin_unlock(&wqe->lock); ++ } ++ } while (work); ++ ++ raw_spin_lock(&wqe->lock); ++ } while (1); ++} ++ ++static int io_wqe_worker(void *data) ++{ ++ struct io_worker *worker = data; ++ struct io_wqe_acct *acct = io_wqe_get_acct(worker); ++ struct io_wqe *wqe = worker->wqe; ++ struct io_wq *wq = wqe->wq; ++ bool last_timeout = false; ++ char buf[TASK_COMM_LEN]; ++ ++ worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); ++ ++ snprintf(buf, sizeof(buf), "iou-wrk-%d", wq->task->pid); ++ set_task_comm(current, buf); ++ ++ while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { ++ long ret; ++ ++ set_current_state(TASK_INTERRUPTIBLE); ++loop: ++ raw_spin_lock(&wqe->lock); ++ if (io_acct_run_queue(acct)) { ++ io_worker_handle_work(worker); ++ goto loop; ++ } ++ /* timed out, exit unless we're the last worker */ ++ if (last_timeout && acct->nr_workers > 1) { ++ acct->nr_workers--; ++ raw_spin_unlock(&wqe->lock); ++ __set_current_state(TASK_RUNNING); ++ break; ++ } ++ last_timeout = false; ++ __io_worker_idle(wqe, worker); ++ raw_spin_unlock(&wqe->lock); ++ if (io_flush_signals()) ++ continue; ++ ret = schedule_timeout(WORKER_IDLE_TIMEOUT); ++ if (signal_pending(current)) { ++ struct ksignal ksig; ++ ++ if (!get_signal(&ksig)) ++ continue; ++ break; ++ } ++ last_timeout = !ret; ++ } ++ ++ if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) { ++ raw_spin_lock(&wqe->lock); ++ io_worker_handle_work(worker); ++ } ++ ++ io_worker_exit(worker); ++ return 0; ++} ++ ++/* ++ * Called when a worker is scheduled in. Mark us as currently running. ++ */ ++void io_wq_worker_running(struct task_struct *tsk) ++{ ++ struct io_worker *worker = tsk->pf_io_worker; ++ ++ if (!worker) ++ return; ++ if (!(worker->flags & IO_WORKER_F_UP)) ++ return; ++ if (worker->flags & IO_WORKER_F_RUNNING) ++ return; ++ worker->flags |= IO_WORKER_F_RUNNING; ++ io_wqe_inc_running(worker); ++} ++ ++/* ++ * Called when worker is going to sleep. If there are no workers currently ++ * running and we have work pending, wake up a free one or create a new one. ++ */ ++void io_wq_worker_sleeping(struct task_struct *tsk) ++{ ++ struct io_worker *worker = tsk->pf_io_worker; ++ ++ if (!worker) ++ return; ++ if (!(worker->flags & IO_WORKER_F_UP)) ++ return; ++ if (!(worker->flags & IO_WORKER_F_RUNNING)) ++ return; ++ ++ worker->flags &= ~IO_WORKER_F_RUNNING; ++ ++ raw_spin_lock(&worker->wqe->lock); ++ io_wqe_dec_running(worker); ++ raw_spin_unlock(&worker->wqe->lock); ++} ++ ++static void io_init_new_worker(struct io_wqe *wqe, struct io_worker *worker, ++ struct task_struct *tsk) ++{ ++ tsk->pf_io_worker = worker; ++ worker->task = tsk; ++ set_cpus_allowed_ptr(tsk, wqe->cpu_mask); ++ tsk->flags |= PF_NO_SETAFFINITY; ++ ++ raw_spin_lock(&wqe->lock); ++ hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); ++ list_add_tail_rcu(&worker->all_list, &wqe->all_list); ++ worker->flags |= IO_WORKER_F_FREE; ++ raw_spin_unlock(&wqe->lock); ++ wake_up_new_task(tsk); ++} ++ ++static bool io_wq_work_match_all(struct io_wq_work *work, void *data) ++{ ++ return true; ++} ++ ++static inline bool io_should_retry_thread(long err) ++{ ++ /* ++ * Prevent perpetual task_work retry, if the task (or its group) is ++ * exiting. ++ */ ++ if (fatal_signal_pending(current)) ++ return false; ++ ++ switch (err) { ++ case -EAGAIN: ++ case -ERESTARTSYS: ++ case -ERESTARTNOINTR: ++ case -ERESTARTNOHAND: ++ return true; ++ default: ++ return false; ++ } ++} ++ ++static void create_worker_cont(struct callback_head *cb) ++{ ++ struct io_worker *worker; ++ struct task_struct *tsk; ++ struct io_wqe *wqe; ++ ++ worker = container_of(cb, struct io_worker, create_work); ++ clear_bit_unlock(0, &worker->create_state); ++ wqe = worker->wqe; ++ tsk = create_io_thread(io_wqe_worker, worker, wqe->node); ++ if (!IS_ERR(tsk)) { ++ io_init_new_worker(wqe, worker, tsk); ++ io_worker_release(worker); ++ return; ++ } else if (!io_should_retry_thread(PTR_ERR(tsk))) { ++ struct io_wqe_acct *acct = io_wqe_get_acct(worker); ++ ++ atomic_dec(&acct->nr_running); ++ raw_spin_lock(&wqe->lock); ++ acct->nr_workers--; ++ if (!acct->nr_workers) { ++ struct io_cb_cancel_data match = { ++ .fn = io_wq_work_match_all, ++ .cancel_all = true, ++ }; ++ ++ while (io_acct_cancel_pending_work(wqe, acct, &match)) ++ raw_spin_lock(&wqe->lock); ++ } ++ raw_spin_unlock(&wqe->lock); ++ io_worker_ref_put(wqe->wq); ++ kfree(worker); ++ return; ++ } ++ ++ /* re-create attempts grab a new worker ref, drop the existing one */ ++ io_worker_release(worker); ++ schedule_work(&worker->work); ++} ++ ++static void io_workqueue_create(struct work_struct *work) ++{ ++ struct io_worker *worker = container_of(work, struct io_worker, work); ++ struct io_wqe_acct *acct = io_wqe_get_acct(worker); ++ ++ if (!io_queue_worker_create(worker, acct, create_worker_cont)) ++ kfree(worker); ++} ++ ++static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) ++{ ++ struct io_wqe_acct *acct = &wqe->acct[index]; ++ struct io_worker *worker; ++ struct task_struct *tsk; ++ ++ __set_current_state(TASK_RUNNING); ++ ++ worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, wqe->node); ++ if (!worker) { ++fail: ++ atomic_dec(&acct->nr_running); ++ raw_spin_lock(&wqe->lock); ++ acct->nr_workers--; ++ raw_spin_unlock(&wqe->lock); ++ io_worker_ref_put(wq); ++ return false; ++ } ++ ++ refcount_set(&worker->ref, 1); ++ worker->wqe = wqe; ++ spin_lock_init(&worker->lock); ++ init_completion(&worker->ref_done); ++ ++ if (index == IO_WQ_ACCT_BOUND) ++ worker->flags |= IO_WORKER_F_BOUND; ++ ++ tsk = create_io_thread(io_wqe_worker, worker, wqe->node); ++ if (!IS_ERR(tsk)) { ++ io_init_new_worker(wqe, worker, tsk); ++ } else if (!io_should_retry_thread(PTR_ERR(tsk))) { ++ kfree(worker); ++ goto fail; ++ } else { ++ INIT_WORK(&worker->work, io_workqueue_create); ++ schedule_work(&worker->work); ++ } ++ ++ return true; ++} ++ ++/* ++ * Iterate the passed in list and call the specific function for each ++ * worker that isn't exiting ++ */ ++static bool io_wq_for_each_worker(struct io_wqe *wqe, ++ bool (*func)(struct io_worker *, void *), ++ void *data) ++{ ++ struct io_worker *worker; ++ bool ret = false; ++ ++ list_for_each_entry_rcu(worker, &wqe->all_list, all_list) { ++ if (io_worker_get(worker)) { ++ /* no task if node is/was offline */ ++ if (worker->task) ++ ret = func(worker, data); ++ io_worker_release(worker); ++ if (ret) ++ break; ++ } ++ } ++ ++ return ret; ++} ++ ++static bool io_wq_worker_wake(struct io_worker *worker, void *data) ++{ ++ set_notify_signal(worker->task); ++ wake_up_process(worker->task); ++ return false; ++} ++ ++static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe) ++{ ++ struct io_wq *wq = wqe->wq; ++ ++ do { ++ work->flags |= IO_WQ_WORK_CANCEL; ++ wq->do_work(work); ++ work = wq->free_work(work); ++ } while (work); ++} ++ ++static void io_wqe_insert_work(struct io_wqe *wqe, struct io_wq_work *work) ++{ ++ struct io_wqe_acct *acct = io_work_get_acct(wqe, work); ++ unsigned int hash; ++ struct io_wq_work *tail; ++ ++ if (!io_wq_is_hashed(work)) { ++append: ++ wq_list_add_tail(&work->list, &acct->work_list); ++ return; ++ } ++ ++ hash = io_get_work_hash(work); ++ tail = wqe->hash_tail[hash]; ++ wqe->hash_tail[hash] = work; ++ if (!tail) ++ goto append; ++ ++ wq_list_add_after(&work->list, &tail->list, &acct->work_list); ++} ++ ++static bool io_wq_work_match_item(struct io_wq_work *work, void *data) ++{ ++ return work == data; ++} ++ ++static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) ++{ ++ struct io_wqe_acct *acct = io_work_get_acct(wqe, work); ++ unsigned work_flags = work->flags; ++ bool do_create; ++ ++ /* ++ * If io-wq is exiting for this task, or if the request has explicitly ++ * been marked as one that should not get executed, cancel it here. ++ */ ++ if (test_bit(IO_WQ_BIT_EXIT, &wqe->wq->state) || ++ (work->flags & IO_WQ_WORK_CANCEL)) { ++ io_run_cancel(work, wqe); ++ return; ++ } ++ ++ raw_spin_lock(&wqe->lock); ++ io_wqe_insert_work(wqe, work); ++ clear_bit(IO_ACCT_STALLED_BIT, &acct->flags); ++ ++ rcu_read_lock(); ++ do_create = !io_wqe_activate_free_worker(wqe, acct); ++ rcu_read_unlock(); ++ ++ raw_spin_unlock(&wqe->lock); ++ ++ if (do_create && ((work_flags & IO_WQ_WORK_CONCURRENT) || ++ !atomic_read(&acct->nr_running))) { ++ bool did_create; ++ ++ did_create = io_wqe_create_worker(wqe, acct); ++ if (likely(did_create)) ++ return; ++ ++ raw_spin_lock(&wqe->lock); ++ /* fatal condition, failed to create the first worker */ ++ if (!acct->nr_workers) { ++ struct io_cb_cancel_data match = { ++ .fn = io_wq_work_match_item, ++ .data = work, ++ .cancel_all = false, ++ }; ++ ++ if (io_acct_cancel_pending_work(wqe, acct, &match)) ++ raw_spin_lock(&wqe->lock); ++ } ++ raw_spin_unlock(&wqe->lock); ++ } ++} ++ ++void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work) ++{ ++ struct io_wqe *wqe = wq->wqes[numa_node_id()]; ++ ++ io_wqe_enqueue(wqe, work); ++} ++ ++/* ++ * Work items that hash to the same value will not be done in parallel. ++ * Used to limit concurrent writes, generally hashed by inode. ++ */ ++void io_wq_hash_work(struct io_wq_work *work, void *val) ++{ ++ unsigned int bit; ++ ++ bit = hash_ptr(val, IO_WQ_HASH_ORDER); ++ work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT)); ++} ++ ++static bool io_wq_worker_cancel(struct io_worker *worker, void *data) ++{ ++ struct io_cb_cancel_data *match = data; ++ ++ /* ++ * Hold the lock to avoid ->cur_work going out of scope, caller ++ * may dereference the passed in work. ++ */ ++ spin_lock(&worker->lock); ++ if (worker->cur_work && ++ match->fn(worker->cur_work, match->data)) { ++ set_notify_signal(worker->task); ++ match->nr_running++; ++ } ++ spin_unlock(&worker->lock); ++ ++ return match->nr_running && !match->cancel_all; ++} ++ ++static inline void io_wqe_remove_pending(struct io_wqe *wqe, ++ struct io_wq_work *work, ++ struct io_wq_work_node *prev) ++{ ++ struct io_wqe_acct *acct = io_work_get_acct(wqe, work); ++ unsigned int hash = io_get_work_hash(work); ++ struct io_wq_work *prev_work = NULL; ++ ++ if (io_wq_is_hashed(work) && work == wqe->hash_tail[hash]) { ++ if (prev) ++ prev_work = container_of(prev, struct io_wq_work, list); ++ if (prev_work && io_get_work_hash(prev_work) == hash) ++ wqe->hash_tail[hash] = prev_work; ++ else ++ wqe->hash_tail[hash] = NULL; ++ } ++ wq_list_del(&acct->work_list, &work->list, prev); ++} ++ ++static bool io_acct_cancel_pending_work(struct io_wqe *wqe, ++ struct io_wqe_acct *acct, ++ struct io_cb_cancel_data *match) ++ __releases(wqe->lock) ++{ ++ struct io_wq_work_node *node, *prev; ++ struct io_wq_work *work; ++ ++ wq_list_for_each(node, prev, &acct->work_list) { ++ work = container_of(node, struct io_wq_work, list); ++ if (!match->fn(work, match->data)) ++ continue; ++ io_wqe_remove_pending(wqe, work, prev); ++ raw_spin_unlock(&wqe->lock); ++ io_run_cancel(work, wqe); ++ match->nr_pending++; ++ /* not safe to continue after unlock */ ++ return true; ++ } ++ ++ return false; ++} ++ ++static void io_wqe_cancel_pending_work(struct io_wqe *wqe, ++ struct io_cb_cancel_data *match) ++{ ++ int i; ++retry: ++ raw_spin_lock(&wqe->lock); ++ for (i = 0; i < IO_WQ_ACCT_NR; i++) { ++ struct io_wqe_acct *acct = io_get_acct(wqe, i == 0); ++ ++ if (io_acct_cancel_pending_work(wqe, acct, match)) { ++ if (match->cancel_all) ++ goto retry; ++ return; ++ } ++ } ++ raw_spin_unlock(&wqe->lock); ++} ++ ++static void io_wqe_cancel_running_work(struct io_wqe *wqe, ++ struct io_cb_cancel_data *match) ++{ ++ rcu_read_lock(); ++ io_wq_for_each_worker(wqe, io_wq_worker_cancel, match); ++ rcu_read_unlock(); ++} ++ ++enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, ++ void *data, bool cancel_all) ++{ ++ struct io_cb_cancel_data match = { ++ .fn = cancel, ++ .data = data, ++ .cancel_all = cancel_all, ++ }; ++ int node; ++ ++ /* ++ * First check pending list, if we're lucky we can just remove it ++ * from there. CANCEL_OK means that the work is returned as-new, ++ * no completion will be posted for it. ++ */ ++ for_each_node(node) { ++ struct io_wqe *wqe = wq->wqes[node]; ++ ++ io_wqe_cancel_pending_work(wqe, &match); ++ if (match.nr_pending && !match.cancel_all) ++ return IO_WQ_CANCEL_OK; ++ } ++ ++ /* ++ * Now check if a free (going busy) or busy worker has the work ++ * currently running. If we find it there, we'll return CANCEL_RUNNING ++ * as an indication that we attempt to signal cancellation. The ++ * completion will run normally in this case. ++ */ ++ for_each_node(node) { ++ struct io_wqe *wqe = wq->wqes[node]; ++ ++ io_wqe_cancel_running_work(wqe, &match); ++ if (match.nr_running && !match.cancel_all) ++ return IO_WQ_CANCEL_RUNNING; ++ } ++ ++ if (match.nr_running) ++ return IO_WQ_CANCEL_RUNNING; ++ if (match.nr_pending) ++ return IO_WQ_CANCEL_OK; ++ return IO_WQ_CANCEL_NOTFOUND; ++} ++ ++static int io_wqe_hash_wake(struct wait_queue_entry *wait, unsigned mode, ++ int sync, void *key) ++{ ++ struct io_wqe *wqe = container_of(wait, struct io_wqe, wait); ++ int i; ++ ++ list_del_init(&wait->entry); ++ ++ rcu_read_lock(); ++ for (i = 0; i < IO_WQ_ACCT_NR; i++) { ++ struct io_wqe_acct *acct = &wqe->acct[i]; ++ ++ if (test_and_clear_bit(IO_ACCT_STALLED_BIT, &acct->flags)) ++ io_wqe_activate_free_worker(wqe, acct); ++ } ++ rcu_read_unlock(); ++ return 1; ++} ++ ++struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) ++{ ++ int ret, node, i; ++ struct io_wq *wq; ++ ++ if (WARN_ON_ONCE(!data->free_work || !data->do_work)) ++ return ERR_PTR(-EINVAL); ++ if (WARN_ON_ONCE(!bounded)) ++ return ERR_PTR(-EINVAL); ++ ++ wq = kzalloc(struct_size(wq, wqes, nr_node_ids), GFP_KERNEL); ++ if (!wq) ++ return ERR_PTR(-ENOMEM); ++ ret = cpuhp_state_add_instance_nocalls(io_wq_online, &wq->cpuhp_node); ++ if (ret) ++ goto err_wq; ++ ++ refcount_inc(&data->hash->refs); ++ wq->hash = data->hash; ++ wq->free_work = data->free_work; ++ wq->do_work = data->do_work; ++ ++ ret = -ENOMEM; ++ for_each_node(node) { ++ struct io_wqe *wqe; ++ int alloc_node = node; ++ ++ if (!node_online(alloc_node)) ++ alloc_node = NUMA_NO_NODE; ++ wqe = kzalloc_node(sizeof(struct io_wqe), GFP_KERNEL, alloc_node); ++ if (!wqe) ++ goto err; ++ wq->wqes[node] = wqe; ++ if (!alloc_cpumask_var(&wqe->cpu_mask, GFP_KERNEL)) ++ goto err; ++ cpumask_copy(wqe->cpu_mask, cpumask_of_node(node)); ++ wqe->node = alloc_node; ++ wqe->acct[IO_WQ_ACCT_BOUND].max_workers = bounded; ++ wqe->acct[IO_WQ_ACCT_UNBOUND].max_workers = ++ task_rlimit(current, RLIMIT_NPROC); ++ INIT_LIST_HEAD(&wqe->wait.entry); ++ wqe->wait.func = io_wqe_hash_wake; ++ for (i = 0; i < IO_WQ_ACCT_NR; i++) { ++ struct io_wqe_acct *acct = &wqe->acct[i]; ++ ++ acct->index = i; ++ atomic_set(&acct->nr_running, 0); ++ INIT_WQ_LIST(&acct->work_list); ++ } ++ wqe->wq = wq; ++ raw_spin_lock_init(&wqe->lock); ++ INIT_HLIST_NULLS_HEAD(&wqe->free_list, 0); ++ INIT_LIST_HEAD(&wqe->all_list); ++ } ++ ++ wq->task = get_task_struct(data->task); ++ atomic_set(&wq->worker_refs, 1); ++ init_completion(&wq->worker_done); ++ return wq; ++err: ++ io_wq_put_hash(data->hash); ++ cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); ++ for_each_node(node) { ++ if (!wq->wqes[node]) ++ continue; ++ free_cpumask_var(wq->wqes[node]->cpu_mask); ++ kfree(wq->wqes[node]); ++ } ++err_wq: ++ kfree(wq); ++ return ERR_PTR(ret); ++} ++ ++static bool io_task_work_match(struct callback_head *cb, void *data) ++{ ++ struct io_worker *worker; ++ ++ if (cb->func != create_worker_cb && cb->func != create_worker_cont) ++ return false; ++ worker = container_of(cb, struct io_worker, create_work); ++ return worker->wqe->wq == data; ++} ++ ++void io_wq_exit_start(struct io_wq *wq) ++{ ++ set_bit(IO_WQ_BIT_EXIT, &wq->state); ++} ++ ++static void io_wq_cancel_tw_create(struct io_wq *wq) ++{ ++ struct callback_head *cb; ++ ++ while ((cb = task_work_cancel_match(wq->task, io_task_work_match, wq)) != NULL) { ++ struct io_worker *worker; ++ ++ worker = container_of(cb, struct io_worker, create_work); ++ io_worker_cancel_cb(worker); ++ } ++} ++ ++static void io_wq_exit_workers(struct io_wq *wq) ++{ ++ int node; ++ ++ if (!wq->task) ++ return; ++ ++ io_wq_cancel_tw_create(wq); ++ ++ rcu_read_lock(); ++ for_each_node(node) { ++ struct io_wqe *wqe = wq->wqes[node]; ++ ++ io_wq_for_each_worker(wqe, io_wq_worker_wake, NULL); ++ } ++ rcu_read_unlock(); ++ io_worker_ref_put(wq); ++ wait_for_completion(&wq->worker_done); ++ ++ for_each_node(node) { ++ spin_lock_irq(&wq->hash->wait.lock); ++ list_del_init(&wq->wqes[node]->wait.entry); ++ spin_unlock_irq(&wq->hash->wait.lock); ++ } ++ put_task_struct(wq->task); ++ wq->task = NULL; ++} ++ ++static void io_wq_destroy(struct io_wq *wq) ++{ ++ int node; ++ ++ cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); ++ ++ for_each_node(node) { ++ struct io_wqe *wqe = wq->wqes[node]; ++ struct io_cb_cancel_data match = { ++ .fn = io_wq_work_match_all, ++ .cancel_all = true, ++ }; ++ io_wqe_cancel_pending_work(wqe, &match); ++ free_cpumask_var(wqe->cpu_mask); ++ kfree(wqe); ++ } ++ io_wq_put_hash(wq->hash); ++ kfree(wq); ++} ++ ++void io_wq_put_and_exit(struct io_wq *wq) ++{ ++ WARN_ON_ONCE(!test_bit(IO_WQ_BIT_EXIT, &wq->state)); ++ ++ io_wq_exit_workers(wq); ++ io_wq_destroy(wq); ++} ++ ++struct online_data { ++ unsigned int cpu; ++ bool online; ++}; ++ ++static bool io_wq_worker_affinity(struct io_worker *worker, void *data) ++{ ++ struct online_data *od = data; ++ ++ if (od->online) ++ cpumask_set_cpu(od->cpu, worker->wqe->cpu_mask); ++ else ++ cpumask_clear_cpu(od->cpu, worker->wqe->cpu_mask); ++ return false; ++} ++ ++static int __io_wq_cpu_online(struct io_wq *wq, unsigned int cpu, bool online) ++{ ++ struct online_data od = { ++ .cpu = cpu, ++ .online = online ++ }; ++ int i; ++ ++ rcu_read_lock(); ++ for_each_node(i) ++ io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, &od); ++ rcu_read_unlock(); ++ return 0; ++} ++ ++static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node) ++{ ++ struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); ++ ++ return __io_wq_cpu_online(wq, cpu, true); ++} ++ ++static int io_wq_cpu_offline(unsigned int cpu, struct hlist_node *node) ++{ ++ struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); ++ ++ return __io_wq_cpu_online(wq, cpu, false); ++} ++ ++int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask) ++{ ++ int i; ++ ++ rcu_read_lock(); ++ for_each_node(i) { ++ struct io_wqe *wqe = wq->wqes[i]; ++ ++ if (mask) ++ cpumask_copy(wqe->cpu_mask, mask); ++ else ++ cpumask_copy(wqe->cpu_mask, cpumask_of_node(i)); ++ } ++ rcu_read_unlock(); ++ return 0; ++} ++ ++/* ++ * Set max number of unbounded workers, returns old value. If new_count is 0, ++ * then just return the old value. ++ */ ++int io_wq_max_workers(struct io_wq *wq, int *new_count) ++{ ++ int prev[IO_WQ_ACCT_NR]; ++ bool first_node = true; ++ int i, node; ++ ++ BUILD_BUG_ON((int) IO_WQ_ACCT_BOUND != (int) IO_WQ_BOUND); ++ BUILD_BUG_ON((int) IO_WQ_ACCT_UNBOUND != (int) IO_WQ_UNBOUND); ++ BUILD_BUG_ON((int) IO_WQ_ACCT_NR != 2); ++ ++ for (i = 0; i < 2; i++) { ++ if (new_count[i] > task_rlimit(current, RLIMIT_NPROC)) ++ new_count[i] = task_rlimit(current, RLIMIT_NPROC); ++ } ++ ++ for (i = 0; i < IO_WQ_ACCT_NR; i++) ++ prev[i] = 0; ++ ++ rcu_read_lock(); ++ for_each_node(node) { ++ struct io_wqe *wqe = wq->wqes[node]; ++ struct io_wqe_acct *acct; ++ ++ raw_spin_lock(&wqe->lock); ++ for (i = 0; i < IO_WQ_ACCT_NR; i++) { ++ acct = &wqe->acct[i]; ++ if (first_node) ++ prev[i] = max_t(int, acct->max_workers, prev[i]); ++ if (new_count[i]) ++ acct->max_workers = new_count[i]; ++ } ++ raw_spin_unlock(&wqe->lock); ++ first_node = false; ++ } ++ rcu_read_unlock(); ++ ++ for (i = 0; i < IO_WQ_ACCT_NR; i++) ++ new_count[i] = prev[i]; ++ ++ return 0; ++} ++ ++static __init int io_wq_init(void) ++{ ++ int ret; ++ ++ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online", ++ io_wq_cpu_online, io_wq_cpu_offline); ++ if (ret < 0) ++ return ret; ++ io_wq_online = ret; ++ return 0; ++} ++subsys_initcall(io_wq_init); +--- /dev/null ++++ b/io_uring/io-wq.h +@@ -0,0 +1,160 @@ ++#ifndef INTERNAL_IO_WQ_H ++#define INTERNAL_IO_WQ_H ++ ++#include ++ ++struct io_wq; ++ ++enum { ++ IO_WQ_WORK_CANCEL = 1, ++ IO_WQ_WORK_HASHED = 2, ++ IO_WQ_WORK_UNBOUND = 4, ++ IO_WQ_WORK_CONCURRENT = 16, ++ ++ IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */ ++}; ++ ++enum io_wq_cancel { ++ IO_WQ_CANCEL_OK, /* cancelled before started */ ++ IO_WQ_CANCEL_RUNNING, /* found, running, and attempted cancelled */ ++ IO_WQ_CANCEL_NOTFOUND, /* work not found */ ++}; ++ ++struct io_wq_work_node { ++ struct io_wq_work_node *next; ++}; ++ ++struct io_wq_work_list { ++ struct io_wq_work_node *first; ++ struct io_wq_work_node *last; ++}; ++ ++static inline void wq_list_add_after(struct io_wq_work_node *node, ++ struct io_wq_work_node *pos, ++ struct io_wq_work_list *list) ++{ ++ struct io_wq_work_node *next = pos->next; ++ ++ pos->next = node; ++ node->next = next; ++ if (!next) ++ list->last = node; ++} ++ ++static inline void wq_list_add_tail(struct io_wq_work_node *node, ++ struct io_wq_work_list *list) ++{ ++ node->next = NULL; ++ if (!list->first) { ++ list->last = node; ++ WRITE_ONCE(list->first, node); ++ } else { ++ list->last->next = node; ++ list->last = node; ++ } ++} ++ ++static inline void wq_list_cut(struct io_wq_work_list *list, ++ struct io_wq_work_node *last, ++ struct io_wq_work_node *prev) ++{ ++ /* first in the list, if prev==NULL */ ++ if (!prev) ++ WRITE_ONCE(list->first, last->next); ++ else ++ prev->next = last->next; ++ ++ if (last == list->last) ++ list->last = prev; ++ last->next = NULL; ++} ++ ++static inline void wq_list_del(struct io_wq_work_list *list, ++ struct io_wq_work_node *node, ++ struct io_wq_work_node *prev) ++{ ++ wq_list_cut(list, node, prev); ++} ++ ++#define wq_list_for_each(pos, prv, head) \ ++ for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next) ++ ++#define wq_list_empty(list) (READ_ONCE((list)->first) == NULL) ++#define INIT_WQ_LIST(list) do { \ ++ (list)->first = NULL; \ ++ (list)->last = NULL; \ ++} while (0) ++ ++struct io_wq_work { ++ struct io_wq_work_node list; ++ unsigned flags; ++}; ++ ++static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) ++{ ++ if (!work->list.next) ++ return NULL; ++ ++ return container_of(work->list.next, struct io_wq_work, list); ++} ++ ++typedef struct io_wq_work *(free_work_fn)(struct io_wq_work *); ++typedef void (io_wq_work_fn)(struct io_wq_work *); ++ ++struct io_wq_hash { ++ refcount_t refs; ++ unsigned long map; ++ struct wait_queue_head wait; ++}; ++ ++static inline void io_wq_put_hash(struct io_wq_hash *hash) ++{ ++ if (refcount_dec_and_test(&hash->refs)) ++ kfree(hash); ++} ++ ++struct io_wq_data { ++ struct io_wq_hash *hash; ++ struct task_struct *task; ++ io_wq_work_fn *do_work; ++ free_work_fn *free_work; ++}; ++ ++struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); ++void io_wq_exit_start(struct io_wq *wq); ++void io_wq_put_and_exit(struct io_wq *wq); ++ ++void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); ++void io_wq_hash_work(struct io_wq_work *work, void *val); ++ ++int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask); ++int io_wq_max_workers(struct io_wq *wq, int *new_count); ++ ++static inline bool io_wq_is_hashed(struct io_wq_work *work) ++{ ++ return work->flags & IO_WQ_WORK_HASHED; ++} ++ ++typedef bool (work_cancel_fn)(struct io_wq_work *, void *); ++ ++enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, ++ void *data, bool cancel_all); ++ ++#if defined(CONFIG_IO_WQ) ++extern void io_wq_worker_sleeping(struct task_struct *); ++extern void io_wq_worker_running(struct task_struct *); ++#else ++static inline void io_wq_worker_sleeping(struct task_struct *tsk) ++{ ++} ++static inline void io_wq_worker_running(struct task_struct *tsk) ++{ ++} ++#endif ++ ++static inline bool io_wq_current_is_worker(void) ++{ ++ return in_task() && (current->flags & PF_IO_WORKER) && ++ current->pf_io_worker; ++} ++#endif +--- /dev/null ++++ b/io_uring/io_uring.c +@@ -0,0 +1,10945 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Shared application/kernel submission and completion ring pairs, for ++ * supporting fast/efficient IO. ++ * ++ * A note on the read/write ordering memory barriers that are matched between ++ * the application and kernel side. ++ * ++ * After the application reads the CQ ring tail, it must use an ++ * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses ++ * before writing the tail (using smp_load_acquire to read the tail will ++ * do). It also needs a smp_mb() before updating CQ head (ordering the ++ * entry load(s) with the head store), pairing with an implicit barrier ++ * through a control-dependency in io_get_cqe (smp_store_release to ++ * store head will do). Failure to do so could lead to reading invalid ++ * CQ entries. ++ * ++ * Likewise, the application must use an appropriate smp_wmb() before ++ * writing the SQ tail (ordering SQ entry stores with the tail store), ++ * which pairs with smp_load_acquire in io_get_sqring (smp_store_release ++ * to store the tail will do). And it needs a barrier ordering the SQ ++ * head load before writing new SQ entries (smp_load_acquire to read ++ * head will do). ++ * ++ * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application ++ * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after* ++ * updating the SQ tail; a full memory barrier smp_mb() is needed ++ * between. ++ * ++ * Also see the examples in the liburing library: ++ * ++ * git://git.kernel.dk/liburing ++ * ++ * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens ++ * from data shared between the kernel and application. This is done both ++ * for ordering purposes, but also to ensure that once a value is loaded from ++ * data that the application could potentially modify, it remains stable. ++ * ++ * Copyright (C) 2018-2019 Jens Axboe ++ * Copyright (c) 2018-2019 Christoph Hellwig ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define CREATE_TRACE_POINTS ++#include ++ ++#include ++ ++#include "../fs/internal.h" ++#include "io-wq.h" ++ ++#define IORING_MAX_ENTRIES 32768 ++#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) ++#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8 ++ ++/* only define max */ ++#define IORING_MAX_FIXED_FILES (1U << 15) ++#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ ++ IORING_REGISTER_LAST + IORING_OP_LAST) ++ ++#define IO_RSRC_TAG_TABLE_SHIFT (PAGE_SHIFT - 3) ++#define IO_RSRC_TAG_TABLE_MAX (1U << IO_RSRC_TAG_TABLE_SHIFT) ++#define IO_RSRC_TAG_TABLE_MASK (IO_RSRC_TAG_TABLE_MAX - 1) ++ ++#define IORING_MAX_REG_BUFFERS (1U << 14) ++ ++#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ ++ IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ ++ IOSQE_BUFFER_SELECT) ++#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \ ++ REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS) ++ ++#define IO_TCTX_REFS_CACHE_NR (1U << 10) ++ ++struct io_uring { ++ u32 head ____cacheline_aligned_in_smp; ++ u32 tail ____cacheline_aligned_in_smp; ++}; ++ ++/* ++ * This data is shared with the application through the mmap at offsets ++ * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING. ++ * ++ * The offsets to the member fields are published through struct ++ * io_sqring_offsets when calling io_uring_setup. ++ */ ++struct io_rings { ++ /* ++ * Head and tail offsets into the ring; the offsets need to be ++ * masked to get valid indices. ++ * ++ * The kernel controls head of the sq ring and the tail of the cq ring, ++ * and the application controls tail of the sq ring and the head of the ++ * cq ring. ++ */ ++ struct io_uring sq, cq; ++ /* ++ * Bitmasks to apply to head and tail offsets (constant, equals ++ * ring_entries - 1) ++ */ ++ u32 sq_ring_mask, cq_ring_mask; ++ /* Ring sizes (constant, power of 2) */ ++ u32 sq_ring_entries, cq_ring_entries; ++ /* ++ * Number of invalid entries dropped by the kernel due to ++ * invalid index stored in array ++ * ++ * Written by the kernel, shouldn't be modified by the ++ * application (i.e. get number of "new events" by comparing to ++ * cached value). ++ * ++ * After a new SQ head value was read by the application this ++ * counter includes all submissions that were dropped reaching ++ * the new SQ head (and possibly more). ++ */ ++ u32 sq_dropped; ++ /* ++ * Runtime SQ flags ++ * ++ * Written by the kernel, shouldn't be modified by the ++ * application. ++ * ++ * The application needs a full memory barrier before checking ++ * for IORING_SQ_NEED_WAKEUP after updating the sq tail. ++ */ ++ u32 sq_flags; ++ /* ++ * Runtime CQ flags ++ * ++ * Written by the application, shouldn't be modified by the ++ * kernel. ++ */ ++ u32 cq_flags; ++ /* ++ * Number of completion events lost because the queue was full; ++ * this should be avoided by the application by making sure ++ * there are not more requests pending than there is space in ++ * the completion queue. ++ * ++ * Written by the kernel, shouldn't be modified by the ++ * application (i.e. get number of "new events" by comparing to ++ * cached value). ++ * ++ * As completion events come in out of order this counter is not ++ * ordered with any other data. ++ */ ++ u32 cq_overflow; ++ /* ++ * Ring buffer of completion events. ++ * ++ * The kernel writes completion events fresh every time they are ++ * produced, so the application is allowed to modify pending ++ * entries. ++ */ ++ struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp; ++}; ++ ++enum io_uring_cmd_flags { ++ IO_URING_F_NONBLOCK = 1, ++ IO_URING_F_COMPLETE_DEFER = 2, ++}; ++ ++struct io_mapped_ubuf { ++ u64 ubuf; ++ u64 ubuf_end; ++ unsigned int nr_bvecs; ++ unsigned long acct_pages; ++ struct bio_vec bvec[]; ++}; ++ ++struct io_ring_ctx; ++ ++struct io_overflow_cqe { ++ struct io_uring_cqe cqe; ++ struct list_head list; ++}; ++ ++struct io_fixed_file { ++ /* file * with additional FFS_* flags */ ++ unsigned long file_ptr; ++}; ++ ++struct io_rsrc_put { ++ struct list_head list; ++ u64 tag; ++ union { ++ void *rsrc; ++ struct file *file; ++ struct io_mapped_ubuf *buf; ++ }; ++}; ++ ++struct io_file_table { ++ struct io_fixed_file *files; ++}; ++ ++struct io_rsrc_node { ++ struct percpu_ref refs; ++ struct list_head node; ++ struct list_head rsrc_list; ++ struct io_rsrc_data *rsrc_data; ++ struct llist_node llist; ++ bool done; ++}; ++ ++typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc); ++ ++struct io_rsrc_data { ++ struct io_ring_ctx *ctx; ++ ++ u64 **tags; ++ unsigned int nr; ++ rsrc_put_fn *do_put; ++ atomic_t refs; ++ struct completion done; ++ bool quiesce; ++}; ++ ++struct io_buffer { ++ struct list_head list; ++ __u64 addr; ++ __u32 len; ++ __u16 bid; ++}; ++ ++struct io_restriction { ++ DECLARE_BITMAP(register_op, IORING_REGISTER_LAST); ++ DECLARE_BITMAP(sqe_op, IORING_OP_LAST); ++ u8 sqe_flags_allowed; ++ u8 sqe_flags_required; ++ bool registered; ++}; ++ ++enum { ++ IO_SQ_THREAD_SHOULD_STOP = 0, ++ IO_SQ_THREAD_SHOULD_PARK, ++}; ++ ++struct io_sq_data { ++ refcount_t refs; ++ atomic_t park_pending; ++ struct mutex lock; ++ ++ /* ctx's that are using this sqd */ ++ struct list_head ctx_list; ++ ++ struct task_struct *thread; ++ struct wait_queue_head wait; ++ ++ unsigned sq_thread_idle; ++ int sq_cpu; ++ pid_t task_pid; ++ pid_t task_tgid; ++ ++ unsigned long state; ++ struct completion exited; ++}; ++ ++#define IO_COMPL_BATCH 32 ++#define IO_REQ_CACHE_SIZE 32 ++#define IO_REQ_ALLOC_BATCH 8 ++ ++struct io_submit_link { ++ struct io_kiocb *head; ++ struct io_kiocb *last; ++}; ++ ++struct io_submit_state { ++ struct blk_plug plug; ++ struct io_submit_link link; ++ ++ /* ++ * io_kiocb alloc cache ++ */ ++ void *reqs[IO_REQ_CACHE_SIZE]; ++ unsigned int free_reqs; ++ ++ bool plug_started; ++ ++ /* ++ * Batch completion logic ++ */ ++ struct io_kiocb *compl_reqs[IO_COMPL_BATCH]; ++ unsigned int compl_nr; ++ /* inline/task_work completion list, under ->uring_lock */ ++ struct list_head free_list; ++ ++ unsigned int ios_left; ++}; ++ ++struct io_ring_ctx { ++ /* const or read-mostly hot data */ ++ struct { ++ struct percpu_ref refs; ++ ++ struct io_rings *rings; ++ unsigned int flags; ++ unsigned int compat: 1; ++ unsigned int drain_next: 1; ++ unsigned int eventfd_async: 1; ++ unsigned int restricted: 1; ++ unsigned int off_timeout_used: 1; ++ unsigned int drain_active: 1; ++ } ____cacheline_aligned_in_smp; ++ ++ /* submission data */ ++ struct { ++ struct mutex uring_lock; ++ ++ /* ++ * Ring buffer of indices into array of io_uring_sqe, which is ++ * mmapped by the application using the IORING_OFF_SQES offset. ++ * ++ * This indirection could e.g. be used to assign fixed ++ * io_uring_sqe entries to operations and only submit them to ++ * the queue when needed. ++ * ++ * The kernel modifies neither the indices array nor the entries ++ * array. ++ */ ++ u32 *sq_array; ++ struct io_uring_sqe *sq_sqes; ++ unsigned cached_sq_head; ++ unsigned sq_entries; ++ struct list_head defer_list; ++ ++ /* ++ * Fixed resources fast path, should be accessed only under ++ * uring_lock, and updated through io_uring_register(2) ++ */ ++ struct io_rsrc_node *rsrc_node; ++ struct io_file_table file_table; ++ unsigned nr_user_files; ++ unsigned nr_user_bufs; ++ struct io_mapped_ubuf **user_bufs; ++ ++ struct io_submit_state submit_state; ++ struct list_head timeout_list; ++ struct list_head ltimeout_list; ++ struct list_head cq_overflow_list; ++ struct xarray io_buffers; ++ struct xarray personalities; ++ u32 pers_next; ++ unsigned sq_thread_idle; ++ } ____cacheline_aligned_in_smp; ++ ++ /* IRQ completion list, under ->completion_lock */ ++ struct list_head locked_free_list; ++ unsigned int locked_free_nr; ++ ++ const struct cred *sq_creds; /* cred used for __io_sq_thread() */ ++ struct io_sq_data *sq_data; /* if using sq thread polling */ ++ ++ struct wait_queue_head sqo_sq_wait; ++ struct list_head sqd_list; ++ ++ unsigned long check_cq_overflow; ++ ++ struct { ++ unsigned cached_cq_tail; ++ unsigned cq_entries; ++ struct eventfd_ctx *cq_ev_fd; ++ struct wait_queue_head poll_wait; ++ struct wait_queue_head cq_wait; ++ unsigned cq_extra; ++ atomic_t cq_timeouts; ++ unsigned cq_last_tm_flush; ++ } ____cacheline_aligned_in_smp; ++ ++ struct { ++ spinlock_t completion_lock; ++ ++ spinlock_t timeout_lock; ++ ++ /* ++ * ->iopoll_list is protected by the ctx->uring_lock for ++ * io_uring instances that don't use IORING_SETUP_SQPOLL. ++ * For SQPOLL, only the single threaded io_sq_thread() will ++ * manipulate the list, hence no extra locking is needed there. ++ */ ++ struct list_head iopoll_list; ++ struct hlist_head *cancel_hash; ++ unsigned cancel_hash_bits; ++ bool poll_multi_queue; ++ } ____cacheline_aligned_in_smp; ++ ++ struct io_restriction restrictions; ++ ++ /* slow path rsrc auxilary data, used by update/register */ ++ struct { ++ struct io_rsrc_node *rsrc_backup_node; ++ struct io_mapped_ubuf *dummy_ubuf; ++ struct io_rsrc_data *file_data; ++ struct io_rsrc_data *buf_data; ++ ++ struct delayed_work rsrc_put_work; ++ struct llist_head rsrc_put_llist; ++ struct list_head rsrc_ref_list; ++ spinlock_t rsrc_ref_lock; ++ }; ++ ++ /* Keep this last, we don't need it for the fast path */ ++ struct { ++ #if defined(CONFIG_UNIX) ++ struct socket *ring_sock; ++ #endif ++ /* hashed buffered write serialization */ ++ struct io_wq_hash *hash_map; ++ ++ /* Only used for accounting purposes */ ++ struct user_struct *user; ++ struct mm_struct *mm_account; ++ ++ /* ctx exit and cancelation */ ++ struct llist_head fallback_llist; ++ struct delayed_work fallback_work; ++ struct work_struct exit_work; ++ struct list_head tctx_list; ++ struct completion ref_comp; ++ u32 iowq_limits[2]; ++ bool iowq_limits_set; ++ }; ++}; ++ ++struct io_uring_task { ++ /* submission side */ ++ int cached_refs; ++ struct xarray xa; ++ struct wait_queue_head wait; ++ const struct io_ring_ctx *last; ++ struct io_wq *io_wq; ++ struct percpu_counter inflight; ++ atomic_t inflight_tracked; ++ atomic_t in_idle; ++ ++ spinlock_t task_lock; ++ struct io_wq_work_list task_list; ++ struct callback_head task_work; ++ bool task_running; ++}; ++ ++/* ++ * First field must be the file pointer in all the ++ * iocb unions! See also 'struct kiocb' in ++ */ ++struct io_poll_iocb { ++ struct file *file; ++ struct wait_queue_head *head; ++ __poll_t events; ++ struct wait_queue_entry wait; ++}; ++ ++struct io_poll_update { ++ struct file *file; ++ u64 old_user_data; ++ u64 new_user_data; ++ __poll_t events; ++ bool update_events; ++ bool update_user_data; ++}; ++ ++struct io_close { ++ struct file *file; ++ int fd; ++ u32 file_slot; ++}; ++ ++struct io_timeout_data { ++ struct io_kiocb *req; ++ struct hrtimer timer; ++ struct timespec64 ts; ++ enum hrtimer_mode mode; ++ u32 flags; ++}; ++ ++struct io_accept { ++ struct file *file; ++ struct sockaddr __user *addr; ++ int __user *addr_len; ++ int flags; ++ u32 file_slot; ++ unsigned long nofile; ++}; ++ ++struct io_sync { ++ struct file *file; ++ loff_t len; ++ loff_t off; ++ int flags; ++ int mode; ++}; ++ ++struct io_cancel { ++ struct file *file; ++ u64 addr; ++}; ++ ++struct io_timeout { ++ struct file *file; ++ u32 off; ++ u32 target_seq; ++ struct list_head list; ++ /* head of the link, used by linked timeouts only */ ++ struct io_kiocb *head; ++ /* for linked completions */ ++ struct io_kiocb *prev; ++}; ++ ++struct io_timeout_rem { ++ struct file *file; ++ u64 addr; ++ ++ /* timeout update */ ++ struct timespec64 ts; ++ u32 flags; ++ bool ltimeout; ++}; ++ ++struct io_rw { ++ /* NOTE: kiocb has the file as the first member, so don't do it here */ ++ struct kiocb kiocb; ++ u64 addr; ++ u64 len; ++}; ++ ++struct io_connect { ++ struct file *file; ++ struct sockaddr __user *addr; ++ int addr_len; ++}; ++ ++struct io_sr_msg { ++ struct file *file; ++ union { ++ struct compat_msghdr __user *umsg_compat; ++ struct user_msghdr __user *umsg; ++ void __user *buf; ++ }; ++ int msg_flags; ++ int bgid; ++ size_t len; ++ struct io_buffer *kbuf; ++}; ++ ++struct io_open { ++ struct file *file; ++ int dfd; ++ u32 file_slot; ++ struct filename *filename; ++ struct open_how how; ++ unsigned long nofile; ++}; ++ ++struct io_rsrc_update { ++ struct file *file; ++ u64 arg; ++ u32 nr_args; ++ u32 offset; ++}; ++ ++struct io_fadvise { ++ struct file *file; ++ u64 offset; ++ u32 len; ++ u32 advice; ++}; ++ ++struct io_madvise { ++ struct file *file; ++ u64 addr; ++ u32 len; ++ u32 advice; ++}; ++ ++struct io_epoll { ++ struct file *file; ++ int epfd; ++ int op; ++ int fd; ++ struct epoll_event event; ++}; ++ ++struct io_splice { ++ struct file *file_out; ++ loff_t off_out; ++ loff_t off_in; ++ u64 len; ++ int splice_fd_in; ++ unsigned int flags; ++}; ++ ++struct io_provide_buf { ++ struct file *file; ++ __u64 addr; ++ __u32 len; ++ __u32 bgid; ++ __u16 nbufs; ++ __u16 bid; ++}; ++ ++struct io_statx { ++ struct file *file; ++ int dfd; ++ unsigned int mask; ++ unsigned int flags; ++ const char __user *filename; ++ struct statx __user *buffer; ++}; ++ ++struct io_shutdown { ++ struct file *file; ++ int how; ++}; ++ ++struct io_rename { ++ struct file *file; ++ int old_dfd; ++ int new_dfd; ++ struct filename *oldpath; ++ struct filename *newpath; ++ int flags; ++}; ++ ++struct io_unlink { ++ struct file *file; ++ int dfd; ++ int flags; ++ struct filename *filename; ++}; ++ ++struct io_mkdir { ++ struct file *file; ++ int dfd; ++ umode_t mode; ++ struct filename *filename; ++}; ++ ++struct io_symlink { ++ struct file *file; ++ int new_dfd; ++ struct filename *oldpath; ++ struct filename *newpath; ++}; ++ ++struct io_hardlink { ++ struct file *file; ++ int old_dfd; ++ int new_dfd; ++ struct filename *oldpath; ++ struct filename *newpath; ++ int flags; ++}; ++ ++struct io_completion { ++ struct file *file; ++ u32 cflags; ++}; ++ ++struct io_async_connect { ++ struct sockaddr_storage address; ++}; ++ ++struct io_async_msghdr { ++ struct iovec fast_iov[UIO_FASTIOV]; ++ /* points to an allocated iov, if NULL we use fast_iov instead */ ++ struct iovec *free_iov; ++ struct sockaddr __user *uaddr; ++ struct msghdr msg; ++ struct sockaddr_storage addr; ++}; ++ ++struct io_async_rw { ++ struct iovec fast_iov[UIO_FASTIOV]; ++ const struct iovec *free_iovec; ++ struct iov_iter iter; ++ struct iov_iter_state iter_state; ++ size_t bytes_done; ++ struct wait_page_queue wpq; ++}; ++ ++enum { ++ REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT, ++ REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT, ++ REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT, ++ REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT, ++ REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT, ++ REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT, ++ ++ /* first byte is taken by user flags, shift it to not overlap */ ++ REQ_F_FAIL_BIT = 8, ++ REQ_F_INFLIGHT_BIT, ++ REQ_F_CUR_POS_BIT, ++ REQ_F_NOWAIT_BIT, ++ REQ_F_LINK_TIMEOUT_BIT, ++ REQ_F_NEED_CLEANUP_BIT, ++ REQ_F_POLLED_BIT, ++ REQ_F_BUFFER_SELECTED_BIT, ++ REQ_F_COMPLETE_INLINE_BIT, ++ REQ_F_REISSUE_BIT, ++ REQ_F_CREDS_BIT, ++ REQ_F_REFCOUNT_BIT, ++ REQ_F_ARM_LTIMEOUT_BIT, ++ /* keep async read/write and isreg together and in order */ ++ REQ_F_NOWAIT_READ_BIT, ++ REQ_F_NOWAIT_WRITE_BIT, ++ REQ_F_ISREG_BIT, ++ ++ /* not a real bit, just to check we're not overflowing the space */ ++ __REQ_F_LAST_BIT, ++}; ++ ++enum { ++ /* ctx owns file */ ++ REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT), ++ /* drain existing IO first */ ++ REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT), ++ /* linked sqes */ ++ REQ_F_LINK = BIT(REQ_F_LINK_BIT), ++ /* doesn't sever on completion < 0 */ ++ REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT), ++ /* IOSQE_ASYNC */ ++ REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT), ++ /* IOSQE_BUFFER_SELECT */ ++ REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT), ++ ++ /* fail rest of links */ ++ REQ_F_FAIL = BIT(REQ_F_FAIL_BIT), ++ /* on inflight list, should be cancelled and waited on exit reliably */ ++ REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT), ++ /* read/write uses file position */ ++ REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT), ++ /* must not punt to workers */ ++ REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT), ++ /* has or had linked timeout */ ++ REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT), ++ /* needs cleanup */ ++ REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT), ++ /* already went through poll handler */ ++ REQ_F_POLLED = BIT(REQ_F_POLLED_BIT), ++ /* buffer already selected */ ++ REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT), ++ /* completion is deferred through io_comp_state */ ++ REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT), ++ /* caller should reissue async */ ++ REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT), ++ /* supports async reads */ ++ REQ_F_NOWAIT_READ = BIT(REQ_F_NOWAIT_READ_BIT), ++ /* supports async writes */ ++ REQ_F_NOWAIT_WRITE = BIT(REQ_F_NOWAIT_WRITE_BIT), ++ /* regular file */ ++ REQ_F_ISREG = BIT(REQ_F_ISREG_BIT), ++ /* has creds assigned */ ++ REQ_F_CREDS = BIT(REQ_F_CREDS_BIT), ++ /* skip refcounting if not set */ ++ REQ_F_REFCOUNT = BIT(REQ_F_REFCOUNT_BIT), ++ /* there is a linked timeout that has to be armed */ ++ REQ_F_ARM_LTIMEOUT = BIT(REQ_F_ARM_LTIMEOUT_BIT), ++}; ++ ++struct async_poll { ++ struct io_poll_iocb poll; ++ struct io_poll_iocb *double_poll; ++}; ++ ++typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked); ++ ++struct io_task_work { ++ union { ++ struct io_wq_work_node node; ++ struct llist_node fallback_node; ++ }; ++ io_req_tw_func_t func; ++}; ++ ++enum { ++ IORING_RSRC_FILE = 0, ++ IORING_RSRC_BUFFER = 1, ++}; ++ ++/* ++ * NOTE! Each of the iocb union members has the file pointer ++ * as the first entry in their struct definition. So you can ++ * access the file pointer through any of the sub-structs, ++ * or directly as just 'ki_filp' in this struct. ++ */ ++struct io_kiocb { ++ union { ++ struct file *file; ++ struct io_rw rw; ++ struct io_poll_iocb poll; ++ struct io_poll_update poll_update; ++ struct io_accept accept; ++ struct io_sync sync; ++ struct io_cancel cancel; ++ struct io_timeout timeout; ++ struct io_timeout_rem timeout_rem; ++ struct io_connect connect; ++ struct io_sr_msg sr_msg; ++ struct io_open open; ++ struct io_close close; ++ struct io_rsrc_update rsrc_update; ++ struct io_fadvise fadvise; ++ struct io_madvise madvise; ++ struct io_epoll epoll; ++ struct io_splice splice; ++ struct io_provide_buf pbuf; ++ struct io_statx statx; ++ struct io_shutdown shutdown; ++ struct io_rename rename; ++ struct io_unlink unlink; ++ struct io_mkdir mkdir; ++ struct io_symlink symlink; ++ struct io_hardlink hardlink; ++ /* use only after cleaning per-op data, see io_clean_op() */ ++ struct io_completion compl; ++ }; ++ ++ /* opcode allocated if it needs to store data for async defer */ ++ void *async_data; ++ u8 opcode; ++ /* polled IO has completed */ ++ u8 iopoll_completed; ++ ++ u16 buf_index; ++ u32 result; ++ ++ struct io_ring_ctx *ctx; ++ unsigned int flags; ++ atomic_t refs; ++ struct task_struct *task; ++ u64 user_data; ++ ++ struct io_kiocb *link; ++ struct percpu_ref *fixed_rsrc_refs; ++ ++ /* used with ctx->iopoll_list with reads/writes */ ++ struct list_head inflight_entry; ++ struct io_task_work io_task_work; ++ /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ ++ struct hlist_node hash_node; ++ struct async_poll *apoll; ++ struct io_wq_work work; ++ const struct cred *creds; ++ ++ /* store used ubuf, so we can prevent reloading */ ++ struct io_mapped_ubuf *imu; ++ /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */ ++ struct io_buffer *kbuf; ++ atomic_t poll_refs; ++}; ++ ++struct io_tctx_node { ++ struct list_head ctx_node; ++ struct task_struct *task; ++ struct io_ring_ctx *ctx; ++}; ++ ++struct io_defer_entry { ++ struct list_head list; ++ struct io_kiocb *req; ++ u32 seq; ++}; ++ ++struct io_op_def { ++ /* needs req->file assigned */ ++ unsigned needs_file : 1; ++ /* hash wq insertion if file is a regular file */ ++ unsigned hash_reg_file : 1; ++ /* unbound wq insertion if file is a non-regular file */ ++ unsigned unbound_nonreg_file : 1; ++ /* opcode is not supported by this kernel */ ++ unsigned not_supported : 1; ++ /* set if opcode supports polled "wait" */ ++ unsigned pollin : 1; ++ unsigned pollout : 1; ++ /* op supports buffer selection */ ++ unsigned buffer_select : 1; ++ /* do prep async if is going to be punted */ ++ unsigned needs_async_setup : 1; ++ /* should block plug */ ++ unsigned plug : 1; ++ /* size of async data needed, if any */ ++ unsigned short async_size; ++}; ++ ++static const struct io_op_def io_op_defs[] = { ++ [IORING_OP_NOP] = {}, ++ [IORING_OP_READV] = { ++ .needs_file = 1, ++ .unbound_nonreg_file = 1, ++ .pollin = 1, ++ .buffer_select = 1, ++ .needs_async_setup = 1, ++ .plug = 1, ++ .async_size = sizeof(struct io_async_rw), ++ }, ++ [IORING_OP_WRITEV] = { ++ .needs_file = 1, ++ .hash_reg_file = 1, ++ .unbound_nonreg_file = 1, ++ .pollout = 1, ++ .needs_async_setup = 1, ++ .plug = 1, ++ .async_size = sizeof(struct io_async_rw), ++ }, ++ [IORING_OP_FSYNC] = { ++ .needs_file = 1, ++ }, ++ [IORING_OP_READ_FIXED] = { ++ .needs_file = 1, ++ .unbound_nonreg_file = 1, ++ .pollin = 1, ++ .plug = 1, ++ .async_size = sizeof(struct io_async_rw), ++ }, ++ [IORING_OP_WRITE_FIXED] = { ++ .needs_file = 1, ++ .hash_reg_file = 1, ++ .unbound_nonreg_file = 1, ++ .pollout = 1, ++ .plug = 1, ++ .async_size = sizeof(struct io_async_rw), ++ }, ++ [IORING_OP_POLL_ADD] = { ++ .needs_file = 1, ++ .unbound_nonreg_file = 1, ++ }, ++ [IORING_OP_POLL_REMOVE] = {}, ++ [IORING_OP_SYNC_FILE_RANGE] = { ++ .needs_file = 1, ++ }, ++ [IORING_OP_SENDMSG] = { ++ .needs_file = 1, ++ .unbound_nonreg_file = 1, ++ .pollout = 1, ++ .needs_async_setup = 1, ++ .async_size = sizeof(struct io_async_msghdr), ++ }, ++ [IORING_OP_RECVMSG] = { ++ .needs_file = 1, ++ .unbound_nonreg_file = 1, ++ .pollin = 1, ++ .buffer_select = 1, ++ .needs_async_setup = 1, ++ .async_size = sizeof(struct io_async_msghdr), ++ }, ++ [IORING_OP_TIMEOUT] = { ++ .async_size = sizeof(struct io_timeout_data), ++ }, ++ [IORING_OP_TIMEOUT_REMOVE] = { ++ /* used by timeout updates' prep() */ ++ }, ++ [IORING_OP_ACCEPT] = { ++ .needs_file = 1, ++ .unbound_nonreg_file = 1, ++ .pollin = 1, ++ }, ++ [IORING_OP_ASYNC_CANCEL] = {}, ++ [IORING_OP_LINK_TIMEOUT] = { ++ .async_size = sizeof(struct io_timeout_data), ++ }, ++ [IORING_OP_CONNECT] = { ++ .needs_file = 1, ++ .unbound_nonreg_file = 1, ++ .pollout = 1, ++ .needs_async_setup = 1, ++ .async_size = sizeof(struct io_async_connect), ++ }, ++ [IORING_OP_FALLOCATE] = { ++ .needs_file = 1, ++ }, ++ [IORING_OP_OPENAT] = {}, ++ [IORING_OP_CLOSE] = {}, ++ [IORING_OP_FILES_UPDATE] = {}, ++ [IORING_OP_STATX] = {}, ++ [IORING_OP_READ] = { ++ .needs_file = 1, ++ .unbound_nonreg_file = 1, ++ .pollin = 1, ++ .buffer_select = 1, ++ .plug = 1, ++ .async_size = sizeof(struct io_async_rw), ++ }, ++ [IORING_OP_WRITE] = { ++ .needs_file = 1, ++ .hash_reg_file = 1, ++ .unbound_nonreg_file = 1, ++ .pollout = 1, ++ .plug = 1, ++ .async_size = sizeof(struct io_async_rw), ++ }, ++ [IORING_OP_FADVISE] = { ++ .needs_file = 1, ++ }, ++ [IORING_OP_MADVISE] = {}, ++ [IORING_OP_SEND] = { ++ .needs_file = 1, ++ .unbound_nonreg_file = 1, ++ .pollout = 1, ++ }, ++ [IORING_OP_RECV] = { ++ .needs_file = 1, ++ .unbound_nonreg_file = 1, ++ .pollin = 1, ++ .buffer_select = 1, ++ }, ++ [IORING_OP_OPENAT2] = { ++ }, ++ [IORING_OP_EPOLL_CTL] = { ++ .unbound_nonreg_file = 1, ++ }, ++ [IORING_OP_SPLICE] = { ++ .needs_file = 1, ++ .hash_reg_file = 1, ++ .unbound_nonreg_file = 1, ++ }, ++ [IORING_OP_PROVIDE_BUFFERS] = {}, ++ [IORING_OP_REMOVE_BUFFERS] = {}, ++ [IORING_OP_TEE] = { ++ .needs_file = 1, ++ .hash_reg_file = 1, ++ .unbound_nonreg_file = 1, ++ }, ++ [IORING_OP_SHUTDOWN] = { ++ .needs_file = 1, ++ }, ++ [IORING_OP_RENAMEAT] = {}, ++ [IORING_OP_UNLINKAT] = {}, ++}; ++ ++/* requests with any of those set should undergo io_disarm_next() */ ++#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL) ++ ++static bool io_disarm_next(struct io_kiocb *req); ++static void io_uring_del_tctx_node(unsigned long index); ++static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, ++ struct task_struct *task, ++ bool cancel_all); ++static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd); ++ ++static void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags); ++ ++static void io_put_req(struct io_kiocb *req); ++static void io_put_req_deferred(struct io_kiocb *req); ++static void io_dismantle_req(struct io_kiocb *req); ++static void io_queue_linked_timeout(struct io_kiocb *req); ++static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, ++ struct io_uring_rsrc_update2 *up, ++ unsigned nr_args); ++static void io_clean_op(struct io_kiocb *req); ++static struct file *io_file_get(struct io_ring_ctx *ctx, ++ struct io_kiocb *req, int fd, bool fixed); ++static void __io_queue_sqe(struct io_kiocb *req); ++static void io_rsrc_put_work(struct work_struct *work); ++ ++static void io_req_task_queue(struct io_kiocb *req); ++static void io_submit_flush_completions(struct io_ring_ctx *ctx); ++static int io_req_prep_async(struct io_kiocb *req); ++ ++static int io_install_fixed_file(struct io_kiocb *req, struct file *file, ++ unsigned int issue_flags, u32 slot_index); ++static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags); ++ ++static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer); ++ ++static struct kmem_cache *req_cachep; ++ ++static const struct file_operations io_uring_fops; ++ ++struct sock *io_uring_get_socket(struct file *file) ++{ ++#if defined(CONFIG_UNIX) ++ if (file->f_op == &io_uring_fops) { ++ struct io_ring_ctx *ctx = file->private_data; ++ ++ return ctx->ring_sock->sk; ++ } ++#endif ++ return NULL; ++} ++EXPORT_SYMBOL(io_uring_get_socket); ++ ++static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked) ++{ ++ if (!*locked) { ++ mutex_lock(&ctx->uring_lock); ++ *locked = true; ++ } ++} ++ ++#define io_for_each_link(pos, head) \ ++ for (pos = (head); pos; pos = pos->link) ++ ++/* ++ * Shamelessly stolen from the mm implementation of page reference checking, ++ * see commit f958d7b528b1 for details. ++ */ ++#define req_ref_zero_or_close_to_overflow(req) \ ++ ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u) ++ ++static inline bool req_ref_inc_not_zero(struct io_kiocb *req) ++{ ++ WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT)); ++ return atomic_inc_not_zero(&req->refs); ++} ++ ++static inline bool req_ref_put_and_test(struct io_kiocb *req) ++{ ++ if (likely(!(req->flags & REQ_F_REFCOUNT))) ++ return true; ++ ++ WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req)); ++ return atomic_dec_and_test(&req->refs); ++} ++ ++static inline void req_ref_get(struct io_kiocb *req) ++{ ++ WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT)); ++ WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req)); ++ atomic_inc(&req->refs); ++} ++ ++static inline void __io_req_set_refcount(struct io_kiocb *req, int nr) ++{ ++ if (!(req->flags & REQ_F_REFCOUNT)) { ++ req->flags |= REQ_F_REFCOUNT; ++ atomic_set(&req->refs, nr); ++ } ++} ++ ++static inline void io_req_set_refcount(struct io_kiocb *req) ++{ ++ __io_req_set_refcount(req, 1); ++} ++ ++static inline void io_req_set_rsrc_node(struct io_kiocb *req) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ if (!req->fixed_rsrc_refs) { ++ req->fixed_rsrc_refs = &ctx->rsrc_node->refs; ++ percpu_ref_get(req->fixed_rsrc_refs); ++ } ++} ++ ++static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl) ++{ ++ bool got = percpu_ref_tryget(ref); ++ ++ /* already at zero, wait for ->release() */ ++ if (!got) ++ wait_for_completion(compl); ++ percpu_ref_resurrect(ref); ++ if (got) ++ percpu_ref_put(ref); ++} ++ ++static bool io_match_task(struct io_kiocb *head, struct task_struct *task, ++ bool cancel_all) ++ __must_hold(&req->ctx->timeout_lock) ++{ ++ struct io_kiocb *req; ++ ++ if (task && head->task != task) ++ return false; ++ if (cancel_all) ++ return true; ++ ++ io_for_each_link(req, head) { ++ if (req->flags & REQ_F_INFLIGHT) ++ return true; ++ } ++ return false; ++} ++ ++static bool io_match_linked(struct io_kiocb *head) ++{ ++ struct io_kiocb *req; ++ ++ io_for_each_link(req, head) { ++ if (req->flags & REQ_F_INFLIGHT) ++ return true; ++ } ++ return false; ++} ++ ++/* ++ * As io_match_task() but protected against racing with linked timeouts. ++ * User must not hold timeout_lock. ++ */ ++static bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task, ++ bool cancel_all) ++{ ++ bool matched; ++ ++ if (task && head->task != task) ++ return false; ++ if (cancel_all) ++ return true; ++ ++ if (head->flags & REQ_F_LINK_TIMEOUT) { ++ struct io_ring_ctx *ctx = head->ctx; ++ ++ /* protect against races with linked timeouts */ ++ spin_lock_irq(&ctx->timeout_lock); ++ matched = io_match_linked(head); ++ spin_unlock_irq(&ctx->timeout_lock); ++ } else { ++ matched = io_match_linked(head); ++ } ++ return matched; ++} ++ ++static inline void req_set_fail(struct io_kiocb *req) ++{ ++ req->flags |= REQ_F_FAIL; ++} ++ ++static inline void req_fail_link_node(struct io_kiocb *req, int res) ++{ ++ req_set_fail(req); ++ req->result = res; ++} ++ ++static void io_ring_ctx_ref_free(struct percpu_ref *ref) ++{ ++ struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); ++ ++ complete(&ctx->ref_comp); ++} ++ ++static inline bool io_is_timeout_noseq(struct io_kiocb *req) ++{ ++ return !req->timeout.off; ++} ++ ++static void io_fallback_req_func(struct work_struct *work) ++{ ++ struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, ++ fallback_work.work); ++ struct llist_node *node = llist_del_all(&ctx->fallback_llist); ++ struct io_kiocb *req, *tmp; ++ bool locked = false; ++ ++ percpu_ref_get(&ctx->refs); ++ llist_for_each_entry_safe(req, tmp, node, io_task_work.fallback_node) ++ req->io_task_work.func(req, &locked); ++ ++ if (locked) { ++ if (ctx->submit_state.compl_nr) ++ io_submit_flush_completions(ctx); ++ mutex_unlock(&ctx->uring_lock); ++ } ++ percpu_ref_put(&ctx->refs); ++ ++} ++ ++static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) ++{ ++ struct io_ring_ctx *ctx; ++ int hash_bits; ++ ++ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); ++ if (!ctx) ++ return NULL; ++ ++ /* ++ * Use 5 bits less than the max cq entries, that should give us around ++ * 32 entries per hash list if totally full and uniformly spread. ++ */ ++ hash_bits = ilog2(p->cq_entries); ++ hash_bits -= 5; ++ if (hash_bits <= 0) ++ hash_bits = 1; ++ ctx->cancel_hash_bits = hash_bits; ++ ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head), ++ GFP_KERNEL); ++ if (!ctx->cancel_hash) ++ goto err; ++ __hash_init(ctx->cancel_hash, 1U << hash_bits); ++ ++ ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL); ++ if (!ctx->dummy_ubuf) ++ goto err; ++ /* set invalid range, so io_import_fixed() fails meeting it */ ++ ctx->dummy_ubuf->ubuf = -1UL; ++ ++ if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free, ++ PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) ++ goto err; ++ ++ ctx->flags = p->flags; ++ init_waitqueue_head(&ctx->sqo_sq_wait); ++ INIT_LIST_HEAD(&ctx->sqd_list); ++ init_waitqueue_head(&ctx->poll_wait); ++ INIT_LIST_HEAD(&ctx->cq_overflow_list); ++ init_completion(&ctx->ref_comp); ++ xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1); ++ xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1); ++ mutex_init(&ctx->uring_lock); ++ init_waitqueue_head(&ctx->cq_wait); ++ spin_lock_init(&ctx->completion_lock); ++ spin_lock_init(&ctx->timeout_lock); ++ INIT_LIST_HEAD(&ctx->iopoll_list); ++ INIT_LIST_HEAD(&ctx->defer_list); ++ INIT_LIST_HEAD(&ctx->timeout_list); ++ INIT_LIST_HEAD(&ctx->ltimeout_list); ++ spin_lock_init(&ctx->rsrc_ref_lock); ++ INIT_LIST_HEAD(&ctx->rsrc_ref_list); ++ INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); ++ init_llist_head(&ctx->rsrc_put_llist); ++ INIT_LIST_HEAD(&ctx->tctx_list); ++ INIT_LIST_HEAD(&ctx->submit_state.free_list); ++ INIT_LIST_HEAD(&ctx->locked_free_list); ++ INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func); ++ return ctx; ++err: ++ kfree(ctx->dummy_ubuf); ++ kfree(ctx->cancel_hash); ++ kfree(ctx); ++ return NULL; ++} ++ ++static void io_account_cq_overflow(struct io_ring_ctx *ctx) ++{ ++ struct io_rings *r = ctx->rings; ++ ++ WRITE_ONCE(r->cq_overflow, READ_ONCE(r->cq_overflow) + 1); ++ ctx->cq_extra--; ++} ++ ++static bool req_need_defer(struct io_kiocb *req, u32 seq) ++{ ++ if (unlikely(req->flags & REQ_F_IO_DRAIN)) { ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ return seq + READ_ONCE(ctx->cq_extra) != ctx->cached_cq_tail; ++ } ++ ++ return false; ++} ++ ++#define FFS_ASYNC_READ 0x1UL ++#define FFS_ASYNC_WRITE 0x2UL ++#ifdef CONFIG_64BIT ++#define FFS_ISREG 0x4UL ++#else ++#define FFS_ISREG 0x0UL ++#endif ++#define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG) ++ ++static inline bool io_req_ffs_set(struct io_kiocb *req) ++{ ++ return IS_ENABLED(CONFIG_64BIT) && (req->flags & REQ_F_FIXED_FILE); ++} ++ ++static void io_req_track_inflight(struct io_kiocb *req) ++{ ++ if (!(req->flags & REQ_F_INFLIGHT)) { ++ req->flags |= REQ_F_INFLIGHT; ++ atomic_inc(&req->task->io_uring->inflight_tracked); ++ } ++} ++ ++static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req) ++{ ++ if (WARN_ON_ONCE(!req->link)) ++ return NULL; ++ ++ req->flags &= ~REQ_F_ARM_LTIMEOUT; ++ req->flags |= REQ_F_LINK_TIMEOUT; ++ ++ /* linked timeouts should have two refs once prep'ed */ ++ io_req_set_refcount(req); ++ __io_req_set_refcount(req->link, 2); ++ return req->link; ++} ++ ++static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) ++{ ++ if (likely(!(req->flags & REQ_F_ARM_LTIMEOUT))) ++ return NULL; ++ return __io_prep_linked_timeout(req); ++} ++ ++static void io_prep_async_work(struct io_kiocb *req) ++{ ++ const struct io_op_def *def = &io_op_defs[req->opcode]; ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ if (!(req->flags & REQ_F_CREDS)) { ++ req->flags |= REQ_F_CREDS; ++ req->creds = get_current_cred(); ++ } ++ ++ req->work.list.next = NULL; ++ req->work.flags = 0; ++ if (req->flags & REQ_F_FORCE_ASYNC) ++ req->work.flags |= IO_WQ_WORK_CONCURRENT; ++ ++ if (req->flags & REQ_F_ISREG) { ++ if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL)) ++ io_wq_hash_work(&req->work, file_inode(req->file)); ++ } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) { ++ if (def->unbound_nonreg_file) ++ req->work.flags |= IO_WQ_WORK_UNBOUND; ++ } ++} ++ ++static void io_prep_async_link(struct io_kiocb *req) ++{ ++ struct io_kiocb *cur; ++ ++ if (req->flags & REQ_F_LINK_TIMEOUT) { ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ spin_lock_irq(&ctx->timeout_lock); ++ io_for_each_link(cur, req) ++ io_prep_async_work(cur); ++ spin_unlock_irq(&ctx->timeout_lock); ++ } else { ++ io_for_each_link(cur, req) ++ io_prep_async_work(cur); ++ } ++} ++ ++static void io_queue_async_work(struct io_kiocb *req, bool *locked) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ struct io_kiocb *link = io_prep_linked_timeout(req); ++ struct io_uring_task *tctx = req->task->io_uring; ++ ++ /* must not take the lock, NULL it as a precaution */ ++ locked = NULL; ++ ++ BUG_ON(!tctx); ++ BUG_ON(!tctx->io_wq); ++ ++ /* init ->work of the whole link before punting */ ++ io_prep_async_link(req); ++ ++ /* ++ * Not expected to happen, but if we do have a bug where this _can_ ++ * happen, catch it here and ensure the request is marked as ++ * canceled. That will make io-wq go through the usual work cancel ++ * procedure rather than attempt to run this request (or create a new ++ * worker for it). ++ */ ++ if (WARN_ON_ONCE(!same_thread_group(req->task, current))) ++ req->work.flags |= IO_WQ_WORK_CANCEL; ++ ++ trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req, ++ &req->work, req->flags); ++ io_wq_enqueue(tctx->io_wq, &req->work); ++ if (link) ++ io_queue_linked_timeout(link); ++} ++ ++static void io_kill_timeout(struct io_kiocb *req, int status) ++ __must_hold(&req->ctx->completion_lock) ++ __must_hold(&req->ctx->timeout_lock) ++{ ++ struct io_timeout_data *io = req->async_data; ++ ++ if (hrtimer_try_to_cancel(&io->timer) != -1) { ++ if (status) ++ req_set_fail(req); ++ atomic_set(&req->ctx->cq_timeouts, ++ atomic_read(&req->ctx->cq_timeouts) + 1); ++ list_del_init(&req->timeout.list); ++ io_fill_cqe_req(req, status, 0); ++ io_put_req_deferred(req); ++ } ++} ++ ++static void io_queue_deferred(struct io_ring_ctx *ctx) ++{ ++ while (!list_empty(&ctx->defer_list)) { ++ struct io_defer_entry *de = list_first_entry(&ctx->defer_list, ++ struct io_defer_entry, list); ++ ++ if (req_need_defer(de->req, de->seq)) ++ break; ++ list_del_init(&de->list); ++ io_req_task_queue(de->req); ++ kfree(de); ++ } ++} ++ ++static void io_flush_timeouts(struct io_ring_ctx *ctx) ++ __must_hold(&ctx->completion_lock) ++{ ++ u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); ++ struct io_kiocb *req, *tmp; ++ ++ spin_lock_irq(&ctx->timeout_lock); ++ list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { ++ u32 events_needed, events_got; ++ ++ if (io_is_timeout_noseq(req)) ++ break; ++ ++ /* ++ * Since seq can easily wrap around over time, subtract ++ * the last seq at which timeouts were flushed before comparing. ++ * Assuming not more than 2^31-1 events have happened since, ++ * these subtractions won't have wrapped, so we can check if ++ * target is in [last_seq, current_seq] by comparing the two. ++ */ ++ events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush; ++ events_got = seq - ctx->cq_last_tm_flush; ++ if (events_got < events_needed) ++ break; ++ ++ io_kill_timeout(req, 0); ++ } ++ ctx->cq_last_tm_flush = seq; ++ spin_unlock_irq(&ctx->timeout_lock); ++} ++ ++static void __io_commit_cqring_flush(struct io_ring_ctx *ctx) ++{ ++ if (ctx->off_timeout_used) ++ io_flush_timeouts(ctx); ++ if (ctx->drain_active) ++ io_queue_deferred(ctx); ++} ++ ++static inline void io_commit_cqring(struct io_ring_ctx *ctx) ++{ ++ if (unlikely(ctx->off_timeout_used || ctx->drain_active)) ++ __io_commit_cqring_flush(ctx); ++ /* order cqe stores with ring update */ ++ smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); ++} ++ ++static inline bool io_sqring_full(struct io_ring_ctx *ctx) ++{ ++ struct io_rings *r = ctx->rings; ++ ++ return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == ctx->sq_entries; ++} ++ ++static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) ++{ ++ return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); ++} ++ ++static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) ++{ ++ struct io_rings *rings = ctx->rings; ++ unsigned tail, mask = ctx->cq_entries - 1; ++ ++ /* ++ * writes to the cq entry need to come after reading head; the ++ * control dependency is enough as we're using WRITE_ONCE to ++ * fill the cq entry ++ */ ++ if (__io_cqring_events(ctx) == ctx->cq_entries) ++ return NULL; ++ ++ tail = ctx->cached_cq_tail++; ++ return &rings->cqes[tail & mask]; ++} ++ ++static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx) ++{ ++ if (likely(!ctx->cq_ev_fd)) ++ return false; ++ if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) ++ return false; ++ return !ctx->eventfd_async || io_wq_current_is_worker(); ++} ++ ++/* ++ * This should only get called when at least one event has been posted. ++ * Some applications rely on the eventfd notification count only changing ++ * IFF a new CQE has been added to the CQ ring. There's no depedency on ++ * 1:1 relationship between how many times this function is called (and ++ * hence the eventfd count) and number of CQEs posted to the CQ ring. ++ */ ++static void io_cqring_ev_posted(struct io_ring_ctx *ctx) ++{ ++ /* ++ * wake_up_all() may seem excessive, but io_wake_function() and ++ * io_should_wake() handle the termination of the loop and only ++ * wake as many waiters as we need to. ++ */ ++ if (wq_has_sleeper(&ctx->cq_wait)) ++ wake_up_all(&ctx->cq_wait); ++ if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait)) ++ wake_up(&ctx->sq_data->wait); ++ if (io_should_trigger_evfd(ctx)) ++ eventfd_signal(ctx->cq_ev_fd, 1); ++ if (waitqueue_active(&ctx->poll_wait)) ++ wake_up_interruptible(&ctx->poll_wait); ++} ++ ++static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) ++{ ++ /* see waitqueue_active() comment */ ++ smp_mb(); ++ ++ if (ctx->flags & IORING_SETUP_SQPOLL) { ++ if (waitqueue_active(&ctx->cq_wait)) ++ wake_up_all(&ctx->cq_wait); ++ } ++ if (io_should_trigger_evfd(ctx)) ++ eventfd_signal(ctx->cq_ev_fd, 1); ++ if (waitqueue_active(&ctx->poll_wait)) ++ wake_up_interruptible(&ctx->poll_wait); ++} ++ ++/* Returns true if there are no backlogged entries after the flush */ ++static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) ++{ ++ bool all_flushed, posted; ++ ++ if (!force && __io_cqring_events(ctx) == ctx->cq_entries) ++ return false; ++ ++ posted = false; ++ spin_lock(&ctx->completion_lock); ++ while (!list_empty(&ctx->cq_overflow_list)) { ++ struct io_uring_cqe *cqe = io_get_cqe(ctx); ++ struct io_overflow_cqe *ocqe; ++ ++ if (!cqe && !force) ++ break; ++ ocqe = list_first_entry(&ctx->cq_overflow_list, ++ struct io_overflow_cqe, list); ++ if (cqe) ++ memcpy(cqe, &ocqe->cqe, sizeof(*cqe)); ++ else ++ io_account_cq_overflow(ctx); ++ ++ posted = true; ++ list_del(&ocqe->list); ++ kfree(ocqe); ++ } ++ ++ all_flushed = list_empty(&ctx->cq_overflow_list); ++ if (all_flushed) { ++ clear_bit(0, &ctx->check_cq_overflow); ++ WRITE_ONCE(ctx->rings->sq_flags, ++ ctx->rings->sq_flags & ~IORING_SQ_CQ_OVERFLOW); ++ } ++ ++ if (posted) ++ io_commit_cqring(ctx); ++ spin_unlock(&ctx->completion_lock); ++ if (posted) ++ io_cqring_ev_posted(ctx); ++ return all_flushed; ++} ++ ++static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx) ++{ ++ bool ret = true; ++ ++ if (test_bit(0, &ctx->check_cq_overflow)) { ++ /* iopoll syncs against uring_lock, not completion_lock */ ++ if (ctx->flags & IORING_SETUP_IOPOLL) ++ mutex_lock(&ctx->uring_lock); ++ ret = __io_cqring_overflow_flush(ctx, false); ++ if (ctx->flags & IORING_SETUP_IOPOLL) ++ mutex_unlock(&ctx->uring_lock); ++ } ++ ++ return ret; ++} ++ ++/* must to be called somewhat shortly after putting a request */ ++static inline void io_put_task(struct task_struct *task, int nr) ++{ ++ struct io_uring_task *tctx = task->io_uring; ++ ++ if (likely(task == current)) { ++ tctx->cached_refs += nr; ++ } else { ++ percpu_counter_sub(&tctx->inflight, nr); ++ if (unlikely(atomic_read(&tctx->in_idle))) ++ wake_up(&tctx->wait); ++ put_task_struct_many(task, nr); ++ } ++} ++ ++static void io_task_refs_refill(struct io_uring_task *tctx) ++{ ++ unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR; ++ ++ percpu_counter_add(&tctx->inflight, refill); ++ refcount_add(refill, ¤t->usage); ++ tctx->cached_refs += refill; ++} ++ ++static inline void io_get_task_refs(int nr) ++{ ++ struct io_uring_task *tctx = current->io_uring; ++ ++ tctx->cached_refs -= nr; ++ if (unlikely(tctx->cached_refs < 0)) ++ io_task_refs_refill(tctx); ++} ++ ++static __cold void io_uring_drop_tctx_refs(struct task_struct *task) ++{ ++ struct io_uring_task *tctx = task->io_uring; ++ unsigned int refs = tctx->cached_refs; ++ ++ if (refs) { ++ tctx->cached_refs = 0; ++ percpu_counter_sub(&tctx->inflight, refs); ++ put_task_struct_many(task, refs); ++ } ++} ++ ++static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, ++ s32 res, u32 cflags) ++{ ++ struct io_overflow_cqe *ocqe; ++ ++ ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT); ++ if (!ocqe) { ++ /* ++ * If we're in ring overflow flush mode, or in task cancel mode, ++ * or cannot allocate an overflow entry, then we need to drop it ++ * on the floor. ++ */ ++ io_account_cq_overflow(ctx); ++ return false; ++ } ++ if (list_empty(&ctx->cq_overflow_list)) { ++ set_bit(0, &ctx->check_cq_overflow); ++ WRITE_ONCE(ctx->rings->sq_flags, ++ ctx->rings->sq_flags | IORING_SQ_CQ_OVERFLOW); ++ ++ } ++ ocqe->cqe.user_data = user_data; ++ ocqe->cqe.res = res; ++ ocqe->cqe.flags = cflags; ++ list_add_tail(&ocqe->list, &ctx->cq_overflow_list); ++ return true; ++} ++ ++static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data, ++ s32 res, u32 cflags) ++{ ++ struct io_uring_cqe *cqe; ++ ++ trace_io_uring_complete(ctx, user_data, res, cflags); ++ ++ /* ++ * If we can't get a cq entry, userspace overflowed the ++ * submission (by quite a lot). Increment the overflow count in ++ * the ring. ++ */ ++ cqe = io_get_cqe(ctx); ++ if (likely(cqe)) { ++ WRITE_ONCE(cqe->user_data, user_data); ++ WRITE_ONCE(cqe->res, res); ++ WRITE_ONCE(cqe->flags, cflags); ++ return true; ++ } ++ return io_cqring_event_overflow(ctx, user_data, res, cflags); ++} ++ ++static noinline void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags) ++{ ++ __io_fill_cqe(req->ctx, req->user_data, res, cflags); ++} ++ ++static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, ++ s32 res, u32 cflags) ++{ ++ ctx->cq_extra++; ++ return __io_fill_cqe(ctx, user_data, res, cflags); ++} ++ ++static void io_req_complete_post(struct io_kiocb *req, s32 res, ++ u32 cflags) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ spin_lock(&ctx->completion_lock); ++ __io_fill_cqe(ctx, req->user_data, res, cflags); ++ /* ++ * If we're the last reference to this request, add to our locked ++ * free_list cache. ++ */ ++ if (req_ref_put_and_test(req)) { ++ if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { ++ if (req->flags & IO_DISARM_MASK) ++ io_disarm_next(req); ++ if (req->link) { ++ io_req_task_queue(req->link); ++ req->link = NULL; ++ } ++ } ++ io_dismantle_req(req); ++ io_put_task(req->task, 1); ++ list_add(&req->inflight_entry, &ctx->locked_free_list); ++ ctx->locked_free_nr++; ++ } else { ++ if (!percpu_ref_tryget(&ctx->refs)) ++ req = NULL; ++ } ++ io_commit_cqring(ctx); ++ spin_unlock(&ctx->completion_lock); ++ ++ if (req) { ++ io_cqring_ev_posted(ctx); ++ percpu_ref_put(&ctx->refs); ++ } ++} ++ ++static inline bool io_req_needs_clean(struct io_kiocb *req) ++{ ++ return req->flags & IO_REQ_CLEAN_FLAGS; ++} ++ ++static inline void io_req_complete_state(struct io_kiocb *req, s32 res, ++ u32 cflags) ++{ ++ if (io_req_needs_clean(req)) ++ io_clean_op(req); ++ req->result = res; ++ req->compl.cflags = cflags; ++ req->flags |= REQ_F_COMPLETE_INLINE; ++} ++ ++static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags, ++ s32 res, u32 cflags) ++{ ++ if (issue_flags & IO_URING_F_COMPLETE_DEFER) ++ io_req_complete_state(req, res, cflags); ++ else ++ io_req_complete_post(req, res, cflags); ++} ++ ++static inline void io_req_complete(struct io_kiocb *req, s32 res) ++{ ++ __io_req_complete(req, 0, res, 0); ++} ++ ++static void io_req_complete_failed(struct io_kiocb *req, s32 res) ++{ ++ req_set_fail(req); ++ io_req_complete_post(req, res, 0); ++} ++ ++static void io_req_complete_fail_submit(struct io_kiocb *req) ++{ ++ /* ++ * We don't submit, fail them all, for that replace hardlinks with ++ * normal links. Extra REQ_F_LINK is tolerated. ++ */ ++ req->flags &= ~REQ_F_HARDLINK; ++ req->flags |= REQ_F_LINK; ++ io_req_complete_failed(req, req->result); ++} ++ ++/* ++ * Don't initialise the fields below on every allocation, but do that in ++ * advance and keep them valid across allocations. ++ */ ++static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx) ++{ ++ req->ctx = ctx; ++ req->link = NULL; ++ req->async_data = NULL; ++ /* not necessary, but safer to zero */ ++ req->result = 0; ++} ++ ++static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx, ++ struct io_submit_state *state) ++{ ++ spin_lock(&ctx->completion_lock); ++ list_splice_init(&ctx->locked_free_list, &state->free_list); ++ ctx->locked_free_nr = 0; ++ spin_unlock(&ctx->completion_lock); ++} ++ ++/* Returns true IFF there are requests in the cache */ ++static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) ++{ ++ struct io_submit_state *state = &ctx->submit_state; ++ int nr; ++ ++ /* ++ * If we have more than a batch's worth of requests in our IRQ side ++ * locked cache, grab the lock and move them over to our submission ++ * side cache. ++ */ ++ if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH) ++ io_flush_cached_locked_reqs(ctx, state); ++ ++ nr = state->free_reqs; ++ while (!list_empty(&state->free_list)) { ++ struct io_kiocb *req = list_first_entry(&state->free_list, ++ struct io_kiocb, inflight_entry); ++ ++ list_del(&req->inflight_entry); ++ state->reqs[nr++] = req; ++ if (nr == ARRAY_SIZE(state->reqs)) ++ break; ++ } ++ ++ state->free_reqs = nr; ++ return nr != 0; ++} ++ ++/* ++ * A request might get retired back into the request caches even before opcode ++ * handlers and io_issue_sqe() are done with it, e.g. inline completion path. ++ * Because of that, io_alloc_req() should be called only under ->uring_lock ++ * and with extra caution to not get a request that is still worked on. ++ */ ++static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) ++ __must_hold(&ctx->uring_lock) ++{ ++ struct io_submit_state *state = &ctx->submit_state; ++ gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; ++ int ret, i; ++ ++ BUILD_BUG_ON(ARRAY_SIZE(state->reqs) < IO_REQ_ALLOC_BATCH); ++ ++ if (likely(state->free_reqs || io_flush_cached_reqs(ctx))) ++ goto got_req; ++ ++ ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH, ++ state->reqs); ++ ++ /* ++ * Bulk alloc is all-or-nothing. If we fail to get a batch, ++ * retry single alloc to be on the safe side. ++ */ ++ if (unlikely(ret <= 0)) { ++ state->reqs[0] = kmem_cache_alloc(req_cachep, gfp); ++ if (!state->reqs[0]) ++ return NULL; ++ ret = 1; ++ } ++ ++ for (i = 0; i < ret; i++) ++ io_preinit_req(state->reqs[i], ctx); ++ state->free_reqs = ret; ++got_req: ++ state->free_reqs--; ++ return state->reqs[state->free_reqs]; ++} ++ ++static inline void io_put_file(struct file *file) ++{ ++ if (file) ++ fput(file); ++} ++ ++static void io_dismantle_req(struct io_kiocb *req) ++{ ++ unsigned int flags = req->flags; ++ ++ if (io_req_needs_clean(req)) ++ io_clean_op(req); ++ if (!(flags & REQ_F_FIXED_FILE)) ++ io_put_file(req->file); ++ if (req->fixed_rsrc_refs) ++ percpu_ref_put(req->fixed_rsrc_refs); ++ if (req->async_data) { ++ kfree(req->async_data); ++ req->async_data = NULL; ++ } ++} ++ ++static void __io_free_req(struct io_kiocb *req) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ io_dismantle_req(req); ++ io_put_task(req->task, 1); ++ ++ spin_lock(&ctx->completion_lock); ++ list_add(&req->inflight_entry, &ctx->locked_free_list); ++ ctx->locked_free_nr++; ++ spin_unlock(&ctx->completion_lock); ++ ++ percpu_ref_put(&ctx->refs); ++} ++ ++static inline void io_remove_next_linked(struct io_kiocb *req) ++{ ++ struct io_kiocb *nxt = req->link; ++ ++ req->link = nxt->link; ++ nxt->link = NULL; ++} ++ ++static bool io_kill_linked_timeout(struct io_kiocb *req) ++ __must_hold(&req->ctx->completion_lock) ++ __must_hold(&req->ctx->timeout_lock) ++{ ++ struct io_kiocb *link = req->link; ++ ++ if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { ++ struct io_timeout_data *io = link->async_data; ++ ++ io_remove_next_linked(req); ++ link->timeout.head = NULL; ++ if (hrtimer_try_to_cancel(&io->timer) != -1) { ++ list_del(&link->timeout.list); ++ io_fill_cqe_req(link, -ECANCELED, 0); ++ io_put_req_deferred(link); ++ return true; ++ } ++ } ++ return false; ++} ++ ++static void io_fail_links(struct io_kiocb *req) ++ __must_hold(&req->ctx->completion_lock) ++{ ++ struct io_kiocb *nxt, *link = req->link; ++ ++ req->link = NULL; ++ while (link) { ++ long res = -ECANCELED; ++ ++ if (link->flags & REQ_F_FAIL) ++ res = link->result; ++ ++ nxt = link->link; ++ link->link = NULL; ++ ++ trace_io_uring_fail_link(req, link); ++ io_fill_cqe_req(link, res, 0); ++ io_put_req_deferred(link); ++ link = nxt; ++ } ++} ++ ++static bool io_disarm_next(struct io_kiocb *req) ++ __must_hold(&req->ctx->completion_lock) ++{ ++ bool posted = false; ++ ++ if (req->flags & REQ_F_ARM_LTIMEOUT) { ++ struct io_kiocb *link = req->link; ++ ++ req->flags &= ~REQ_F_ARM_LTIMEOUT; ++ if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { ++ io_remove_next_linked(req); ++ io_fill_cqe_req(link, -ECANCELED, 0); ++ io_put_req_deferred(link); ++ posted = true; ++ } ++ } else if (req->flags & REQ_F_LINK_TIMEOUT) { ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ spin_lock_irq(&ctx->timeout_lock); ++ posted = io_kill_linked_timeout(req); ++ spin_unlock_irq(&ctx->timeout_lock); ++ } ++ if (unlikely((req->flags & REQ_F_FAIL) && ++ !(req->flags & REQ_F_HARDLINK))) { ++ posted |= (req->link != NULL); ++ io_fail_links(req); ++ } ++ return posted; ++} ++ ++static struct io_kiocb *__io_req_find_next(struct io_kiocb *req) ++{ ++ struct io_kiocb *nxt; ++ ++ /* ++ * If LINK is set, we have dependent requests in this chain. If we ++ * didn't fail this request, queue the first one up, moving any other ++ * dependencies to the next request. In case of failure, fail the rest ++ * of the chain. ++ */ ++ if (req->flags & IO_DISARM_MASK) { ++ struct io_ring_ctx *ctx = req->ctx; ++ bool posted; ++ ++ spin_lock(&ctx->completion_lock); ++ posted = io_disarm_next(req); ++ if (posted) ++ io_commit_cqring(req->ctx); ++ spin_unlock(&ctx->completion_lock); ++ if (posted) ++ io_cqring_ev_posted(ctx); ++ } ++ nxt = req->link; ++ req->link = NULL; ++ return nxt; ++} ++ ++static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) ++{ ++ if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK)))) ++ return NULL; ++ return __io_req_find_next(req); ++} ++ ++static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked) ++{ ++ if (!ctx) ++ return; ++ if (*locked) { ++ if (ctx->submit_state.compl_nr) ++ io_submit_flush_completions(ctx); ++ mutex_unlock(&ctx->uring_lock); ++ *locked = false; ++ } ++ percpu_ref_put(&ctx->refs); ++} ++ ++static void tctx_task_work(struct callback_head *cb) ++{ ++ bool locked = false; ++ struct io_ring_ctx *ctx = NULL; ++ struct io_uring_task *tctx = container_of(cb, struct io_uring_task, ++ task_work); ++ ++ while (1) { ++ struct io_wq_work_node *node; ++ ++ if (!tctx->task_list.first && locked && ctx->submit_state.compl_nr) ++ io_submit_flush_completions(ctx); ++ ++ spin_lock_irq(&tctx->task_lock); ++ node = tctx->task_list.first; ++ INIT_WQ_LIST(&tctx->task_list); ++ if (!node) ++ tctx->task_running = false; ++ spin_unlock_irq(&tctx->task_lock); ++ if (!node) ++ break; ++ ++ do { ++ struct io_wq_work_node *next = node->next; ++ struct io_kiocb *req = container_of(node, struct io_kiocb, ++ io_task_work.node); ++ ++ if (req->ctx != ctx) { ++ ctx_flush_and_put(ctx, &locked); ++ ctx = req->ctx; ++ /* if not contended, grab and improve batching */ ++ locked = mutex_trylock(&ctx->uring_lock); ++ percpu_ref_get(&ctx->refs); ++ } ++ req->io_task_work.func(req, &locked); ++ node = next; ++ } while (node); ++ ++ cond_resched(); ++ } ++ ++ ctx_flush_and_put(ctx, &locked); ++ ++ /* relaxed read is enough as only the task itself sets ->in_idle */ ++ if (unlikely(atomic_read(&tctx->in_idle))) ++ io_uring_drop_tctx_refs(current); ++} ++ ++static void io_req_task_work_add(struct io_kiocb *req) ++{ ++ struct task_struct *tsk = req->task; ++ struct io_uring_task *tctx = tsk->io_uring; ++ enum task_work_notify_mode notify; ++ struct io_wq_work_node *node; ++ unsigned long flags; ++ bool running; ++ ++ WARN_ON_ONCE(!tctx); ++ ++ spin_lock_irqsave(&tctx->task_lock, flags); ++ wq_list_add_tail(&req->io_task_work.node, &tctx->task_list); ++ running = tctx->task_running; ++ if (!running) ++ tctx->task_running = true; ++ spin_unlock_irqrestore(&tctx->task_lock, flags); ++ ++ /* task_work already pending, we're done */ ++ if (running) ++ return; ++ ++ /* ++ * SQPOLL kernel thread doesn't need notification, just a wakeup. For ++ * all other cases, use TWA_SIGNAL unconditionally to ensure we're ++ * processing task_work. There's no reliable way to tell if TWA_RESUME ++ * will do the job. ++ */ ++ notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL; ++ if (!task_work_add(tsk, &tctx->task_work, notify)) { ++ wake_up_process(tsk); ++ return; ++ } ++ ++ spin_lock_irqsave(&tctx->task_lock, flags); ++ tctx->task_running = false; ++ node = tctx->task_list.first; ++ INIT_WQ_LIST(&tctx->task_list); ++ spin_unlock_irqrestore(&tctx->task_lock, flags); ++ ++ while (node) { ++ req = container_of(node, struct io_kiocb, io_task_work.node); ++ node = node->next; ++ if (llist_add(&req->io_task_work.fallback_node, ++ &req->ctx->fallback_llist)) ++ schedule_delayed_work(&req->ctx->fallback_work, 1); ++ } ++} ++ ++static void io_req_task_cancel(struct io_kiocb *req, bool *locked) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ /* not needed for normal modes, but SQPOLL depends on it */ ++ io_tw_lock(ctx, locked); ++ io_req_complete_failed(req, req->result); ++} ++ ++static void io_req_task_submit(struct io_kiocb *req, bool *locked) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ io_tw_lock(ctx, locked); ++ /* req->task == current here, checking PF_EXITING is safe */ ++ if (likely(!(req->task->flags & PF_EXITING))) ++ __io_queue_sqe(req); ++ else ++ io_req_complete_failed(req, -EFAULT); ++} ++ ++static void io_req_task_queue_fail(struct io_kiocb *req, int ret) ++{ ++ req->result = ret; ++ req->io_task_work.func = io_req_task_cancel; ++ io_req_task_work_add(req); ++} ++ ++static void io_req_task_queue(struct io_kiocb *req) ++{ ++ req->io_task_work.func = io_req_task_submit; ++ io_req_task_work_add(req); ++} ++ ++static void io_req_task_queue_reissue(struct io_kiocb *req) ++{ ++ req->io_task_work.func = io_queue_async_work; ++ io_req_task_work_add(req); ++} ++ ++static inline void io_queue_next(struct io_kiocb *req) ++{ ++ struct io_kiocb *nxt = io_req_find_next(req); ++ ++ if (nxt) ++ io_req_task_queue(nxt); ++} ++ ++static void io_free_req(struct io_kiocb *req) ++{ ++ io_queue_next(req); ++ __io_free_req(req); ++} ++ ++static void io_free_req_work(struct io_kiocb *req, bool *locked) ++{ ++ io_free_req(req); ++} ++ ++struct req_batch { ++ struct task_struct *task; ++ int task_refs; ++ int ctx_refs; ++}; ++ ++static inline void io_init_req_batch(struct req_batch *rb) ++{ ++ rb->task_refs = 0; ++ rb->ctx_refs = 0; ++ rb->task = NULL; ++} ++ ++static void io_req_free_batch_finish(struct io_ring_ctx *ctx, ++ struct req_batch *rb) ++{ ++ if (rb->ctx_refs) ++ percpu_ref_put_many(&ctx->refs, rb->ctx_refs); ++ if (rb->task) ++ io_put_task(rb->task, rb->task_refs); ++} ++ ++static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, ++ struct io_submit_state *state) ++{ ++ io_queue_next(req); ++ io_dismantle_req(req); ++ ++ if (req->task != rb->task) { ++ if (rb->task) ++ io_put_task(rb->task, rb->task_refs); ++ rb->task = req->task; ++ rb->task_refs = 0; ++ } ++ rb->task_refs++; ++ rb->ctx_refs++; ++ ++ if (state->free_reqs != ARRAY_SIZE(state->reqs)) ++ state->reqs[state->free_reqs++] = req; ++ else ++ list_add(&req->inflight_entry, &state->free_list); ++} ++ ++static void io_submit_flush_completions(struct io_ring_ctx *ctx) ++ __must_hold(&ctx->uring_lock) ++{ ++ struct io_submit_state *state = &ctx->submit_state; ++ int i, nr = state->compl_nr; ++ struct req_batch rb; ++ ++ spin_lock(&ctx->completion_lock); ++ for (i = 0; i < nr; i++) { ++ struct io_kiocb *req = state->compl_reqs[i]; ++ ++ __io_fill_cqe(ctx, req->user_data, req->result, ++ req->compl.cflags); ++ } ++ io_commit_cqring(ctx); ++ spin_unlock(&ctx->completion_lock); ++ io_cqring_ev_posted(ctx); ++ ++ io_init_req_batch(&rb); ++ for (i = 0; i < nr; i++) { ++ struct io_kiocb *req = state->compl_reqs[i]; ++ ++ if (req_ref_put_and_test(req)) ++ io_req_free_batch(&rb, req, &ctx->submit_state); ++ } ++ ++ io_req_free_batch_finish(ctx, &rb); ++ state->compl_nr = 0; ++} ++ ++/* ++ * Drop reference to request, return next in chain (if there is one) if this ++ * was the last reference to this request. ++ */ ++static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req) ++{ ++ struct io_kiocb *nxt = NULL; ++ ++ if (req_ref_put_and_test(req)) { ++ nxt = io_req_find_next(req); ++ __io_free_req(req); ++ } ++ return nxt; ++} ++ ++static inline void io_put_req(struct io_kiocb *req) ++{ ++ if (req_ref_put_and_test(req)) ++ io_free_req(req); ++} ++ ++static inline void io_put_req_deferred(struct io_kiocb *req) ++{ ++ if (req_ref_put_and_test(req)) { ++ req->io_task_work.func = io_free_req_work; ++ io_req_task_work_add(req); ++ } ++} ++ ++static unsigned io_cqring_events(struct io_ring_ctx *ctx) ++{ ++ /* See comment at the top of this file */ ++ smp_rmb(); ++ return __io_cqring_events(ctx); ++} ++ ++static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) ++{ ++ struct io_rings *rings = ctx->rings; ++ ++ /* make sure SQ entry isn't read before tail */ ++ return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; ++} ++ ++static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf) ++{ ++ unsigned int cflags; ++ ++ cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT; ++ cflags |= IORING_CQE_F_BUFFER; ++ req->flags &= ~REQ_F_BUFFER_SELECTED; ++ kfree(kbuf); ++ return cflags; ++} ++ ++static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req) ++{ ++ struct io_buffer *kbuf; ++ ++ if (likely(!(req->flags & REQ_F_BUFFER_SELECTED))) ++ return 0; ++ kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; ++ return io_put_kbuf(req, kbuf); ++} ++ ++static inline bool io_run_task_work(void) ++{ ++ if (test_thread_flag(TIF_NOTIFY_SIGNAL) || current->task_works) { ++ __set_current_state(TASK_RUNNING); ++ tracehook_notify_signal(); ++ return true; ++ } ++ ++ return false; ++} ++ ++/* ++ * Find and free completed poll iocbs ++ */ ++static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, ++ struct list_head *done) ++{ ++ struct req_batch rb; ++ struct io_kiocb *req; ++ ++ /* order with ->result store in io_complete_rw_iopoll() */ ++ smp_rmb(); ++ ++ io_init_req_batch(&rb); ++ while (!list_empty(done)) { ++ req = list_first_entry(done, struct io_kiocb, inflight_entry); ++ list_del(&req->inflight_entry); ++ ++ io_fill_cqe_req(req, req->result, io_put_rw_kbuf(req)); ++ (*nr_events)++; ++ ++ if (req_ref_put_and_test(req)) ++ io_req_free_batch(&rb, req, &ctx->submit_state); ++ } ++ ++ io_commit_cqring(ctx); ++ io_cqring_ev_posted_iopoll(ctx); ++ io_req_free_batch_finish(ctx, &rb); ++} ++ ++static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, ++ long min) ++{ ++ struct io_kiocb *req, *tmp; ++ LIST_HEAD(done); ++ bool spin; ++ ++ /* ++ * Only spin for completions if we don't have multiple devices hanging ++ * off our complete list, and we're under the requested amount. ++ */ ++ spin = !ctx->poll_multi_queue && *nr_events < min; ++ ++ list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) { ++ struct kiocb *kiocb = &req->rw.kiocb; ++ int ret; ++ ++ /* ++ * Move completed and retryable entries to our local lists. ++ * If we find a request that requires polling, break out ++ * and complete those lists first, if we have entries there. ++ */ ++ if (READ_ONCE(req->iopoll_completed)) { ++ list_move_tail(&req->inflight_entry, &done); ++ continue; ++ } ++ if (!list_empty(&done)) ++ break; ++ ++ ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin); ++ if (unlikely(ret < 0)) ++ return ret; ++ else if (ret) ++ spin = false; ++ ++ /* iopoll may have completed current req */ ++ if (READ_ONCE(req->iopoll_completed)) ++ list_move_tail(&req->inflight_entry, &done); ++ } ++ ++ if (!list_empty(&done)) ++ io_iopoll_complete(ctx, nr_events, &done); ++ ++ return 0; ++} ++ ++/* ++ * We can't just wait for polled events to come to us, we have to actively ++ * find and complete them. ++ */ ++static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx) ++{ ++ if (!(ctx->flags & IORING_SETUP_IOPOLL)) ++ return; ++ ++ mutex_lock(&ctx->uring_lock); ++ while (!list_empty(&ctx->iopoll_list)) { ++ unsigned int nr_events = 0; ++ ++ io_do_iopoll(ctx, &nr_events, 0); ++ ++ /* let it sleep and repeat later if can't complete a request */ ++ if (nr_events == 0) ++ break; ++ /* ++ * Ensure we allow local-to-the-cpu processing to take place, ++ * in this case we need to ensure that we reap all events. ++ * Also let task_work, etc. to progress by releasing the mutex ++ */ ++ if (need_resched()) { ++ mutex_unlock(&ctx->uring_lock); ++ cond_resched(); ++ mutex_lock(&ctx->uring_lock); ++ } ++ } ++ mutex_unlock(&ctx->uring_lock); ++} ++ ++static int io_iopoll_check(struct io_ring_ctx *ctx, long min) ++{ ++ unsigned int nr_events = 0; ++ int ret = 0; ++ ++ /* ++ * We disallow the app entering submit/complete with polling, but we ++ * still need to lock the ring to prevent racing with polled issue ++ * that got punted to a workqueue. ++ */ ++ mutex_lock(&ctx->uring_lock); ++ /* ++ * Don't enter poll loop if we already have events pending. ++ * If we do, we can potentially be spinning for commands that ++ * already triggered a CQE (eg in error). ++ */ ++ if (test_bit(0, &ctx->check_cq_overflow)) ++ __io_cqring_overflow_flush(ctx, false); ++ if (io_cqring_events(ctx)) ++ goto out; ++ do { ++ /* ++ * If a submit got punted to a workqueue, we can have the ++ * application entering polling for a command before it gets ++ * issued. That app will hold the uring_lock for the duration ++ * of the poll right here, so we need to take a breather every ++ * now and then to ensure that the issue has a chance to add ++ * the poll to the issued list. Otherwise we can spin here ++ * forever, while the workqueue is stuck trying to acquire the ++ * very same mutex. ++ */ ++ if (list_empty(&ctx->iopoll_list)) { ++ u32 tail = ctx->cached_cq_tail; ++ ++ mutex_unlock(&ctx->uring_lock); ++ io_run_task_work(); ++ mutex_lock(&ctx->uring_lock); ++ ++ /* some requests don't go through iopoll_list */ ++ if (tail != ctx->cached_cq_tail || ++ list_empty(&ctx->iopoll_list)) ++ break; ++ } ++ ret = io_do_iopoll(ctx, &nr_events, min); ++ } while (!ret && nr_events < min && !need_resched()); ++out: ++ mutex_unlock(&ctx->uring_lock); ++ return ret; ++} ++ ++static void kiocb_end_write(struct io_kiocb *req) ++{ ++ /* ++ * Tell lockdep we inherited freeze protection from submission ++ * thread. ++ */ ++ if (req->flags & REQ_F_ISREG) { ++ struct super_block *sb = file_inode(req->file)->i_sb; ++ ++ __sb_writers_acquired(sb, SB_FREEZE_WRITE); ++ sb_end_write(sb); ++ } ++} ++ ++#ifdef CONFIG_BLOCK ++static bool io_resubmit_prep(struct io_kiocb *req) ++{ ++ struct io_async_rw *rw = req->async_data; ++ ++ if (!rw) ++ return !io_req_prep_async(req); ++ iov_iter_restore(&rw->iter, &rw->iter_state); ++ return true; ++} ++ ++static bool io_rw_should_reissue(struct io_kiocb *req) ++{ ++ umode_t mode = file_inode(req->file)->i_mode; ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ if (!S_ISBLK(mode) && !S_ISREG(mode)) ++ return false; ++ if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() && ++ !(ctx->flags & IORING_SETUP_IOPOLL))) ++ return false; ++ /* ++ * If ref is dying, we might be running poll reap from the exit work. ++ * Don't attempt to reissue from that path, just let it fail with ++ * -EAGAIN. ++ */ ++ if (percpu_ref_is_dying(&ctx->refs)) ++ return false; ++ /* ++ * Play it safe and assume not safe to re-import and reissue if we're ++ * not in the original thread group (or in task context). ++ */ ++ if (!same_thread_group(req->task, current) || !in_task()) ++ return false; ++ return true; ++} ++#else ++static bool io_resubmit_prep(struct io_kiocb *req) ++{ ++ return false; ++} ++static bool io_rw_should_reissue(struct io_kiocb *req) ++{ ++ return false; ++} ++#endif ++ ++static bool __io_complete_rw_common(struct io_kiocb *req, long res) ++{ ++ if (req->rw.kiocb.ki_flags & IOCB_WRITE) { ++ kiocb_end_write(req); ++ fsnotify_modify(req->file); ++ } else { ++ fsnotify_access(req->file); ++ } ++ if (res != req->result) { ++ if ((res == -EAGAIN || res == -EOPNOTSUPP) && ++ io_rw_should_reissue(req)) { ++ req->flags |= REQ_F_REISSUE; ++ return true; ++ } ++ req_set_fail(req); ++ req->result = res; ++ } ++ return false; ++} ++ ++static inline int io_fixup_rw_res(struct io_kiocb *req, unsigned res) ++{ ++ struct io_async_rw *io = req->async_data; ++ ++ /* add previously done IO, if any */ ++ if (io && io->bytes_done > 0) { ++ if (res < 0) ++ res = io->bytes_done; ++ else ++ res += io->bytes_done; ++ } ++ return res; ++} ++ ++static void io_req_task_complete(struct io_kiocb *req, bool *locked) ++{ ++ unsigned int cflags = io_put_rw_kbuf(req); ++ int res = req->result; ++ ++ if (*locked) { ++ struct io_ring_ctx *ctx = req->ctx; ++ struct io_submit_state *state = &ctx->submit_state; ++ ++ io_req_complete_state(req, res, cflags); ++ state->compl_reqs[state->compl_nr++] = req; ++ if (state->compl_nr == ARRAY_SIZE(state->compl_reqs)) ++ io_submit_flush_completions(ctx); ++ } else { ++ io_req_complete_post(req, res, cflags); ++ } ++} ++ ++static void __io_complete_rw(struct io_kiocb *req, long res, long res2, ++ unsigned int issue_flags) ++{ ++ if (__io_complete_rw_common(req, res)) ++ return; ++ __io_req_complete(req, issue_flags, io_fixup_rw_res(req, res), io_put_rw_kbuf(req)); ++} ++ ++static void io_complete_rw(struct kiocb *kiocb, long res, long res2) ++{ ++ struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); ++ ++ if (__io_complete_rw_common(req, res)) ++ return; ++ req->result = io_fixup_rw_res(req, res); ++ req->io_task_work.func = io_req_task_complete; ++ io_req_task_work_add(req); ++} ++ ++static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) ++{ ++ struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); ++ ++ if (kiocb->ki_flags & IOCB_WRITE) ++ kiocb_end_write(req); ++ if (unlikely(res != req->result)) { ++ if (res == -EAGAIN && io_rw_should_reissue(req)) { ++ req->flags |= REQ_F_REISSUE; ++ return; ++ } ++ } ++ ++ WRITE_ONCE(req->result, res); ++ /* order with io_iopoll_complete() checking ->result */ ++ smp_wmb(); ++ WRITE_ONCE(req->iopoll_completed, 1); ++} ++ ++/* ++ * After the iocb has been issued, it's safe to be found on the poll list. ++ * Adding the kiocb to the list AFTER submission ensures that we don't ++ * find it from a io_do_iopoll() thread before the issuer is done ++ * accessing the kiocb cookie. ++ */ ++static void io_iopoll_req_issued(struct io_kiocb *req) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ const bool in_async = io_wq_current_is_worker(); ++ ++ /* workqueue context doesn't hold uring_lock, grab it now */ ++ if (unlikely(in_async)) ++ mutex_lock(&ctx->uring_lock); ++ ++ /* ++ * Track whether we have multiple files in our lists. This will impact ++ * how we do polling eventually, not spinning if we're on potentially ++ * different devices. ++ */ ++ if (list_empty(&ctx->iopoll_list)) { ++ ctx->poll_multi_queue = false; ++ } else if (!ctx->poll_multi_queue) { ++ struct io_kiocb *list_req; ++ unsigned int queue_num0, queue_num1; ++ ++ list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb, ++ inflight_entry); ++ ++ if (list_req->file != req->file) { ++ ctx->poll_multi_queue = true; ++ } else { ++ queue_num0 = blk_qc_t_to_queue_num(list_req->rw.kiocb.ki_cookie); ++ queue_num1 = blk_qc_t_to_queue_num(req->rw.kiocb.ki_cookie); ++ if (queue_num0 != queue_num1) ++ ctx->poll_multi_queue = true; ++ } ++ } ++ ++ /* ++ * For fast devices, IO may have already completed. If it has, add ++ * it to the front so we find it first. ++ */ ++ if (READ_ONCE(req->iopoll_completed)) ++ list_add(&req->inflight_entry, &ctx->iopoll_list); ++ else ++ list_add_tail(&req->inflight_entry, &ctx->iopoll_list); ++ ++ if (unlikely(in_async)) { ++ /* ++ * If IORING_SETUP_SQPOLL is enabled, sqes are either handle ++ * in sq thread task context or in io worker task context. If ++ * current task context is sq thread, we don't need to check ++ * whether should wake up sq thread. ++ */ ++ if ((ctx->flags & IORING_SETUP_SQPOLL) && ++ wq_has_sleeper(&ctx->sq_data->wait)) ++ wake_up(&ctx->sq_data->wait); ++ ++ mutex_unlock(&ctx->uring_lock); ++ } ++} ++ ++static bool io_bdev_nowait(struct block_device *bdev) ++{ ++ return !bdev || blk_queue_nowait(bdev_get_queue(bdev)); ++} ++ ++/* ++ * If we tracked the file through the SCM inflight mechanism, we could support ++ * any file. For now, just ensure that anything potentially problematic is done ++ * inline. ++ */ ++static bool __io_file_supports_nowait(struct file *file, int rw) ++{ ++ umode_t mode = file_inode(file)->i_mode; ++ ++ if (S_ISBLK(mode)) { ++ if (IS_ENABLED(CONFIG_BLOCK) && ++ io_bdev_nowait(I_BDEV(file->f_mapping->host))) ++ return true; ++ return false; ++ } ++ if (S_ISSOCK(mode)) ++ return true; ++ if (S_ISREG(mode)) { ++ if (IS_ENABLED(CONFIG_BLOCK) && ++ io_bdev_nowait(file->f_inode->i_sb->s_bdev) && ++ file->f_op != &io_uring_fops) ++ return true; ++ return false; ++ } ++ ++ /* any ->read/write should understand O_NONBLOCK */ ++ if (file->f_flags & O_NONBLOCK) ++ return true; ++ ++ if (!(file->f_mode & FMODE_NOWAIT)) ++ return false; ++ ++ if (rw == READ) ++ return file->f_op->read_iter != NULL; ++ ++ return file->f_op->write_iter != NULL; ++} ++ ++static bool io_file_supports_nowait(struct io_kiocb *req, int rw) ++{ ++ if (rw == READ && (req->flags & REQ_F_NOWAIT_READ)) ++ return true; ++ else if (rw == WRITE && (req->flags & REQ_F_NOWAIT_WRITE)) ++ return true; ++ ++ return __io_file_supports_nowait(req->file, rw); ++} ++ ++static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, ++ int rw) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ struct kiocb *kiocb = &req->rw.kiocb; ++ struct file *file = req->file; ++ unsigned ioprio; ++ int ret; ++ ++ if (!io_req_ffs_set(req) && S_ISREG(file_inode(file)->i_mode)) ++ req->flags |= REQ_F_ISREG; ++ ++ kiocb->ki_pos = READ_ONCE(sqe->off); ++ if (kiocb->ki_pos == -1) { ++ if (!(file->f_mode & FMODE_STREAM)) { ++ req->flags |= REQ_F_CUR_POS; ++ kiocb->ki_pos = file->f_pos; ++ } else { ++ kiocb->ki_pos = 0; ++ } ++ } ++ kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp)); ++ kiocb->ki_flags = iocb_flags(kiocb->ki_filp); ++ ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags)); ++ if (unlikely(ret)) ++ return ret; ++ ++ /* ++ * If the file is marked O_NONBLOCK, still allow retry for it if it ++ * supports async. Otherwise it's impossible to use O_NONBLOCK files ++ * reliably. If not, or it IOCB_NOWAIT is set, don't retry. ++ */ ++ if ((kiocb->ki_flags & IOCB_NOWAIT) || ++ ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req, rw))) ++ req->flags |= REQ_F_NOWAIT; ++ ++ ioprio = READ_ONCE(sqe->ioprio); ++ if (ioprio) { ++ ret = ioprio_check_cap(ioprio); ++ if (ret) ++ return ret; ++ ++ kiocb->ki_ioprio = ioprio; ++ } else ++ kiocb->ki_ioprio = get_current_ioprio(); ++ ++ if (ctx->flags & IORING_SETUP_IOPOLL) { ++ if (!(kiocb->ki_flags & IOCB_DIRECT) || ++ !kiocb->ki_filp->f_op->iopoll) ++ return -EOPNOTSUPP; ++ ++ kiocb->ki_flags |= IOCB_HIPRI; ++ kiocb->ki_complete = io_complete_rw_iopoll; ++ req->iopoll_completed = 0; ++ } else { ++ if (kiocb->ki_flags & IOCB_HIPRI) ++ return -EINVAL; ++ kiocb->ki_complete = io_complete_rw; ++ } ++ ++ /* used for fixed read/write too - just read unconditionally */ ++ req->buf_index = READ_ONCE(sqe->buf_index); ++ req->imu = NULL; ++ ++ if (req->opcode == IORING_OP_READ_FIXED || ++ req->opcode == IORING_OP_WRITE_FIXED) { ++ struct io_ring_ctx *ctx = req->ctx; ++ u16 index; ++ ++ if (unlikely(req->buf_index >= ctx->nr_user_bufs)) ++ return -EFAULT; ++ index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); ++ req->imu = ctx->user_bufs[index]; ++ io_req_set_rsrc_node(req); ++ } ++ ++ req->rw.addr = READ_ONCE(sqe->addr); ++ req->rw.len = READ_ONCE(sqe->len); ++ return 0; ++} ++ ++static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) ++{ ++ switch (ret) { ++ case -EIOCBQUEUED: ++ break; ++ case -ERESTARTSYS: ++ case -ERESTARTNOINTR: ++ case -ERESTARTNOHAND: ++ case -ERESTART_RESTARTBLOCK: ++ /* ++ * We can't just restart the syscall, since previously ++ * submitted sqes may already be in progress. Just fail this ++ * IO with EINTR. ++ */ ++ ret = -EINTR; ++ fallthrough; ++ default: ++ kiocb->ki_complete(kiocb, ret, 0); ++ } ++} ++ ++static void kiocb_done(struct kiocb *kiocb, ssize_t ret, ++ unsigned int issue_flags) ++{ ++ struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); ++ ++ if (req->flags & REQ_F_CUR_POS) ++ req->file->f_pos = kiocb->ki_pos; ++ if (ret >= 0 && (kiocb->ki_complete == io_complete_rw)) ++ __io_complete_rw(req, ret, 0, issue_flags); ++ else ++ io_rw_done(kiocb, ret); ++ ++ if (req->flags & REQ_F_REISSUE) { ++ req->flags &= ~REQ_F_REISSUE; ++ if (io_resubmit_prep(req)) { ++ io_req_task_queue_reissue(req); ++ } else { ++ unsigned int cflags = io_put_rw_kbuf(req); ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ ret = io_fixup_rw_res(req, ret); ++ req_set_fail(req); ++ if (!(issue_flags & IO_URING_F_NONBLOCK)) { ++ mutex_lock(&ctx->uring_lock); ++ __io_req_complete(req, issue_flags, ret, cflags); ++ mutex_unlock(&ctx->uring_lock); ++ } else { ++ __io_req_complete(req, issue_flags, ret, cflags); ++ } ++ } ++ } ++} ++ ++static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter, ++ struct io_mapped_ubuf *imu) ++{ ++ size_t len = req->rw.len; ++ u64 buf_end, buf_addr = req->rw.addr; ++ size_t offset; ++ ++ if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end))) ++ return -EFAULT; ++ /* not inside the mapped region */ ++ if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end)) ++ return -EFAULT; ++ ++ /* ++ * May not be a start of buffer, set size appropriately ++ * and advance us to the beginning. ++ */ ++ offset = buf_addr - imu->ubuf; ++ iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len); ++ ++ if (offset) { ++ /* ++ * Don't use iov_iter_advance() here, as it's really slow for ++ * using the latter parts of a big fixed buffer - it iterates ++ * over each segment manually. We can cheat a bit here, because ++ * we know that: ++ * ++ * 1) it's a BVEC iter, we set it up ++ * 2) all bvecs are PAGE_SIZE in size, except potentially the ++ * first and last bvec ++ * ++ * So just find our index, and adjust the iterator afterwards. ++ * If the offset is within the first bvec (or the whole first ++ * bvec, just use iov_iter_advance(). This makes it easier ++ * since we can just skip the first segment, which may not ++ * be PAGE_SIZE aligned. ++ */ ++ const struct bio_vec *bvec = imu->bvec; ++ ++ if (offset <= bvec->bv_len) { ++ iov_iter_advance(iter, offset); ++ } else { ++ unsigned long seg_skip; ++ ++ /* skip first vec */ ++ offset -= bvec->bv_len; ++ seg_skip = 1 + (offset >> PAGE_SHIFT); ++ ++ iter->bvec = bvec + seg_skip; ++ iter->nr_segs -= seg_skip; ++ iter->count -= bvec->bv_len + offset; ++ iter->iov_offset = offset & ~PAGE_MASK; ++ } ++ } ++ ++ return 0; ++} ++ ++static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter) ++{ ++ if (WARN_ON_ONCE(!req->imu)) ++ return -EFAULT; ++ return __io_import_fixed(req, rw, iter, req->imu); ++} ++ ++static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock) ++{ ++ if (needs_lock) ++ mutex_unlock(&ctx->uring_lock); ++} ++ ++static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock) ++{ ++ /* ++ * "Normal" inline submissions always hold the uring_lock, since we ++ * grab it from the system call. Same is true for the SQPOLL offload. ++ * The only exception is when we've detached the request and issue it ++ * from an async worker thread, grab the lock for that case. ++ */ ++ if (needs_lock) ++ mutex_lock(&ctx->uring_lock); ++} ++ ++static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len, ++ int bgid, struct io_buffer *kbuf, ++ bool needs_lock) ++{ ++ struct io_buffer *head; ++ ++ if (req->flags & REQ_F_BUFFER_SELECTED) ++ return kbuf; ++ ++ io_ring_submit_lock(req->ctx, needs_lock); ++ ++ lockdep_assert_held(&req->ctx->uring_lock); ++ ++ head = xa_load(&req->ctx->io_buffers, bgid); ++ if (head) { ++ if (!list_empty(&head->list)) { ++ kbuf = list_last_entry(&head->list, struct io_buffer, ++ list); ++ list_del(&kbuf->list); ++ } else { ++ kbuf = head; ++ xa_erase(&req->ctx->io_buffers, bgid); ++ } ++ if (*len > kbuf->len) ++ *len = kbuf->len; ++ } else { ++ kbuf = ERR_PTR(-ENOBUFS); ++ } ++ ++ io_ring_submit_unlock(req->ctx, needs_lock); ++ ++ return kbuf; ++} ++ ++static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len, ++ bool needs_lock) ++{ ++ struct io_buffer *kbuf; ++ u16 bgid; ++ ++ kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; ++ bgid = req->buf_index; ++ kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock); ++ if (IS_ERR(kbuf)) ++ return kbuf; ++ req->rw.addr = (u64) (unsigned long) kbuf; ++ req->flags |= REQ_F_BUFFER_SELECTED; ++ return u64_to_user_ptr(kbuf->addr); ++} ++ ++#ifdef CONFIG_COMPAT ++static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, ++ bool needs_lock) ++{ ++ struct compat_iovec __user *uiov; ++ compat_ssize_t clen; ++ void __user *buf; ++ ssize_t len; ++ ++ uiov = u64_to_user_ptr(req->rw.addr); ++ if (!access_ok(uiov, sizeof(*uiov))) ++ return -EFAULT; ++ if (__get_user(clen, &uiov->iov_len)) ++ return -EFAULT; ++ if (clen < 0) ++ return -EINVAL; ++ ++ len = clen; ++ buf = io_rw_buffer_select(req, &len, needs_lock); ++ if (IS_ERR(buf)) ++ return PTR_ERR(buf); ++ iov[0].iov_base = buf; ++ iov[0].iov_len = (compat_size_t) len; ++ return 0; ++} ++#endif ++ ++static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, ++ bool needs_lock) ++{ ++ struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr); ++ void __user *buf; ++ ssize_t len; ++ ++ if (copy_from_user(iov, uiov, sizeof(*uiov))) ++ return -EFAULT; ++ ++ len = iov[0].iov_len; ++ if (len < 0) ++ return -EINVAL; ++ buf = io_rw_buffer_select(req, &len, needs_lock); ++ if (IS_ERR(buf)) ++ return PTR_ERR(buf); ++ iov[0].iov_base = buf; ++ iov[0].iov_len = len; ++ return 0; ++} ++ ++static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, ++ bool needs_lock) ++{ ++ if (req->flags & REQ_F_BUFFER_SELECTED) { ++ struct io_buffer *kbuf; ++ ++ kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; ++ iov[0].iov_base = u64_to_user_ptr(kbuf->addr); ++ iov[0].iov_len = kbuf->len; ++ return 0; ++ } ++ if (req->rw.len != 1) ++ return -EINVAL; ++ ++#ifdef CONFIG_COMPAT ++ if (req->ctx->compat) ++ return io_compat_import(req, iov, needs_lock); ++#endif ++ ++ return __io_iov_buffer_select(req, iov, needs_lock); ++} ++ ++static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, ++ struct iov_iter *iter, bool needs_lock) ++{ ++ void __user *buf = u64_to_user_ptr(req->rw.addr); ++ size_t sqe_len = req->rw.len; ++ u8 opcode = req->opcode; ++ ssize_t ret; ++ ++ if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) { ++ *iovec = NULL; ++ return io_import_fixed(req, rw, iter); ++ } ++ ++ /* buffer index only valid with fixed read/write, or buffer select */ ++ if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT)) ++ return -EINVAL; ++ ++ if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) { ++ if (req->flags & REQ_F_BUFFER_SELECT) { ++ buf = io_rw_buffer_select(req, &sqe_len, needs_lock); ++ if (IS_ERR(buf)) ++ return PTR_ERR(buf); ++ req->rw.len = sqe_len; ++ } ++ ++ ret = import_single_range(rw, buf, sqe_len, *iovec, iter); ++ *iovec = NULL; ++ return ret; ++ } ++ ++ if (req->flags & REQ_F_BUFFER_SELECT) { ++ ret = io_iov_buffer_select(req, *iovec, needs_lock); ++ if (!ret) ++ iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len); ++ *iovec = NULL; ++ return ret; ++ } ++ ++ return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter, ++ req->ctx->compat); ++} ++ ++static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb) ++{ ++ return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos; ++} ++ ++/* ++ * For files that don't have ->read_iter() and ->write_iter(), handle them ++ * by looping over ->read() or ->write() manually. ++ */ ++static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) ++{ ++ struct kiocb *kiocb = &req->rw.kiocb; ++ struct file *file = req->file; ++ ssize_t ret = 0; ++ ++ /* ++ * Don't support polled IO through this interface, and we can't ++ * support non-blocking either. For the latter, this just causes ++ * the kiocb to be handled from an async context. ++ */ ++ if (kiocb->ki_flags & IOCB_HIPRI) ++ return -EOPNOTSUPP; ++ if (kiocb->ki_flags & IOCB_NOWAIT) ++ return -EAGAIN; ++ ++ while (iov_iter_count(iter)) { ++ struct iovec iovec; ++ ssize_t nr; ++ ++ if (!iov_iter_is_bvec(iter)) { ++ iovec = iov_iter_iovec(iter); ++ } else { ++ iovec.iov_base = u64_to_user_ptr(req->rw.addr); ++ iovec.iov_len = req->rw.len; ++ } ++ ++ if (rw == READ) { ++ nr = file->f_op->read(file, iovec.iov_base, ++ iovec.iov_len, io_kiocb_ppos(kiocb)); ++ } else { ++ nr = file->f_op->write(file, iovec.iov_base, ++ iovec.iov_len, io_kiocb_ppos(kiocb)); ++ } ++ ++ if (nr < 0) { ++ if (!ret) ++ ret = nr; ++ break; ++ } ++ ret += nr; ++ if (!iov_iter_is_bvec(iter)) { ++ iov_iter_advance(iter, nr); ++ } else { ++ req->rw.addr += nr; ++ req->rw.len -= nr; ++ if (!req->rw.len) ++ break; ++ } ++ if (nr != iovec.iov_len) ++ break; ++ } ++ ++ return ret; ++} ++ ++static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, ++ const struct iovec *fast_iov, struct iov_iter *iter) ++{ ++ struct io_async_rw *rw = req->async_data; ++ ++ memcpy(&rw->iter, iter, sizeof(*iter)); ++ rw->free_iovec = iovec; ++ rw->bytes_done = 0; ++ /* can only be fixed buffers, no need to do anything */ ++ if (iov_iter_is_bvec(iter)) ++ return; ++ if (!iovec) { ++ unsigned iov_off = 0; ++ ++ rw->iter.iov = rw->fast_iov; ++ if (iter->iov != fast_iov) { ++ iov_off = iter->iov - fast_iov; ++ rw->iter.iov += iov_off; ++ } ++ if (rw->fast_iov != fast_iov) ++ memcpy(rw->fast_iov + iov_off, fast_iov + iov_off, ++ sizeof(struct iovec) * iter->nr_segs); ++ } else { ++ req->flags |= REQ_F_NEED_CLEANUP; ++ } ++} ++ ++static inline int io_alloc_async_data(struct io_kiocb *req) ++{ ++ WARN_ON_ONCE(!io_op_defs[req->opcode].async_size); ++ req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL); ++ return req->async_data == NULL; ++} ++ ++static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, ++ const struct iovec *fast_iov, ++ struct iov_iter *iter, bool force) ++{ ++ if (!force && !io_op_defs[req->opcode].needs_async_setup) ++ return 0; ++ if (!req->async_data) { ++ struct io_async_rw *iorw; ++ ++ if (io_alloc_async_data(req)) { ++ kfree(iovec); ++ return -ENOMEM; ++ } ++ ++ io_req_map_rw(req, iovec, fast_iov, iter); ++ iorw = req->async_data; ++ /* we've copied and mapped the iter, ensure state is saved */ ++ iov_iter_save_state(&iorw->iter, &iorw->iter_state); ++ } ++ return 0; ++} ++ ++static inline int io_rw_prep_async(struct io_kiocb *req, int rw) ++{ ++ struct io_async_rw *iorw = req->async_data; ++ struct iovec *iov = iorw->fast_iov; ++ int ret; ++ ++ ret = io_import_iovec(rw, req, &iov, &iorw->iter, false); ++ if (unlikely(ret < 0)) ++ return ret; ++ ++ iorw->bytes_done = 0; ++ iorw->free_iovec = iov; ++ if (iov) ++ req->flags |= REQ_F_NEED_CLEANUP; ++ iov_iter_save_state(&iorw->iter, &iorw->iter_state); ++ return 0; ++} ++ ++static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ if (unlikely(!(req->file->f_mode & FMODE_READ))) ++ return -EBADF; ++ return io_prep_rw(req, sqe, READ); ++} ++ ++/* ++ * This is our waitqueue callback handler, registered through lock_page_async() ++ * when we initially tried to do the IO with the iocb armed our waitqueue. ++ * This gets called when the page is unlocked, and we generally expect that to ++ * happen when the page IO is completed and the page is now uptodate. This will ++ * queue a task_work based retry of the operation, attempting to copy the data ++ * again. If the latter fails because the page was NOT uptodate, then we will ++ * do a thread based blocking retry of the operation. That's the unexpected ++ * slow path. ++ */ ++static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, ++ int sync, void *arg) ++{ ++ struct wait_page_queue *wpq; ++ struct io_kiocb *req = wait->private; ++ struct wait_page_key *key = arg; ++ ++ wpq = container_of(wait, struct wait_page_queue, wait); ++ ++ if (!wake_page_match(wpq, key)) ++ return 0; ++ ++ req->rw.kiocb.ki_flags &= ~IOCB_WAITQ; ++ list_del_init(&wait->entry); ++ io_req_task_queue(req); ++ return 1; ++} ++ ++/* ++ * This controls whether a given IO request should be armed for async page ++ * based retry. If we return false here, the request is handed to the async ++ * worker threads for retry. If we're doing buffered reads on a regular file, ++ * we prepare a private wait_page_queue entry and retry the operation. This ++ * will either succeed because the page is now uptodate and unlocked, or it ++ * will register a callback when the page is unlocked at IO completion. Through ++ * that callback, io_uring uses task_work to setup a retry of the operation. ++ * That retry will attempt the buffered read again. The retry will generally ++ * succeed, or in rare cases where it fails, we then fall back to using the ++ * async worker threads for a blocking retry. ++ */ ++static bool io_rw_should_retry(struct io_kiocb *req) ++{ ++ struct io_async_rw *rw = req->async_data; ++ struct wait_page_queue *wait = &rw->wpq; ++ struct kiocb *kiocb = &req->rw.kiocb; ++ ++ /* never retry for NOWAIT, we just complete with -EAGAIN */ ++ if (req->flags & REQ_F_NOWAIT) ++ return false; ++ ++ /* Only for buffered IO */ ++ if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI)) ++ return false; ++ ++ /* ++ * just use poll if we can, and don't attempt if the fs doesn't ++ * support callback based unlocks ++ */ ++ if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC)) ++ return false; ++ ++ wait->wait.func = io_async_buf_func; ++ wait->wait.private = req; ++ wait->wait.flags = 0; ++ INIT_LIST_HEAD(&wait->wait.entry); ++ kiocb->ki_flags |= IOCB_WAITQ; ++ kiocb->ki_flags &= ~IOCB_NOWAIT; ++ kiocb->ki_waitq = wait; ++ return true; ++} ++ ++static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter) ++{ ++ if (req->file->f_op->read_iter) ++ return call_read_iter(req->file, &req->rw.kiocb, iter); ++ else if (req->file->f_op->read) ++ return loop_rw_iter(READ, req, iter); ++ else ++ return -EINVAL; ++} ++ ++static bool need_read_all(struct io_kiocb *req) ++{ ++ return req->flags & REQ_F_ISREG || ++ S_ISBLK(file_inode(req->file)->i_mode); ++} ++ ++static int io_read(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; ++ struct kiocb *kiocb = &req->rw.kiocb; ++ struct iov_iter __iter, *iter = &__iter; ++ struct io_async_rw *rw = req->async_data; ++ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ++ struct iov_iter_state __state, *state; ++ ssize_t ret, ret2; ++ ++ if (rw) { ++ iter = &rw->iter; ++ state = &rw->iter_state; ++ /* ++ * We come here from an earlier attempt, restore our state to ++ * match in case it doesn't. It's cheap enough that we don't ++ * need to make this conditional. ++ */ ++ iov_iter_restore(iter, state); ++ iovec = NULL; ++ } else { ++ ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock); ++ if (ret < 0) ++ return ret; ++ state = &__state; ++ iov_iter_save_state(iter, state); ++ } ++ req->result = iov_iter_count(iter); ++ ++ /* Ensure we clear previously set non-block flag */ ++ if (!force_nonblock) ++ kiocb->ki_flags &= ~IOCB_NOWAIT; ++ else ++ kiocb->ki_flags |= IOCB_NOWAIT; ++ ++ /* If the file doesn't support async, just async punt */ ++ if (force_nonblock && !io_file_supports_nowait(req, READ)) { ++ ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true); ++ return ret ?: -EAGAIN; ++ } ++ ++ ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), req->result); ++ if (unlikely(ret)) { ++ kfree(iovec); ++ return ret; ++ } ++ ++ ret = io_iter_do_read(req, iter); ++ ++ if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { ++ req->flags &= ~REQ_F_REISSUE; ++ /* IOPOLL retry should happen for io-wq threads */ ++ if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ goto done; ++ /* no retry on NONBLOCK nor RWF_NOWAIT */ ++ if (req->flags & REQ_F_NOWAIT) ++ goto done; ++ ret = 0; ++ } else if (ret == -EIOCBQUEUED) { ++ goto out_free; ++ } else if (ret <= 0 || ret == req->result || !force_nonblock || ++ (req->flags & REQ_F_NOWAIT) || !need_read_all(req)) { ++ /* read all, failed, already did sync or don't want to retry */ ++ goto done; ++ } ++ ++ /* ++ * Don't depend on the iter state matching what was consumed, or being ++ * untouched in case of error. Restore it and we'll advance it ++ * manually if we need to. ++ */ ++ iov_iter_restore(iter, state); ++ ++ ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true); ++ if (ret2) ++ return ret2; ++ ++ iovec = NULL; ++ rw = req->async_data; ++ /* ++ * Now use our persistent iterator and state, if we aren't already. ++ * We've restored and mapped the iter to match. ++ */ ++ if (iter != &rw->iter) { ++ iter = &rw->iter; ++ state = &rw->iter_state; ++ } ++ ++ do { ++ /* ++ * We end up here because of a partial read, either from ++ * above or inside this loop. Advance the iter by the bytes ++ * that were consumed. ++ */ ++ iov_iter_advance(iter, ret); ++ if (!iov_iter_count(iter)) ++ break; ++ rw->bytes_done += ret; ++ iov_iter_save_state(iter, state); ++ ++ /* if we can retry, do so with the callbacks armed */ ++ if (!io_rw_should_retry(req)) { ++ kiocb->ki_flags &= ~IOCB_WAITQ; ++ return -EAGAIN; ++ } ++ ++ req->result = iov_iter_count(iter); ++ /* ++ * Now retry read with the IOCB_WAITQ parts set in the iocb. If ++ * we get -EIOCBQUEUED, then we'll get a notification when the ++ * desired page gets unlocked. We can also get a partial read ++ * here, and if we do, then just retry at the new offset. ++ */ ++ ret = io_iter_do_read(req, iter); ++ if (ret == -EIOCBQUEUED) ++ return 0; ++ /* we got some bytes, but not all. retry. */ ++ kiocb->ki_flags &= ~IOCB_WAITQ; ++ iov_iter_restore(iter, state); ++ } while (ret > 0); ++done: ++ kiocb_done(kiocb, ret, issue_flags); ++out_free: ++ /* it's faster to check here then delegate to kfree */ ++ if (iovec) ++ kfree(iovec); ++ return 0; ++} ++ ++static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ if (unlikely(!(req->file->f_mode & FMODE_WRITE))) ++ return -EBADF; ++ return io_prep_rw(req, sqe, WRITE); ++} ++ ++static int io_write(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; ++ struct kiocb *kiocb = &req->rw.kiocb; ++ struct iov_iter __iter, *iter = &__iter; ++ struct io_async_rw *rw = req->async_data; ++ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ++ struct iov_iter_state __state, *state; ++ ssize_t ret, ret2; ++ ++ if (rw) { ++ iter = &rw->iter; ++ state = &rw->iter_state; ++ iov_iter_restore(iter, state); ++ iovec = NULL; ++ } else { ++ ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock); ++ if (ret < 0) ++ return ret; ++ state = &__state; ++ iov_iter_save_state(iter, state); ++ } ++ req->result = iov_iter_count(iter); ++ ++ /* Ensure we clear previously set non-block flag */ ++ if (!force_nonblock) ++ kiocb->ki_flags &= ~IOCB_NOWAIT; ++ else ++ kiocb->ki_flags |= IOCB_NOWAIT; ++ ++ /* If the file doesn't support async, just async punt */ ++ if (force_nonblock && !io_file_supports_nowait(req, WRITE)) ++ goto copy_iov; ++ ++ /* file path doesn't support NOWAIT for non-direct_IO */ ++ if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) && ++ (req->flags & REQ_F_ISREG)) ++ goto copy_iov; ++ ++ ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), req->result); ++ if (unlikely(ret)) ++ goto out_free; ++ ++ /* ++ * Open-code file_start_write here to grab freeze protection, ++ * which will be released by another thread in ++ * io_complete_rw(). Fool lockdep by telling it the lock got ++ * released so that it doesn't complain about the held lock when ++ * we return to userspace. ++ */ ++ if (req->flags & REQ_F_ISREG) { ++ sb_start_write(file_inode(req->file)->i_sb); ++ __sb_writers_release(file_inode(req->file)->i_sb, ++ SB_FREEZE_WRITE); ++ } ++ kiocb->ki_flags |= IOCB_WRITE; ++ ++ if (req->file->f_op->write_iter) ++ ret2 = call_write_iter(req->file, kiocb, iter); ++ else if (req->file->f_op->write) ++ ret2 = loop_rw_iter(WRITE, req, iter); ++ else ++ ret2 = -EINVAL; ++ ++ if (req->flags & REQ_F_REISSUE) { ++ req->flags &= ~REQ_F_REISSUE; ++ ret2 = -EAGAIN; ++ } ++ ++ /* ++ * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just ++ * retry them without IOCB_NOWAIT. ++ */ ++ if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) ++ ret2 = -EAGAIN; ++ /* no retry on NONBLOCK nor RWF_NOWAIT */ ++ if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT)) ++ goto done; ++ if (!force_nonblock || ret2 != -EAGAIN) { ++ /* IOPOLL retry should happen for io-wq threads */ ++ if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN) ++ goto copy_iov; ++done: ++ kiocb_done(kiocb, ret2, issue_flags); ++ } else { ++copy_iov: ++ iov_iter_restore(iter, state); ++ ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false); ++ if (!ret) { ++ if (kiocb->ki_flags & IOCB_WRITE) ++ kiocb_end_write(req); ++ return -EAGAIN; ++ } ++ return ret; ++ } ++out_free: ++ /* it's reportedly faster than delegating the null check to kfree() */ ++ if (iovec) ++ kfree(iovec); ++ return ret; ++} ++ ++static int io_renameat_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++ struct io_rename *ren = &req->rename; ++ const char __user *oldf, *newf; ++ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) ++ return -EINVAL; ++ if (unlikely(req->flags & REQ_F_FIXED_FILE)) ++ return -EBADF; ++ ++ ren->old_dfd = READ_ONCE(sqe->fd); ++ oldf = u64_to_user_ptr(READ_ONCE(sqe->addr)); ++ newf = u64_to_user_ptr(READ_ONCE(sqe->addr2)); ++ ren->new_dfd = READ_ONCE(sqe->len); ++ ren->flags = READ_ONCE(sqe->rename_flags); ++ ++ ren->oldpath = getname(oldf); ++ if (IS_ERR(ren->oldpath)) ++ return PTR_ERR(ren->oldpath); ++ ++ ren->newpath = getname(newf); ++ if (IS_ERR(ren->newpath)) { ++ putname(ren->oldpath); ++ return PTR_ERR(ren->newpath); ++ } ++ ++ req->flags |= REQ_F_NEED_CLEANUP; ++ return 0; ++} ++ ++static int io_renameat(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_rename *ren = &req->rename; ++ int ret; ++ ++ if (issue_flags & IO_URING_F_NONBLOCK) ++ return -EAGAIN; ++ ++ ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd, ++ ren->newpath, ren->flags); ++ ++ req->flags &= ~REQ_F_NEED_CLEANUP; ++ if (ret < 0) ++ req_set_fail(req); ++ io_req_complete(req, ret); ++ return 0; ++} ++ ++static int io_unlinkat_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++ struct io_unlink *un = &req->unlink; ++ const char __user *fname; ++ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index || ++ sqe->splice_fd_in) ++ return -EINVAL; ++ if (unlikely(req->flags & REQ_F_FIXED_FILE)) ++ return -EBADF; ++ ++ un->dfd = READ_ONCE(sqe->fd); ++ ++ un->flags = READ_ONCE(sqe->unlink_flags); ++ if (un->flags & ~AT_REMOVEDIR) ++ return -EINVAL; ++ ++ fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); ++ un->filename = getname(fname); ++ if (IS_ERR(un->filename)) ++ return PTR_ERR(un->filename); ++ ++ req->flags |= REQ_F_NEED_CLEANUP; ++ return 0; ++} ++ ++static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_unlink *un = &req->unlink; ++ int ret; ++ ++ if (issue_flags & IO_URING_F_NONBLOCK) ++ return -EAGAIN; ++ ++ if (un->flags & AT_REMOVEDIR) ++ ret = do_rmdir(un->dfd, un->filename); ++ else ++ ret = do_unlinkat(un->dfd, un->filename); ++ ++ req->flags &= ~REQ_F_NEED_CLEANUP; ++ if (ret < 0) ++ req_set_fail(req); ++ io_req_complete(req, ret); ++ return 0; ++} ++ ++static int io_shutdown_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++#if defined(CONFIG_NET) ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (unlikely(sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags || ++ sqe->buf_index || sqe->splice_fd_in)) ++ return -EINVAL; ++ ++ req->shutdown.how = READ_ONCE(sqe->len); ++ return 0; ++#else ++ return -EOPNOTSUPP; ++#endif ++} ++ ++static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) ++{ ++#if defined(CONFIG_NET) ++ struct socket *sock; ++ int ret; ++ ++ if (issue_flags & IO_URING_F_NONBLOCK) ++ return -EAGAIN; ++ ++ sock = sock_from_file(req->file, &ret); ++ if (unlikely(!sock)) ++ return ret; ++ ++ ret = __sys_shutdown_sock(sock, req->shutdown.how); ++ if (ret < 0) ++ req_set_fail(req); ++ io_req_complete(req, ret); ++ return 0; ++#else ++ return -EOPNOTSUPP; ++#endif ++} ++ ++static int __io_splice_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++ struct io_splice *sp = &req->splice; ++ unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL; ++ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ ++ sp->len = READ_ONCE(sqe->len); ++ sp->flags = READ_ONCE(sqe->splice_flags); ++ if (unlikely(sp->flags & ~valid_flags)) ++ return -EINVAL; ++ sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in); ++ return 0; ++} ++ ++static int io_tee_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++ if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off)) ++ return -EINVAL; ++ return __io_splice_prep(req, sqe); ++} ++ ++static int io_tee(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_splice *sp = &req->splice; ++ struct file *out = sp->file_out; ++ unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; ++ struct file *in; ++ long ret = 0; ++ ++ if (issue_flags & IO_URING_F_NONBLOCK) ++ return -EAGAIN; ++ ++ in = io_file_get(req->ctx, req, sp->splice_fd_in, ++ (sp->flags & SPLICE_F_FD_IN_FIXED)); ++ if (!in) { ++ ret = -EBADF; ++ goto done; ++ } ++ ++ if (sp->len) ++ ret = do_tee(in, out, sp->len, flags); ++ ++ if (!(sp->flags & SPLICE_F_FD_IN_FIXED)) ++ io_put_file(in); ++done: ++ if (ret != sp->len) ++ req_set_fail(req); ++ io_req_complete(req, ret); ++ return 0; ++} ++ ++static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ struct io_splice *sp = &req->splice; ++ ++ sp->off_in = READ_ONCE(sqe->splice_off_in); ++ sp->off_out = READ_ONCE(sqe->off); ++ return __io_splice_prep(req, sqe); ++} ++ ++static int io_splice(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_splice *sp = &req->splice; ++ struct file *out = sp->file_out; ++ unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; ++ loff_t *poff_in, *poff_out; ++ struct file *in; ++ long ret = 0; ++ ++ if (issue_flags & IO_URING_F_NONBLOCK) ++ return -EAGAIN; ++ ++ in = io_file_get(req->ctx, req, sp->splice_fd_in, ++ (sp->flags & SPLICE_F_FD_IN_FIXED)); ++ if (!in) { ++ ret = -EBADF; ++ goto done; ++ } ++ ++ poff_in = (sp->off_in == -1) ? NULL : &sp->off_in; ++ poff_out = (sp->off_out == -1) ? NULL : &sp->off_out; ++ ++ if (sp->len) ++ ret = do_splice(in, poff_in, out, poff_out, sp->len, flags); ++ ++ if (!(sp->flags & SPLICE_F_FD_IN_FIXED)) ++ io_put_file(in); ++done: ++ if (ret != sp->len) ++ req_set_fail(req); ++ io_req_complete(req, ret); ++ return 0; ++} ++ ++/* ++ * IORING_OP_NOP just posts a completion event, nothing else. ++ */ ++static int io_nop(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ ++ __io_req_complete(req, issue_flags, 0, 0); ++ return 0; ++} ++ ++static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index || ++ sqe->splice_fd_in)) ++ return -EINVAL; ++ ++ req->sync.flags = READ_ONCE(sqe->fsync_flags); ++ if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC)) ++ return -EINVAL; ++ ++ req->sync.off = READ_ONCE(sqe->off); ++ req->sync.len = READ_ONCE(sqe->len); ++ return 0; ++} ++ ++static int io_fsync(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ loff_t end = req->sync.off + req->sync.len; ++ int ret; ++ ++ /* fsync always requires a blocking context */ ++ if (issue_flags & IO_URING_F_NONBLOCK) ++ return -EAGAIN; ++ ++ ret = vfs_fsync_range(req->file, req->sync.off, ++ end > 0 ? end : LLONG_MAX, ++ req->sync.flags & IORING_FSYNC_DATASYNC); ++ if (ret < 0) ++ req_set_fail(req); ++ io_req_complete(req, ret); ++ return 0; ++} ++ ++static int io_fallocate_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++ if (sqe->ioprio || sqe->buf_index || sqe->rw_flags || ++ sqe->splice_fd_in) ++ return -EINVAL; ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ ++ req->sync.off = READ_ONCE(sqe->off); ++ req->sync.len = READ_ONCE(sqe->addr); ++ req->sync.mode = READ_ONCE(sqe->len); ++ return 0; ++} ++ ++static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ int ret; ++ ++ /* fallocate always requiring blocking context */ ++ if (issue_flags & IO_URING_F_NONBLOCK) ++ return -EAGAIN; ++ ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off, ++ req->sync.len); ++ if (ret < 0) ++ req_set_fail(req); ++ else ++ fsnotify_modify(req->file); ++ io_req_complete(req, ret); ++ return 0; ++} ++ ++static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ const char __user *fname; ++ int ret; ++ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (unlikely(sqe->ioprio || sqe->buf_index)) ++ return -EINVAL; ++ if (unlikely(req->flags & REQ_F_FIXED_FILE)) ++ return -EBADF; ++ ++ /* open.how should be already initialised */ ++ if (!(req->open.how.flags & O_PATH) && force_o_largefile()) ++ req->open.how.flags |= O_LARGEFILE; ++ ++ req->open.dfd = READ_ONCE(sqe->fd); ++ fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); ++ req->open.filename = getname(fname); ++ if (IS_ERR(req->open.filename)) { ++ ret = PTR_ERR(req->open.filename); ++ req->open.filename = NULL; ++ return ret; ++ } ++ ++ req->open.file_slot = READ_ONCE(sqe->file_index); ++ if (req->open.file_slot && (req->open.how.flags & O_CLOEXEC)) ++ return -EINVAL; ++ ++ req->open.nofile = rlimit(RLIMIT_NOFILE); ++ req->flags |= REQ_F_NEED_CLEANUP; ++ return 0; ++} ++ ++static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ u64 mode = READ_ONCE(sqe->len); ++ u64 flags = READ_ONCE(sqe->open_flags); ++ ++ req->open.how = build_open_how(flags, mode); ++ return __io_openat_prep(req, sqe); ++} ++ ++static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ struct open_how __user *how; ++ size_t len; ++ int ret; ++ ++ how = u64_to_user_ptr(READ_ONCE(sqe->addr2)); ++ len = READ_ONCE(sqe->len); ++ if (len < OPEN_HOW_SIZE_VER0) ++ return -EINVAL; ++ ++ ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how, ++ len); ++ if (ret) ++ return ret; ++ ++ return __io_openat_prep(req, sqe); ++} ++ ++static int io_openat2(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct open_flags op; ++ struct file *file; ++ bool resolve_nonblock, nonblock_set; ++ bool fixed = !!req->open.file_slot; ++ int ret; ++ ++ ret = build_open_flags(&req->open.how, &op); ++ if (ret) ++ goto err; ++ nonblock_set = op.open_flag & O_NONBLOCK; ++ resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED; ++ if (issue_flags & IO_URING_F_NONBLOCK) { ++ /* ++ * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open, ++ * it'll always -EAGAIN ++ */ ++ if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE)) ++ return -EAGAIN; ++ op.lookup_flags |= LOOKUP_CACHED; ++ op.open_flag |= O_NONBLOCK; ++ } ++ ++ if (!fixed) { ++ ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile); ++ if (ret < 0) ++ goto err; ++ } ++ ++ file = do_filp_open(req->open.dfd, req->open.filename, &op); ++ if (IS_ERR(file)) { ++ /* ++ * We could hang on to this 'fd' on retrying, but seems like ++ * marginal gain for something that is now known to be a slower ++ * path. So just put it, and we'll get a new one when we retry. ++ */ ++ if (!fixed) ++ put_unused_fd(ret); ++ ++ ret = PTR_ERR(file); ++ /* only retry if RESOLVE_CACHED wasn't already set by application */ ++ if (ret == -EAGAIN && ++ (!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK))) ++ return -EAGAIN; ++ goto err; ++ } ++ ++ if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set) ++ file->f_flags &= ~O_NONBLOCK; ++ fsnotify_open(file); ++ ++ if (!fixed) ++ fd_install(ret, file); ++ else ++ ret = io_install_fixed_file(req, file, issue_flags, ++ req->open.file_slot - 1); ++err: ++ putname(req->open.filename); ++ req->flags &= ~REQ_F_NEED_CLEANUP; ++ if (ret < 0) ++ req_set_fail(req); ++ __io_req_complete(req, issue_flags, ret, 0); ++ return 0; ++} ++ ++static int io_openat(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ return io_openat2(req, issue_flags); ++} ++ ++static int io_remove_buffers_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++ struct io_provide_buf *p = &req->pbuf; ++ u64 tmp; ++ ++ if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off || ++ sqe->splice_fd_in) ++ return -EINVAL; ++ ++ tmp = READ_ONCE(sqe->fd); ++ if (!tmp || tmp > USHRT_MAX) ++ return -EINVAL; ++ ++ memset(p, 0, sizeof(*p)); ++ p->nbufs = tmp; ++ p->bgid = READ_ONCE(sqe->buf_group); ++ return 0; ++} ++ ++static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf, ++ int bgid, unsigned nbufs) ++{ ++ unsigned i = 0; ++ ++ /* shouldn't happen */ ++ if (!nbufs) ++ return 0; ++ ++ /* the head kbuf is the list itself */ ++ while (!list_empty(&buf->list)) { ++ struct io_buffer *nxt; ++ ++ nxt = list_first_entry(&buf->list, struct io_buffer, list); ++ list_del(&nxt->list); ++ kfree(nxt); ++ if (++i == nbufs) ++ return i; ++ cond_resched(); ++ } ++ i++; ++ kfree(buf); ++ xa_erase(&ctx->io_buffers, bgid); ++ ++ return i; ++} ++ ++static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_provide_buf *p = &req->pbuf; ++ struct io_ring_ctx *ctx = req->ctx; ++ struct io_buffer *head; ++ int ret = 0; ++ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ++ ++ io_ring_submit_lock(ctx, !force_nonblock); ++ ++ lockdep_assert_held(&ctx->uring_lock); ++ ++ ret = -ENOENT; ++ head = xa_load(&ctx->io_buffers, p->bgid); ++ if (head) ++ ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs); ++ if (ret < 0) ++ req_set_fail(req); ++ ++ /* complete before unlock, IOPOLL may need the lock */ ++ __io_req_complete(req, issue_flags, ret, 0); ++ io_ring_submit_unlock(ctx, !force_nonblock); ++ return 0; ++} ++ ++static int io_provide_buffers_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++ unsigned long size, tmp_check; ++ struct io_provide_buf *p = &req->pbuf; ++ u64 tmp; ++ ++ if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in) ++ return -EINVAL; ++ ++ tmp = READ_ONCE(sqe->fd); ++ if (!tmp || tmp > USHRT_MAX) ++ return -E2BIG; ++ p->nbufs = tmp; ++ p->addr = READ_ONCE(sqe->addr); ++ p->len = READ_ONCE(sqe->len); ++ ++ if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs, ++ &size)) ++ return -EOVERFLOW; ++ if (check_add_overflow((unsigned long)p->addr, size, &tmp_check)) ++ return -EOVERFLOW; ++ ++ size = (unsigned long)p->len * p->nbufs; ++ if (!access_ok(u64_to_user_ptr(p->addr), size)) ++ return -EFAULT; ++ ++ p->bgid = READ_ONCE(sqe->buf_group); ++ tmp = READ_ONCE(sqe->off); ++ if (tmp > USHRT_MAX) ++ return -E2BIG; ++ p->bid = tmp; ++ return 0; ++} ++ ++static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head) ++{ ++ struct io_buffer *buf; ++ u64 addr = pbuf->addr; ++ int i, bid = pbuf->bid; ++ ++ for (i = 0; i < pbuf->nbufs; i++) { ++ buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT); ++ if (!buf) ++ break; ++ ++ buf->addr = addr; ++ buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT); ++ buf->bid = bid; ++ addr += pbuf->len; ++ bid++; ++ if (!*head) { ++ INIT_LIST_HEAD(&buf->list); ++ *head = buf; ++ } else { ++ list_add_tail(&buf->list, &(*head)->list); ++ } ++ cond_resched(); ++ } ++ ++ return i ? i : -ENOMEM; ++} ++ ++static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_provide_buf *p = &req->pbuf; ++ struct io_ring_ctx *ctx = req->ctx; ++ struct io_buffer *head, *list; ++ int ret = 0; ++ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ++ ++ io_ring_submit_lock(ctx, !force_nonblock); ++ ++ lockdep_assert_held(&ctx->uring_lock); ++ ++ list = head = xa_load(&ctx->io_buffers, p->bgid); ++ ++ ret = io_add_buffers(p, &head); ++ if (ret >= 0 && !list) { ++ ret = xa_insert(&ctx->io_buffers, p->bgid, head, ++ GFP_KERNEL_ACCOUNT); ++ if (ret < 0) ++ __io_remove_buffers(ctx, head, p->bgid, -1U); ++ } ++ if (ret < 0) ++ req_set_fail(req); ++ /* complete before unlock, IOPOLL may need the lock */ ++ __io_req_complete(req, issue_flags, ret, 0); ++ io_ring_submit_unlock(ctx, !force_nonblock); ++ return 0; ++} ++ ++static int io_epoll_ctl_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++#if defined(CONFIG_EPOLL) ++ if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) ++ return -EINVAL; ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ ++ req->epoll.epfd = READ_ONCE(sqe->fd); ++ req->epoll.op = READ_ONCE(sqe->len); ++ req->epoll.fd = READ_ONCE(sqe->off); ++ ++ if (ep_op_has_event(req->epoll.op)) { ++ struct epoll_event __user *ev; ++ ++ ev = u64_to_user_ptr(READ_ONCE(sqe->addr)); ++ if (copy_from_user(&req->epoll.event, ev, sizeof(*ev))) ++ return -EFAULT; ++ } ++ ++ return 0; ++#else ++ return -EOPNOTSUPP; ++#endif ++} ++ ++static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags) ++{ ++#if defined(CONFIG_EPOLL) ++ struct io_epoll *ie = &req->epoll; ++ int ret; ++ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ++ ++ ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock); ++ if (force_nonblock && ret == -EAGAIN) ++ return -EAGAIN; ++ ++ if (ret < 0) ++ req_set_fail(req); ++ __io_req_complete(req, issue_flags, ret, 0); ++ return 0; ++#else ++ return -EOPNOTSUPP; ++#endif ++} ++ ++static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) ++ if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->splice_fd_in) ++ return -EINVAL; ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ ++ req->madvise.addr = READ_ONCE(sqe->addr); ++ req->madvise.len = READ_ONCE(sqe->len); ++ req->madvise.advice = READ_ONCE(sqe->fadvise_advice); ++ return 0; ++#else ++ return -EOPNOTSUPP; ++#endif ++} ++ ++static int io_madvise(struct io_kiocb *req, unsigned int issue_flags) ++{ ++#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) ++ struct io_madvise *ma = &req->madvise; ++ int ret; ++ ++ if (issue_flags & IO_URING_F_NONBLOCK) ++ return -EAGAIN; ++ ++ ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice); ++ if (ret < 0) ++ req_set_fail(req); ++ io_req_complete(req, ret); ++ return 0; ++#else ++ return -EOPNOTSUPP; ++#endif ++} ++ ++static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ if (sqe->ioprio || sqe->buf_index || sqe->addr || sqe->splice_fd_in) ++ return -EINVAL; ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ ++ req->fadvise.offset = READ_ONCE(sqe->off); ++ req->fadvise.len = READ_ONCE(sqe->len); ++ req->fadvise.advice = READ_ONCE(sqe->fadvise_advice); ++ return 0; ++} ++ ++static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_fadvise *fa = &req->fadvise; ++ int ret; ++ ++ if (issue_flags & IO_URING_F_NONBLOCK) { ++ switch (fa->advice) { ++ case POSIX_FADV_NORMAL: ++ case POSIX_FADV_RANDOM: ++ case POSIX_FADV_SEQUENTIAL: ++ break; ++ default: ++ return -EAGAIN; ++ } ++ } ++ ++ ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice); ++ if (ret < 0) ++ req_set_fail(req); ++ __io_req_complete(req, issue_flags, ret, 0); ++ return 0; ++} ++ ++static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) ++ return -EINVAL; ++ if (req->flags & REQ_F_FIXED_FILE) ++ return -EBADF; ++ ++ req->statx.dfd = READ_ONCE(sqe->fd); ++ req->statx.mask = READ_ONCE(sqe->len); ++ req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr)); ++ req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2)); ++ req->statx.flags = READ_ONCE(sqe->statx_flags); ++ ++ return 0; ++} ++ ++static int io_statx(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_statx *ctx = &req->statx; ++ int ret; ++ ++ if (issue_flags & IO_URING_F_NONBLOCK) ++ return -EAGAIN; ++ ++ ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask, ++ ctx->buffer); ++ ++ if (ret < 0) ++ req_set_fail(req); ++ io_req_complete(req, ret); ++ return 0; ++} ++ ++static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (sqe->ioprio || sqe->off || sqe->addr || sqe->len || ++ sqe->rw_flags || sqe->buf_index) ++ return -EINVAL; ++ if (req->flags & REQ_F_FIXED_FILE) ++ return -EBADF; ++ ++ req->close.fd = READ_ONCE(sqe->fd); ++ req->close.file_slot = READ_ONCE(sqe->file_index); ++ if (req->close.file_slot && req->close.fd) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++static int io_close(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct files_struct *files = current->files; ++ struct io_close *close = &req->close; ++ struct fdtable *fdt; ++ struct file *file = NULL; ++ int ret = -EBADF; ++ ++ if (req->close.file_slot) { ++ ret = io_close_fixed(req, issue_flags); ++ goto err; ++ } ++ ++ spin_lock(&files->file_lock); ++ fdt = files_fdtable(files); ++ if (close->fd >= fdt->max_fds) { ++ spin_unlock(&files->file_lock); ++ goto err; ++ } ++ file = fdt->fd[close->fd]; ++ if (!file || file->f_op == &io_uring_fops) { ++ spin_unlock(&files->file_lock); ++ file = NULL; ++ goto err; ++ } ++ ++ /* if the file has a flush method, be safe and punt to async */ ++ if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) { ++ spin_unlock(&files->file_lock); ++ return -EAGAIN; ++ } ++ ++ ret = __close_fd_get_file(close->fd, &file); ++ spin_unlock(&files->file_lock); ++ if (ret < 0) { ++ if (ret == -ENOENT) ++ ret = -EBADF; ++ goto err; ++ } ++ ++ /* No ->flush() or already async, safely close from here */ ++ ret = filp_close(file, current->files); ++err: ++ if (ret < 0) ++ req_set_fail(req); ++ if (file) ++ fput(file); ++ __io_req_complete(req, issue_flags, ret, 0); ++ return 0; ++} ++ ++static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index || ++ sqe->splice_fd_in)) ++ return -EINVAL; ++ ++ req->sync.off = READ_ONCE(sqe->off); ++ req->sync.len = READ_ONCE(sqe->len); ++ req->sync.flags = READ_ONCE(sqe->sync_range_flags); ++ return 0; ++} ++ ++static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ int ret; ++ ++ /* sync_file_range always requires a blocking context */ ++ if (issue_flags & IO_URING_F_NONBLOCK) ++ return -EAGAIN; ++ ++ ret = sync_file_range(req->file, req->sync.off, req->sync.len, ++ req->sync.flags); ++ if (ret < 0) ++ req_set_fail(req); ++ io_req_complete(req, ret); ++ return 0; ++} ++ ++#if defined(CONFIG_NET) ++static int io_setup_async_msg(struct io_kiocb *req, ++ struct io_async_msghdr *kmsg) ++{ ++ struct io_async_msghdr *async_msg = req->async_data; ++ ++ if (async_msg) ++ return -EAGAIN; ++ if (io_alloc_async_data(req)) { ++ kfree(kmsg->free_iov); ++ return -ENOMEM; ++ } ++ async_msg = req->async_data; ++ req->flags |= REQ_F_NEED_CLEANUP; ++ memcpy(async_msg, kmsg, sizeof(*kmsg)); ++ if (async_msg->msg.msg_name) ++ async_msg->msg.msg_name = &async_msg->addr; ++ /* if were using fast_iov, set it to the new one */ ++ if (!async_msg->free_iov) ++ async_msg->msg.msg_iter.iov = async_msg->fast_iov; ++ ++ return -EAGAIN; ++} ++ ++static int io_sendmsg_copy_hdr(struct io_kiocb *req, ++ struct io_async_msghdr *iomsg) ++{ ++ iomsg->msg.msg_name = &iomsg->addr; ++ iomsg->free_iov = iomsg->fast_iov; ++ return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg, ++ req->sr_msg.msg_flags, &iomsg->free_iov); ++} ++ ++static int io_sendmsg_prep_async(struct io_kiocb *req) ++{ ++ int ret; ++ ++ ret = io_sendmsg_copy_hdr(req, req->async_data); ++ if (!ret) ++ req->flags |= REQ_F_NEED_CLEANUP; ++ return ret; ++} ++ ++static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ struct io_sr_msg *sr = &req->sr_msg; ++ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (unlikely(sqe->addr2 || sqe->file_index)) ++ return -EINVAL; ++ if (unlikely(sqe->addr2 || sqe->file_index || sqe->ioprio)) ++ return -EINVAL; ++ ++ sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); ++ sr->len = READ_ONCE(sqe->len); ++ sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; ++ if (sr->msg_flags & MSG_DONTWAIT) ++ req->flags |= REQ_F_NOWAIT; ++ ++#ifdef CONFIG_COMPAT ++ if (req->ctx->compat) ++ sr->msg_flags |= MSG_CMSG_COMPAT; ++#endif ++ return 0; ++} ++ ++static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_async_msghdr iomsg, *kmsg; ++ struct socket *sock; ++ unsigned flags; ++ int min_ret = 0; ++ int ret; ++ ++ sock = sock_from_file(req->file, &ret); ++ if (unlikely(!sock)) ++ return ret; ++ ++ kmsg = req->async_data; ++ if (!kmsg) { ++ ret = io_sendmsg_copy_hdr(req, &iomsg); ++ if (ret) ++ return ret; ++ kmsg = &iomsg; ++ } ++ ++ flags = req->sr_msg.msg_flags; ++ if (issue_flags & IO_URING_F_NONBLOCK) ++ flags |= MSG_DONTWAIT; ++ if (flags & MSG_WAITALL) ++ min_ret = iov_iter_count(&kmsg->msg.msg_iter); ++ ++ ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); ++ if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN) ++ return io_setup_async_msg(req, kmsg); ++ if (ret == -ERESTARTSYS) ++ ret = -EINTR; ++ ++ /* fast path, check for non-NULL to avoid function call */ ++ if (kmsg->free_iov) ++ kfree(kmsg->free_iov); ++ req->flags &= ~REQ_F_NEED_CLEANUP; ++ if (ret < min_ret) ++ req_set_fail(req); ++ __io_req_complete(req, issue_flags, ret, 0); ++ return 0; ++} ++ ++static int io_send(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_sr_msg *sr = &req->sr_msg; ++ struct msghdr msg; ++ struct iovec iov; ++ struct socket *sock; ++ unsigned flags; ++ int min_ret = 0; ++ int ret; ++ ++ sock = sock_from_file(req->file, &ret); ++ if (unlikely(!sock)) ++ return ret; ++ ++ ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter); ++ if (unlikely(ret)) ++ return ret; ++ ++ msg.msg_name = NULL; ++ msg.msg_control = NULL; ++ msg.msg_controllen = 0; ++ msg.msg_namelen = 0; ++ ++ flags = req->sr_msg.msg_flags; ++ if (issue_flags & IO_URING_F_NONBLOCK) ++ flags |= MSG_DONTWAIT; ++ if (flags & MSG_WAITALL) ++ min_ret = iov_iter_count(&msg.msg_iter); ++ ++ msg.msg_flags = flags; ++ ret = sock_sendmsg(sock, &msg); ++ if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN) ++ return -EAGAIN; ++ if (ret == -ERESTARTSYS) ++ ret = -EINTR; ++ ++ if (ret < min_ret) ++ req_set_fail(req); ++ __io_req_complete(req, issue_flags, ret, 0); ++ return 0; ++} ++ ++static int __io_recvmsg_copy_hdr(struct io_kiocb *req, ++ struct io_async_msghdr *iomsg) ++{ ++ struct io_sr_msg *sr = &req->sr_msg; ++ struct iovec __user *uiov; ++ size_t iov_len; ++ int ret; ++ ++ ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg, ++ &iomsg->uaddr, &uiov, &iov_len); ++ if (ret) ++ return ret; ++ ++ if (req->flags & REQ_F_BUFFER_SELECT) { ++ if (iov_len > 1) ++ return -EINVAL; ++ if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov))) ++ return -EFAULT; ++ sr->len = iomsg->fast_iov[0].iov_len; ++ iomsg->free_iov = NULL; ++ } else { ++ iomsg->free_iov = iomsg->fast_iov; ++ ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV, ++ &iomsg->free_iov, &iomsg->msg.msg_iter, ++ false); ++ if (ret > 0) ++ ret = 0; ++ } ++ ++ return ret; ++} ++ ++#ifdef CONFIG_COMPAT ++static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, ++ struct io_async_msghdr *iomsg) ++{ ++ struct io_sr_msg *sr = &req->sr_msg; ++ struct compat_iovec __user *uiov; ++ compat_uptr_t ptr; ++ compat_size_t len; ++ int ret; ++ ++ ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr, ++ &ptr, &len); ++ if (ret) ++ return ret; ++ ++ uiov = compat_ptr(ptr); ++ if (req->flags & REQ_F_BUFFER_SELECT) { ++ compat_ssize_t clen; ++ ++ if (len > 1) ++ return -EINVAL; ++ if (!access_ok(uiov, sizeof(*uiov))) ++ return -EFAULT; ++ if (__get_user(clen, &uiov->iov_len)) ++ return -EFAULT; ++ if (clen < 0) ++ return -EINVAL; ++ sr->len = clen; ++ iomsg->free_iov = NULL; ++ } else { ++ iomsg->free_iov = iomsg->fast_iov; ++ ret = __import_iovec(READ, (struct iovec __user *)uiov, len, ++ UIO_FASTIOV, &iomsg->free_iov, ++ &iomsg->msg.msg_iter, true); ++ if (ret < 0) ++ return ret; ++ } ++ ++ return 0; ++} ++#endif ++ ++static int io_recvmsg_copy_hdr(struct io_kiocb *req, ++ struct io_async_msghdr *iomsg) ++{ ++ iomsg->msg.msg_name = &iomsg->addr; ++ ++#ifdef CONFIG_COMPAT ++ if (req->ctx->compat) ++ return __io_compat_recvmsg_copy_hdr(req, iomsg); ++#endif ++ ++ return __io_recvmsg_copy_hdr(req, iomsg); ++} ++ ++static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req, ++ bool needs_lock) ++{ ++ struct io_sr_msg *sr = &req->sr_msg; ++ struct io_buffer *kbuf; ++ ++ kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock); ++ if (IS_ERR(kbuf)) ++ return kbuf; ++ ++ sr->kbuf = kbuf; ++ req->flags |= REQ_F_BUFFER_SELECTED; ++ return kbuf; ++} ++ ++static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req) ++{ ++ return io_put_kbuf(req, req->sr_msg.kbuf); ++} ++ ++static int io_recvmsg_prep_async(struct io_kiocb *req) ++{ ++ int ret; ++ ++ ret = io_recvmsg_copy_hdr(req, req->async_data); ++ if (!ret) ++ req->flags |= REQ_F_NEED_CLEANUP; ++ return ret; ++} ++ ++static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ struct io_sr_msg *sr = &req->sr_msg; ++ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (unlikely(sqe->addr2 || sqe->file_index)) ++ return -EINVAL; ++ if (unlikely(sqe->addr2 || sqe->file_index || sqe->ioprio)) ++ return -EINVAL; ++ ++ sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); ++ sr->len = READ_ONCE(sqe->len); ++ sr->bgid = READ_ONCE(sqe->buf_group); ++ sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; ++ if (sr->msg_flags & MSG_DONTWAIT) ++ req->flags |= REQ_F_NOWAIT; ++ ++#ifdef CONFIG_COMPAT ++ if (req->ctx->compat) ++ sr->msg_flags |= MSG_CMSG_COMPAT; ++#endif ++ return 0; ++} ++ ++static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_async_msghdr iomsg, *kmsg; ++ struct socket *sock; ++ struct io_buffer *kbuf; ++ unsigned flags; ++ int min_ret = 0; ++ int ret, cflags = 0; ++ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ++ ++ sock = sock_from_file(req->file, &ret); ++ if (unlikely(!sock)) ++ return ret; ++ ++ kmsg = req->async_data; ++ if (!kmsg) { ++ ret = io_recvmsg_copy_hdr(req, &iomsg); ++ if (ret) ++ return ret; ++ kmsg = &iomsg; ++ } ++ ++ if (req->flags & REQ_F_BUFFER_SELECT) { ++ kbuf = io_recv_buffer_select(req, !force_nonblock); ++ if (IS_ERR(kbuf)) ++ return PTR_ERR(kbuf); ++ kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr); ++ kmsg->fast_iov[0].iov_len = req->sr_msg.len; ++ iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, ++ 1, req->sr_msg.len); ++ } ++ ++ flags = req->sr_msg.msg_flags; ++ if (force_nonblock) ++ flags |= MSG_DONTWAIT; ++ if (flags & MSG_WAITALL) ++ min_ret = iov_iter_count(&kmsg->msg.msg_iter); ++ ++ ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg, ++ kmsg->uaddr, flags); ++ if (force_nonblock && ret == -EAGAIN) ++ return io_setup_async_msg(req, kmsg); ++ if (ret == -ERESTARTSYS) ++ ret = -EINTR; ++ ++ if (req->flags & REQ_F_BUFFER_SELECTED) ++ cflags = io_put_recv_kbuf(req); ++ /* fast path, check for non-NULL to avoid function call */ ++ if (kmsg->free_iov) ++ kfree(kmsg->free_iov); ++ req->flags &= ~REQ_F_NEED_CLEANUP; ++ if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)))) ++ req_set_fail(req); ++ __io_req_complete(req, issue_flags, ret, cflags); ++ return 0; ++} ++ ++static int io_recv(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_buffer *kbuf; ++ struct io_sr_msg *sr = &req->sr_msg; ++ struct msghdr msg; ++ void __user *buf = sr->buf; ++ struct socket *sock; ++ struct iovec iov; ++ unsigned flags; ++ int min_ret = 0; ++ int ret, cflags = 0; ++ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ++ ++ sock = sock_from_file(req->file, &ret); ++ if (unlikely(!sock)) ++ return ret; ++ ++ if (req->flags & REQ_F_BUFFER_SELECT) { ++ kbuf = io_recv_buffer_select(req, !force_nonblock); ++ if (IS_ERR(kbuf)) ++ return PTR_ERR(kbuf); ++ buf = u64_to_user_ptr(kbuf->addr); ++ } ++ ++ ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter); ++ if (unlikely(ret)) ++ goto out_free; ++ ++ msg.msg_name = NULL; ++ msg.msg_control = NULL; ++ msg.msg_controllen = 0; ++ msg.msg_namelen = 0; ++ msg.msg_iocb = NULL; ++ msg.msg_flags = 0; ++ ++ flags = req->sr_msg.msg_flags; ++ if (force_nonblock) ++ flags |= MSG_DONTWAIT; ++ if (flags & MSG_WAITALL) ++ min_ret = iov_iter_count(&msg.msg_iter); ++ ++ ret = sock_recvmsg(sock, &msg, flags); ++ if (force_nonblock && ret == -EAGAIN) ++ return -EAGAIN; ++ if (ret == -ERESTARTSYS) ++ ret = -EINTR; ++out_free: ++ if (req->flags & REQ_F_BUFFER_SELECTED) ++ cflags = io_put_recv_kbuf(req); ++ if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)))) ++ req_set_fail(req); ++ __io_req_complete(req, issue_flags, ret, cflags); ++ return 0; ++} ++ ++static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ struct io_accept *accept = &req->accept; ++ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (sqe->ioprio || sqe->len || sqe->buf_index) ++ return -EINVAL; ++ ++ accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); ++ accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); ++ accept->flags = READ_ONCE(sqe->accept_flags); ++ accept->nofile = rlimit(RLIMIT_NOFILE); ++ ++ accept->file_slot = READ_ONCE(sqe->file_index); ++ if (accept->file_slot && (accept->flags & SOCK_CLOEXEC)) ++ return -EINVAL; ++ if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) ++ return -EINVAL; ++ if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK)) ++ accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK; ++ return 0; ++} ++ ++static int io_accept(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_accept *accept = &req->accept; ++ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ++ unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; ++ bool fixed = !!accept->file_slot; ++ struct file *file; ++ int ret, fd; ++ ++ if (req->file->f_flags & O_NONBLOCK) ++ req->flags |= REQ_F_NOWAIT; ++ ++ if (!fixed) { ++ fd = __get_unused_fd_flags(accept->flags, accept->nofile); ++ if (unlikely(fd < 0)) ++ return fd; ++ } ++ file = do_accept(req->file, file_flags, accept->addr, accept->addr_len, ++ accept->flags); ++ ++ if (IS_ERR(file)) { ++ if (!fixed) ++ put_unused_fd(fd); ++ ret = PTR_ERR(file); ++ if (ret == -EAGAIN && force_nonblock) ++ return -EAGAIN; ++ if (ret == -ERESTARTSYS) ++ ret = -EINTR; ++ req_set_fail(req); ++ } else if (!fixed) { ++ fd_install(fd, file); ++ ret = fd; ++ } else { ++ ret = io_install_fixed_file(req, file, issue_flags, ++ accept->file_slot - 1); ++ } ++ __io_req_complete(req, issue_flags, ret, 0); ++ return 0; ++} ++ ++static int io_connect_prep_async(struct io_kiocb *req) ++{ ++ struct io_async_connect *io = req->async_data; ++ struct io_connect *conn = &req->connect; ++ ++ return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address); ++} ++ ++static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ struct io_connect *conn = &req->connect; ++ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags || ++ sqe->splice_fd_in) ++ return -EINVAL; ++ ++ conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); ++ conn->addr_len = READ_ONCE(sqe->addr2); ++ return 0; ++} ++ ++static int io_connect(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_async_connect __io, *io; ++ unsigned file_flags; ++ int ret; ++ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ++ ++ if (req->async_data) { ++ io = req->async_data; ++ } else { ++ ret = move_addr_to_kernel(req->connect.addr, ++ req->connect.addr_len, ++ &__io.address); ++ if (ret) ++ goto out; ++ io = &__io; ++ } ++ ++ file_flags = force_nonblock ? O_NONBLOCK : 0; ++ ++ ret = __sys_connect_file(req->file, &io->address, ++ req->connect.addr_len, file_flags); ++ if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) { ++ if (req->async_data) ++ return -EAGAIN; ++ if (io_alloc_async_data(req)) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ memcpy(req->async_data, &__io, sizeof(__io)); ++ return -EAGAIN; ++ } ++ if (ret == -ERESTARTSYS) ++ ret = -EINTR; ++out: ++ if (ret < 0) ++ req_set_fail(req); ++ __io_req_complete(req, issue_flags, ret, 0); ++ return 0; ++} ++#else /* !CONFIG_NET */ ++#define IO_NETOP_FN(op) \ ++static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \ ++{ \ ++ return -EOPNOTSUPP; \ ++} ++ ++#define IO_NETOP_PREP(op) \ ++IO_NETOP_FN(op) \ ++static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \ ++{ \ ++ return -EOPNOTSUPP; \ ++} \ ++ ++#define IO_NETOP_PREP_ASYNC(op) \ ++IO_NETOP_PREP(op) \ ++static int io_##op##_prep_async(struct io_kiocb *req) \ ++{ \ ++ return -EOPNOTSUPP; \ ++} ++ ++IO_NETOP_PREP_ASYNC(sendmsg); ++IO_NETOP_PREP_ASYNC(recvmsg); ++IO_NETOP_PREP_ASYNC(connect); ++IO_NETOP_PREP(accept); ++IO_NETOP_FN(send); ++IO_NETOP_FN(recv); ++#endif /* CONFIG_NET */ ++ ++struct io_poll_table { ++ struct poll_table_struct pt; ++ struct io_kiocb *req; ++ int nr_entries; ++ int error; ++}; ++ ++#define IO_POLL_CANCEL_FLAG BIT(31) ++#define IO_POLL_RETRY_FLAG BIT(30) ++#define IO_POLL_REF_MASK GENMASK(29, 0) ++ ++/* ++ * We usually have 1-2 refs taken, 128 is more than enough and we want to ++ * maximise the margin between this amount and the moment when it overflows. ++ */ ++#define IO_POLL_REF_BIAS 128 ++ ++static bool io_poll_get_ownership_slowpath(struct io_kiocb *req) ++{ ++ int v; ++ ++ /* ++ * poll_refs are already elevated and we don't have much hope for ++ * grabbing the ownership. Instead of incrementing set a retry flag ++ * to notify the loop that there might have been some change. ++ */ ++ v = atomic_fetch_or(IO_POLL_RETRY_FLAG, &req->poll_refs); ++ if (v & IO_POLL_REF_MASK) ++ return false; ++ return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); ++} ++ ++/* ++ * If refs part of ->poll_refs (see IO_POLL_REF_MASK) is 0, it's free. We can ++ * bump it and acquire ownership. It's disallowed to modify requests while not ++ * owning it, that prevents from races for enqueueing task_work's and b/w ++ * arming poll and wakeups. ++ */ ++static inline bool io_poll_get_ownership(struct io_kiocb *req) ++{ ++ if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS)) ++ return io_poll_get_ownership_slowpath(req); ++ return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); ++} ++ ++static void io_poll_mark_cancelled(struct io_kiocb *req) ++{ ++ atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs); ++} ++ ++static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req) ++{ ++ /* pure poll stashes this in ->async_data, poll driven retry elsewhere */ ++ if (req->opcode == IORING_OP_POLL_ADD) ++ return req->async_data; ++ return req->apoll->double_poll; ++} ++ ++static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req) ++{ ++ if (req->opcode == IORING_OP_POLL_ADD) ++ return &req->poll; ++ return &req->apoll->poll; ++} ++ ++static void io_poll_req_insert(struct io_kiocb *req) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ struct hlist_head *list; ++ ++ list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)]; ++ hlist_add_head(&req->hash_node, list); ++} ++ ++static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events, ++ wait_queue_func_t wake_func) ++{ ++ poll->head = NULL; ++#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP) ++ /* mask in events that we always want/need */ ++ poll->events = events | IO_POLL_UNMASK; ++ INIT_LIST_HEAD(&poll->wait.entry); ++ init_waitqueue_func_entry(&poll->wait, wake_func); ++} ++ ++static inline void io_poll_remove_entry(struct io_poll_iocb *poll) ++{ ++ struct wait_queue_head *head = smp_load_acquire(&poll->head); ++ ++ if (head) { ++ spin_lock_irq(&head->lock); ++ list_del_init(&poll->wait.entry); ++ poll->head = NULL; ++ spin_unlock_irq(&head->lock); ++ } ++} ++ ++static void io_poll_remove_entries(struct io_kiocb *req) ++{ ++ struct io_poll_iocb *poll = io_poll_get_single(req); ++ struct io_poll_iocb *poll_double = io_poll_get_double(req); ++ ++ /* ++ * While we hold the waitqueue lock and the waitqueue is nonempty, ++ * wake_up_pollfree() will wait for us. However, taking the waitqueue ++ * lock in the first place can race with the waitqueue being freed. ++ * ++ * We solve this as eventpoll does: by taking advantage of the fact that ++ * all users of wake_up_pollfree() will RCU-delay the actual free. If ++ * we enter rcu_read_lock() and see that the pointer to the queue is ++ * non-NULL, we can then lock it without the memory being freed out from ++ * under us. ++ * ++ * Keep holding rcu_read_lock() as long as we hold the queue lock, in ++ * case the caller deletes the entry from the queue, leaving it empty. ++ * In that case, only RCU prevents the queue memory from being freed. ++ */ ++ rcu_read_lock(); ++ io_poll_remove_entry(poll); ++ if (poll_double) ++ io_poll_remove_entry(poll_double); ++ rcu_read_unlock(); ++} ++ ++/* ++ * All poll tw should go through this. Checks for poll events, manages ++ * references, does rewait, etc. ++ * ++ * Returns a negative error on failure. >0 when no action require, which is ++ * either spurious wakeup or multishot CQE is served. 0 when it's done with ++ * the request, then the mask is stored in req->result. ++ */ ++static int io_poll_check_events(struct io_kiocb *req) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ struct io_poll_iocb *poll = io_poll_get_single(req); ++ int v; ++ ++ /* req->task == current here, checking PF_EXITING is safe */ ++ if (unlikely(req->task->flags & PF_EXITING)) ++ io_poll_mark_cancelled(req); ++ ++ do { ++ v = atomic_read(&req->poll_refs); ++ ++ /* tw handler should be the owner, and so have some references */ ++ if (WARN_ON_ONCE(!(v & IO_POLL_REF_MASK))) ++ return 0; ++ if (v & IO_POLL_CANCEL_FLAG) ++ return -ECANCELED; ++ /* ++ * cqe.res contains only events of the first wake up ++ * and all others are be lost. Redo vfs_poll() to get ++ * up to date state. ++ */ ++ if ((v & IO_POLL_REF_MASK) != 1) ++ req->result = 0; ++ if (v & IO_POLL_RETRY_FLAG) { ++ req->result = 0; ++ /* ++ * We won't find new events that came in between ++ * vfs_poll and the ref put unless we clear the ++ * flag in advance. ++ */ ++ atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs); ++ v &= ~IO_POLL_RETRY_FLAG; ++ } ++ ++ if (!req->result) { ++ struct poll_table_struct pt = { ._key = poll->events }; ++ ++ req->result = vfs_poll(req->file, &pt) & poll->events; ++ } ++ ++ /* multishot, just fill an CQE and proceed */ ++ if (req->result && !(poll->events & EPOLLONESHOT)) { ++ __poll_t mask = mangle_poll(req->result & poll->events); ++ bool filled; ++ ++ spin_lock(&ctx->completion_lock); ++ filled = io_fill_cqe_aux(ctx, req->user_data, mask, ++ IORING_CQE_F_MORE); ++ io_commit_cqring(ctx); ++ spin_unlock(&ctx->completion_lock); ++ if (unlikely(!filled)) ++ return -ECANCELED; ++ io_cqring_ev_posted(ctx); ++ } else if (req->result) { ++ return 0; ++ } ++ ++ /* force the next iteration to vfs_poll() */ ++ req->result = 0; ++ ++ /* ++ * Release all references, retry if someone tried to restart ++ * task_work while we were executing it. ++ */ ++ } while (atomic_sub_return(v & IO_POLL_REF_MASK, &req->poll_refs) & ++ IO_POLL_REF_MASK); ++ ++ return 1; ++} ++ ++static void io_poll_task_func(struct io_kiocb *req, bool *locked) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ int ret; ++ ++ ret = io_poll_check_events(req); ++ if (ret > 0) ++ return; ++ ++ if (!ret) { ++ req->result = mangle_poll(req->result & req->poll.events); ++ } else { ++ req->result = ret; ++ req_set_fail(req); ++ } ++ ++ io_poll_remove_entries(req); ++ spin_lock(&ctx->completion_lock); ++ hash_del(&req->hash_node); ++ spin_unlock(&ctx->completion_lock); ++ io_req_complete_post(req, req->result, 0); ++} ++ ++static void io_apoll_task_func(struct io_kiocb *req, bool *locked) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ int ret; ++ ++ ret = io_poll_check_events(req); ++ if (ret > 0) ++ return; ++ ++ io_poll_remove_entries(req); ++ spin_lock(&ctx->completion_lock); ++ hash_del(&req->hash_node); ++ spin_unlock(&ctx->completion_lock); ++ ++ if (!ret) ++ io_req_task_submit(req, locked); ++ else ++ io_req_complete_failed(req, ret); ++} ++ ++static void __io_poll_execute(struct io_kiocb *req, int mask) ++{ ++ req->result = mask; ++ if (req->opcode == IORING_OP_POLL_ADD) ++ req->io_task_work.func = io_poll_task_func; ++ else ++ req->io_task_work.func = io_apoll_task_func; ++ ++ trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask); ++ io_req_task_work_add(req); ++} ++ ++static inline void io_poll_execute(struct io_kiocb *req, int res) ++{ ++ if (io_poll_get_ownership(req)) ++ __io_poll_execute(req, res); ++} ++ ++static void io_poll_cancel_req(struct io_kiocb *req) ++{ ++ io_poll_mark_cancelled(req); ++ /* kick tw, which should complete the request */ ++ io_poll_execute(req, 0); ++} ++ ++static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, ++ void *key) ++{ ++ struct io_kiocb *req = wait->private; ++ struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb, ++ wait); ++ __poll_t mask = key_to_poll(key); ++ ++ if (unlikely(mask & POLLFREE)) { ++ io_poll_mark_cancelled(req); ++ /* we have to kick tw in case it's not already */ ++ io_poll_execute(req, 0); ++ ++ /* ++ * If the waitqueue is being freed early but someone is already ++ * holds ownership over it, we have to tear down the request as ++ * best we can. That means immediately removing the request from ++ * its waitqueue and preventing all further accesses to the ++ * waitqueue via the request. ++ */ ++ list_del_init(&poll->wait.entry); ++ ++ /* ++ * Careful: this *must* be the last step, since as soon ++ * as req->head is NULL'ed out, the request can be ++ * completed and freed, since aio_poll_complete_work() ++ * will no longer need to take the waitqueue lock. ++ */ ++ smp_store_release(&poll->head, NULL); ++ return 1; ++ } ++ ++ /* for instances that support it check for an event match first */ ++ if (mask && !(mask & poll->events)) ++ return 0; ++ ++ if (io_poll_get_ownership(req)) ++ __io_poll_execute(req, mask); ++ return 1; ++} ++ ++static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, ++ struct wait_queue_head *head, ++ struct io_poll_iocb **poll_ptr) ++{ ++ struct io_kiocb *req = pt->req; ++ ++ /* ++ * The file being polled uses multiple waitqueues for poll handling ++ * (e.g. one for read, one for write). Setup a separate io_poll_iocb ++ * if this happens. ++ */ ++ if (unlikely(pt->nr_entries)) { ++ struct io_poll_iocb *first = poll; ++ ++ /* double add on the same waitqueue head, ignore */ ++ if (first->head == head) ++ return; ++ /* already have a 2nd entry, fail a third attempt */ ++ if (*poll_ptr) { ++ if ((*poll_ptr)->head == head) ++ return; ++ pt->error = -EINVAL; ++ return; ++ } ++ ++ poll = kmalloc(sizeof(*poll), GFP_ATOMIC); ++ if (!poll) { ++ pt->error = -ENOMEM; ++ return; ++ } ++ io_init_poll_iocb(poll, first->events, first->wait.func); ++ *poll_ptr = poll; ++ } ++ ++ pt->nr_entries++; ++ poll->head = head; ++ poll->wait.private = req; ++ ++ if (poll->events & EPOLLEXCLUSIVE) ++ add_wait_queue_exclusive(head, &poll->wait); ++ else ++ add_wait_queue(head, &poll->wait); ++} ++ ++static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head, ++ struct poll_table_struct *p) ++{ ++ struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); ++ ++ __io_queue_proc(&pt->req->poll, pt, head, ++ (struct io_poll_iocb **) &pt->req->async_data); ++} ++ ++static int __io_arm_poll_handler(struct io_kiocb *req, ++ struct io_poll_iocb *poll, ++ struct io_poll_table *ipt, __poll_t mask) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ INIT_HLIST_NODE(&req->hash_node); ++ io_init_poll_iocb(poll, mask, io_poll_wake); ++ poll->file = req->file; ++ poll->wait.private = req; ++ ++ ipt->pt._key = mask; ++ ipt->req = req; ++ ipt->error = 0; ++ ipt->nr_entries = 0; ++ ++ /* ++ * Take the ownership to delay any tw execution up until we're done ++ * with poll arming. see io_poll_get_ownership(). ++ */ ++ atomic_set(&req->poll_refs, 1); ++ mask = vfs_poll(req->file, &ipt->pt) & poll->events; ++ ++ if (mask && (poll->events & EPOLLONESHOT)) { ++ io_poll_remove_entries(req); ++ /* no one else has access to the req, forget about the ref */ ++ return mask; ++ } ++ if (!mask && unlikely(ipt->error || !ipt->nr_entries)) { ++ io_poll_remove_entries(req); ++ if (!ipt->error) ++ ipt->error = -EINVAL; ++ return 0; ++ } ++ ++ spin_lock(&ctx->completion_lock); ++ io_poll_req_insert(req); ++ spin_unlock(&ctx->completion_lock); ++ ++ if (mask) { ++ /* can't multishot if failed, just queue the event we've got */ ++ if (unlikely(ipt->error || !ipt->nr_entries)) { ++ poll->events |= EPOLLONESHOT; ++ ipt->error = 0; ++ } ++ __io_poll_execute(req, mask); ++ return 0; ++ } ++ ++ /* ++ * Try to release ownership. If we see a change of state, e.g. ++ * poll was waken up, queue up a tw, it'll deal with it. ++ */ ++ if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1) ++ __io_poll_execute(req, 0); ++ return 0; ++} ++ ++static void io_async_queue_proc(struct file *file, struct wait_queue_head *head, ++ struct poll_table_struct *p) ++{ ++ struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); ++ struct async_poll *apoll = pt->req->apoll; ++ ++ __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll); ++} ++ ++enum { ++ IO_APOLL_OK, ++ IO_APOLL_ABORTED, ++ IO_APOLL_READY ++}; ++ ++static int io_arm_poll_handler(struct io_kiocb *req) ++{ ++ const struct io_op_def *def = &io_op_defs[req->opcode]; ++ struct io_ring_ctx *ctx = req->ctx; ++ struct async_poll *apoll; ++ struct io_poll_table ipt; ++ __poll_t mask = EPOLLONESHOT | POLLERR | POLLPRI; ++ int ret; ++ ++ if (!req->file || !file_can_poll(req->file)) ++ return IO_APOLL_ABORTED; ++ if (req->flags & REQ_F_POLLED) ++ return IO_APOLL_ABORTED; ++ if (!def->pollin && !def->pollout) ++ return IO_APOLL_ABORTED; ++ ++ if (def->pollin) { ++ mask |= POLLIN | POLLRDNORM; ++ ++ /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */ ++ if ((req->opcode == IORING_OP_RECVMSG) && ++ (req->sr_msg.msg_flags & MSG_ERRQUEUE)) ++ mask &= ~POLLIN; ++ } else { ++ mask |= POLLOUT | POLLWRNORM; ++ } ++ ++ apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC); ++ if (unlikely(!apoll)) ++ return IO_APOLL_ABORTED; ++ apoll->double_poll = NULL; ++ req->apoll = apoll; ++ req->flags |= REQ_F_POLLED; ++ ipt.pt._qproc = io_async_queue_proc; ++ ++ ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask); ++ if (ret || ipt.error) ++ return ret ? IO_APOLL_READY : IO_APOLL_ABORTED; ++ ++ trace_io_uring_poll_arm(ctx, req, req->opcode, req->user_data, ++ mask, apoll->poll.events); ++ return IO_APOLL_OK; ++} ++ ++/* ++ * Returns true if we found and killed one or more poll requests ++ */ ++static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, ++ bool cancel_all) ++{ ++ struct hlist_node *tmp; ++ struct io_kiocb *req; ++ bool found = false; ++ int i; ++ ++ spin_lock(&ctx->completion_lock); ++ for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { ++ struct hlist_head *list; ++ ++ list = &ctx->cancel_hash[i]; ++ hlist_for_each_entry_safe(req, tmp, list, hash_node) { ++ if (io_match_task_safe(req, tsk, cancel_all)) { ++ hlist_del_init(&req->hash_node); ++ io_poll_cancel_req(req); ++ found = true; ++ } ++ } ++ } ++ spin_unlock(&ctx->completion_lock); ++ return found; ++} ++ ++static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr, ++ bool poll_only) ++ __must_hold(&ctx->completion_lock) ++{ ++ struct hlist_head *list; ++ struct io_kiocb *req; ++ ++ list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)]; ++ hlist_for_each_entry(req, list, hash_node) { ++ if (sqe_addr != req->user_data) ++ continue; ++ if (poll_only && req->opcode != IORING_OP_POLL_ADD) ++ continue; ++ return req; ++ } ++ return NULL; ++} ++ ++static bool io_poll_disarm(struct io_kiocb *req) ++ __must_hold(&ctx->completion_lock) ++{ ++ if (!io_poll_get_ownership(req)) ++ return false; ++ io_poll_remove_entries(req); ++ hash_del(&req->hash_node); ++ return true; ++} ++ ++static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr, ++ bool poll_only) ++ __must_hold(&ctx->completion_lock) ++{ ++ struct io_kiocb *req = io_poll_find(ctx, sqe_addr, poll_only); ++ ++ if (!req) ++ return -ENOENT; ++ io_poll_cancel_req(req); ++ return 0; ++} ++ ++static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe, ++ unsigned int flags) ++{ ++ u32 events; ++ ++ events = READ_ONCE(sqe->poll32_events); ++#ifdef __BIG_ENDIAN ++ events = swahw32(events); ++#endif ++ if (!(flags & IORING_POLL_ADD_MULTI)) ++ events |= EPOLLONESHOT; ++ return demangle_poll(events) | (events & (EPOLLEXCLUSIVE|EPOLLONESHOT)); ++} ++ ++static int io_poll_update_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++ struct io_poll_update *upd = &req->poll_update; ++ u32 flags; ++ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in) ++ return -EINVAL; ++ flags = READ_ONCE(sqe->len); ++ if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA | ++ IORING_POLL_ADD_MULTI)) ++ return -EINVAL; ++ /* meaningless without update */ ++ if (flags == IORING_POLL_ADD_MULTI) ++ return -EINVAL; ++ ++ upd->old_user_data = READ_ONCE(sqe->addr); ++ upd->update_events = flags & IORING_POLL_UPDATE_EVENTS; ++ upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA; ++ ++ upd->new_user_data = READ_ONCE(sqe->off); ++ if (!upd->update_user_data && upd->new_user_data) ++ return -EINVAL; ++ if (upd->update_events) ++ upd->events = io_poll_parse_events(sqe, flags); ++ else if (sqe->poll32_events) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ struct io_poll_iocb *poll = &req->poll; ++ u32 flags; ++ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->addr) ++ return -EINVAL; ++ flags = READ_ONCE(sqe->len); ++ if (flags & ~IORING_POLL_ADD_MULTI) ++ return -EINVAL; ++ ++ io_req_set_refcount(req); ++ poll->events = io_poll_parse_events(sqe, flags); ++ return 0; ++} ++ ++static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_poll_iocb *poll = &req->poll; ++ struct io_poll_table ipt; ++ int ret; ++ ++ ipt.pt._qproc = io_poll_queue_proc; ++ ++ ret = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events); ++ if (!ret && ipt.error) ++ req_set_fail(req); ++ ret = ret ?: ipt.error; ++ if (ret) ++ __io_req_complete(req, issue_flags, ret, 0); ++ return 0; ++} ++ ++static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ struct io_kiocb *preq; ++ int ret2, ret = 0; ++ ++ spin_lock(&ctx->completion_lock); ++ preq = io_poll_find(ctx, req->poll_update.old_user_data, true); ++ if (!preq || !io_poll_disarm(preq)) { ++ spin_unlock(&ctx->completion_lock); ++ ret = preq ? -EALREADY : -ENOENT; ++ goto out; ++ } ++ spin_unlock(&ctx->completion_lock); ++ ++ if (req->poll_update.update_events || req->poll_update.update_user_data) { ++ /* only mask one event flags, keep behavior flags */ ++ if (req->poll_update.update_events) { ++ preq->poll.events &= ~0xffff; ++ preq->poll.events |= req->poll_update.events & 0xffff; ++ preq->poll.events |= IO_POLL_UNMASK; ++ } ++ if (req->poll_update.update_user_data) ++ preq->user_data = req->poll_update.new_user_data; ++ ++ ret2 = io_poll_add(preq, issue_flags); ++ /* successfully updated, don't complete poll request */ ++ if (!ret2) ++ goto out; ++ } ++ req_set_fail(preq); ++ io_req_complete(preq, -ECANCELED); ++out: ++ if (ret < 0) ++ req_set_fail(req); ++ /* complete update request, we're done with it */ ++ io_req_complete(req, ret); ++ return 0; ++} ++ ++static void io_req_task_timeout(struct io_kiocb *req, bool *locked) ++{ ++ req_set_fail(req); ++ io_req_complete_post(req, -ETIME, 0); ++} ++ ++static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) ++{ ++ struct io_timeout_data *data = container_of(timer, ++ struct io_timeout_data, timer); ++ struct io_kiocb *req = data->req; ++ struct io_ring_ctx *ctx = req->ctx; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&ctx->timeout_lock, flags); ++ list_del_init(&req->timeout.list); ++ atomic_set(&req->ctx->cq_timeouts, ++ atomic_read(&req->ctx->cq_timeouts) + 1); ++ spin_unlock_irqrestore(&ctx->timeout_lock, flags); ++ ++ req->io_task_work.func = io_req_task_timeout; ++ io_req_task_work_add(req); ++ return HRTIMER_NORESTART; ++} ++ ++static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, ++ __u64 user_data) ++ __must_hold(&ctx->timeout_lock) ++{ ++ struct io_timeout_data *io; ++ struct io_kiocb *req; ++ bool found = false; ++ ++ list_for_each_entry(req, &ctx->timeout_list, timeout.list) { ++ found = user_data == req->user_data; ++ if (found) ++ break; ++ } ++ if (!found) ++ return ERR_PTR(-ENOENT); ++ ++ io = req->async_data; ++ if (hrtimer_try_to_cancel(&io->timer) == -1) ++ return ERR_PTR(-EALREADY); ++ list_del_init(&req->timeout.list); ++ return req; ++} ++ ++static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data) ++ __must_hold(&ctx->completion_lock) ++ __must_hold(&ctx->timeout_lock) ++{ ++ struct io_kiocb *req = io_timeout_extract(ctx, user_data); ++ ++ if (IS_ERR(req)) ++ return PTR_ERR(req); ++ ++ req_set_fail(req); ++ io_fill_cqe_req(req, -ECANCELED, 0); ++ io_put_req_deferred(req); ++ return 0; ++} ++ ++static clockid_t io_timeout_get_clock(struct io_timeout_data *data) ++{ ++ switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) { ++ case IORING_TIMEOUT_BOOTTIME: ++ return CLOCK_BOOTTIME; ++ case IORING_TIMEOUT_REALTIME: ++ return CLOCK_REALTIME; ++ default: ++ /* can't happen, vetted at prep time */ ++ WARN_ON_ONCE(1); ++ fallthrough; ++ case 0: ++ return CLOCK_MONOTONIC; ++ } ++} ++ ++static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, ++ struct timespec64 *ts, enum hrtimer_mode mode) ++ __must_hold(&ctx->timeout_lock) ++{ ++ struct io_timeout_data *io; ++ struct io_kiocb *req; ++ bool found = false; ++ ++ list_for_each_entry(req, &ctx->ltimeout_list, timeout.list) { ++ found = user_data == req->user_data; ++ if (found) ++ break; ++ } ++ if (!found) ++ return -ENOENT; ++ ++ io = req->async_data; ++ if (hrtimer_try_to_cancel(&io->timer) == -1) ++ return -EALREADY; ++ hrtimer_init(&io->timer, io_timeout_get_clock(io), mode); ++ io->timer.function = io_link_timeout_fn; ++ hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode); ++ return 0; ++} ++ ++static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, ++ struct timespec64 *ts, enum hrtimer_mode mode) ++ __must_hold(&ctx->timeout_lock) ++{ ++ struct io_kiocb *req = io_timeout_extract(ctx, user_data); ++ struct io_timeout_data *data; ++ ++ if (IS_ERR(req)) ++ return PTR_ERR(req); ++ ++ req->timeout.off = 0; /* noseq */ ++ data = req->async_data; ++ list_add_tail(&req->timeout.list, &ctx->timeout_list); ++ hrtimer_init(&data->timer, io_timeout_get_clock(data), mode); ++ data->timer.function = io_timeout_fn; ++ hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode); ++ return 0; ++} ++ ++static int io_timeout_remove_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++ struct io_timeout_rem *tr = &req->timeout_rem; ++ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) ++ return -EINVAL; ++ if (sqe->ioprio || sqe->buf_index || sqe->len || sqe->splice_fd_in) ++ return -EINVAL; ++ ++ tr->ltimeout = false; ++ tr->addr = READ_ONCE(sqe->addr); ++ tr->flags = READ_ONCE(sqe->timeout_flags); ++ if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) { ++ if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1) ++ return -EINVAL; ++ if (tr->flags & IORING_LINK_TIMEOUT_UPDATE) ++ tr->ltimeout = true; ++ if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS)) ++ return -EINVAL; ++ if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2))) ++ return -EFAULT; ++ } else if (tr->flags) { ++ /* timeout removal doesn't support flags */ ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) ++{ ++ return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS ++ : HRTIMER_MODE_REL; ++} ++ ++/* ++ * Remove or update an existing timeout command ++ */ ++static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_timeout_rem *tr = &req->timeout_rem; ++ struct io_ring_ctx *ctx = req->ctx; ++ int ret; ++ ++ if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE)) { ++ spin_lock(&ctx->completion_lock); ++ spin_lock_irq(&ctx->timeout_lock); ++ ret = io_timeout_cancel(ctx, tr->addr); ++ spin_unlock_irq(&ctx->timeout_lock); ++ spin_unlock(&ctx->completion_lock); ++ } else { ++ enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags); ++ ++ spin_lock_irq(&ctx->timeout_lock); ++ if (tr->ltimeout) ++ ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode); ++ else ++ ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode); ++ spin_unlock_irq(&ctx->timeout_lock); ++ } ++ ++ if (ret < 0) ++ req_set_fail(req); ++ io_req_complete_post(req, ret, 0); ++ return 0; ++} ++ ++static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, ++ bool is_timeout_link) ++{ ++ struct io_timeout_data *data; ++ unsigned flags; ++ u32 off = READ_ONCE(sqe->off); ++ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (sqe->ioprio || sqe->buf_index || sqe->len != 1 || ++ sqe->splice_fd_in) ++ return -EINVAL; ++ if (off && is_timeout_link) ++ return -EINVAL; ++ flags = READ_ONCE(sqe->timeout_flags); ++ if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK)) ++ return -EINVAL; ++ /* more than one clock specified is invalid, obviously */ ++ if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1) ++ return -EINVAL; ++ ++ INIT_LIST_HEAD(&req->timeout.list); ++ req->timeout.off = off; ++ if (unlikely(off && !req->ctx->off_timeout_used)) ++ req->ctx->off_timeout_used = true; ++ ++ if (!req->async_data && io_alloc_async_data(req)) ++ return -ENOMEM; ++ ++ data = req->async_data; ++ data->req = req; ++ data->flags = flags; ++ ++ if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr))) ++ return -EFAULT; ++ ++ INIT_LIST_HEAD(&req->timeout.list); ++ data->mode = io_translate_timeout_mode(flags); ++ hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode); ++ ++ if (is_timeout_link) { ++ struct io_submit_link *link = &req->ctx->submit_state.link; ++ ++ if (!link->head) ++ return -EINVAL; ++ if (link->last->opcode == IORING_OP_LINK_TIMEOUT) ++ return -EINVAL; ++ req->timeout.head = link->last; ++ link->last->flags |= REQ_F_ARM_LTIMEOUT; ++ } ++ return 0; ++} ++ ++static int io_timeout(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ struct io_timeout_data *data = req->async_data; ++ struct list_head *entry; ++ u32 tail, off = req->timeout.off; ++ ++ spin_lock_irq(&ctx->timeout_lock); ++ ++ /* ++ * sqe->off holds how many events that need to occur for this ++ * timeout event to be satisfied. If it isn't set, then this is ++ * a pure timeout request, sequence isn't used. ++ */ ++ if (io_is_timeout_noseq(req)) { ++ entry = ctx->timeout_list.prev; ++ goto add; ++ } ++ ++ tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); ++ req->timeout.target_seq = tail + off; ++ ++ /* Update the last seq here in case io_flush_timeouts() hasn't. ++ * This is safe because ->completion_lock is held, and submissions ++ * and completions are never mixed in the same ->completion_lock section. ++ */ ++ ctx->cq_last_tm_flush = tail; ++ ++ /* ++ * Insertion sort, ensuring the first entry in the list is always ++ * the one we need first. ++ */ ++ list_for_each_prev(entry, &ctx->timeout_list) { ++ struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, ++ timeout.list); ++ ++ if (io_is_timeout_noseq(nxt)) ++ continue; ++ /* nxt.seq is behind @tail, otherwise would've been completed */ ++ if (off >= nxt->timeout.target_seq - tail) ++ break; ++ } ++add: ++ list_add(&req->timeout.list, entry); ++ data->timer.function = io_timeout_fn; ++ hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); ++ spin_unlock_irq(&ctx->timeout_lock); ++ return 0; ++} ++ ++struct io_cancel_data { ++ struct io_ring_ctx *ctx; ++ u64 user_data; ++}; ++ ++static bool io_cancel_cb(struct io_wq_work *work, void *data) ++{ ++ struct io_kiocb *req = container_of(work, struct io_kiocb, work); ++ struct io_cancel_data *cd = data; ++ ++ return req->ctx == cd->ctx && req->user_data == cd->user_data; ++} ++ ++static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data, ++ struct io_ring_ctx *ctx) ++{ ++ struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, }; ++ enum io_wq_cancel cancel_ret; ++ int ret = 0; ++ ++ if (!tctx || !tctx->io_wq) ++ return -ENOENT; ++ ++ cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false); ++ switch (cancel_ret) { ++ case IO_WQ_CANCEL_OK: ++ ret = 0; ++ break; ++ case IO_WQ_CANCEL_RUNNING: ++ ret = -EALREADY; ++ break; ++ case IO_WQ_CANCEL_NOTFOUND: ++ ret = -ENOENT; ++ break; ++ } ++ ++ return ret; ++} ++ ++static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ int ret; ++ ++ WARN_ON_ONCE(!io_wq_current_is_worker() && req->task != current); ++ ++ ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx); ++ if (ret != -ENOENT) ++ return ret; ++ ++ spin_lock(&ctx->completion_lock); ++ spin_lock_irq(&ctx->timeout_lock); ++ ret = io_timeout_cancel(ctx, sqe_addr); ++ spin_unlock_irq(&ctx->timeout_lock); ++ if (ret != -ENOENT) ++ goto out; ++ ret = io_poll_cancel(ctx, sqe_addr, false); ++out: ++ spin_unlock(&ctx->completion_lock); ++ return ret; ++} ++ ++static int io_async_cancel_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ return -EINVAL; ++ if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) ++ return -EINVAL; ++ if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags || ++ sqe->splice_fd_in) ++ return -EINVAL; ++ ++ req->cancel.addr = READ_ONCE(sqe->addr); ++ return 0; ++} ++ ++static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ u64 sqe_addr = req->cancel.addr; ++ struct io_tctx_node *node; ++ int ret; ++ ++ ret = io_try_cancel_userdata(req, sqe_addr); ++ if (ret != -ENOENT) ++ goto done; ++ ++ /* slow path, try all io-wq's */ ++ io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); ++ ret = -ENOENT; ++ list_for_each_entry(node, &ctx->tctx_list, ctx_node) { ++ struct io_uring_task *tctx = node->task->io_uring; ++ ++ ret = io_async_cancel_one(tctx, req->cancel.addr, ctx); ++ if (ret != -ENOENT) ++ break; ++ } ++ io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); ++done: ++ if (ret < 0) ++ req_set_fail(req); ++ io_req_complete_post(req, ret, 0); ++ return 0; ++} ++ ++static int io_rsrc_update_prep(struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++{ ++ if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) ++ return -EINVAL; ++ if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in) ++ return -EINVAL; ++ ++ req->rsrc_update.offset = READ_ONCE(sqe->off); ++ req->rsrc_update.nr_args = READ_ONCE(sqe->len); ++ if (!req->rsrc_update.nr_args) ++ return -EINVAL; ++ req->rsrc_update.arg = READ_ONCE(sqe->addr); ++ return 0; ++} ++ ++static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ struct io_uring_rsrc_update2 up; ++ int ret; ++ ++ up.offset = req->rsrc_update.offset; ++ up.data = req->rsrc_update.arg; ++ up.nr = 0; ++ up.tags = 0; ++ up.resv = 0; ++ up.resv2 = 0; ++ ++ io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); ++ ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE, ++ &up, req->rsrc_update.nr_args); ++ io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); ++ ++ if (ret < 0) ++ req_set_fail(req); ++ __io_req_complete(req, issue_flags, ret, 0); ++ return 0; ++} ++ ++static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ++{ ++ switch (req->opcode) { ++ case IORING_OP_NOP: ++ return 0; ++ case IORING_OP_READV: ++ case IORING_OP_READ_FIXED: ++ case IORING_OP_READ: ++ return io_read_prep(req, sqe); ++ case IORING_OP_WRITEV: ++ case IORING_OP_WRITE_FIXED: ++ case IORING_OP_WRITE: ++ return io_write_prep(req, sqe); ++ case IORING_OP_POLL_ADD: ++ return io_poll_add_prep(req, sqe); ++ case IORING_OP_POLL_REMOVE: ++ return io_poll_update_prep(req, sqe); ++ case IORING_OP_FSYNC: ++ return io_fsync_prep(req, sqe); ++ case IORING_OP_SYNC_FILE_RANGE: ++ return io_sfr_prep(req, sqe); ++ case IORING_OP_SENDMSG: ++ case IORING_OP_SEND: ++ return io_sendmsg_prep(req, sqe); ++ case IORING_OP_RECVMSG: ++ case IORING_OP_RECV: ++ return io_recvmsg_prep(req, sqe); ++ case IORING_OP_CONNECT: ++ return io_connect_prep(req, sqe); ++ case IORING_OP_TIMEOUT: ++ return io_timeout_prep(req, sqe, false); ++ case IORING_OP_TIMEOUT_REMOVE: ++ return io_timeout_remove_prep(req, sqe); ++ case IORING_OP_ASYNC_CANCEL: ++ return io_async_cancel_prep(req, sqe); ++ case IORING_OP_LINK_TIMEOUT: ++ return io_timeout_prep(req, sqe, true); ++ case IORING_OP_ACCEPT: ++ return io_accept_prep(req, sqe); ++ case IORING_OP_FALLOCATE: ++ return io_fallocate_prep(req, sqe); ++ case IORING_OP_OPENAT: ++ return io_openat_prep(req, sqe); ++ case IORING_OP_CLOSE: ++ return io_close_prep(req, sqe); ++ case IORING_OP_FILES_UPDATE: ++ return io_rsrc_update_prep(req, sqe); ++ case IORING_OP_STATX: ++ return io_statx_prep(req, sqe); ++ case IORING_OP_FADVISE: ++ return io_fadvise_prep(req, sqe); ++ case IORING_OP_MADVISE: ++ return io_madvise_prep(req, sqe); ++ case IORING_OP_OPENAT2: ++ return io_openat2_prep(req, sqe); ++ case IORING_OP_EPOLL_CTL: ++ return io_epoll_ctl_prep(req, sqe); ++ case IORING_OP_SPLICE: ++ return io_splice_prep(req, sqe); ++ case IORING_OP_PROVIDE_BUFFERS: ++ return io_provide_buffers_prep(req, sqe); ++ case IORING_OP_REMOVE_BUFFERS: ++ return io_remove_buffers_prep(req, sqe); ++ case IORING_OP_TEE: ++ return io_tee_prep(req, sqe); ++ case IORING_OP_SHUTDOWN: ++ return io_shutdown_prep(req, sqe); ++ case IORING_OP_RENAMEAT: ++ return io_renameat_prep(req, sqe); ++ case IORING_OP_UNLINKAT: ++ return io_unlinkat_prep(req, sqe); ++ } ++ ++ printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n", ++ req->opcode); ++ return -EINVAL; ++} ++ ++static int io_req_prep_async(struct io_kiocb *req) ++{ ++ if (!io_op_defs[req->opcode].needs_async_setup) ++ return 0; ++ if (WARN_ON_ONCE(req->async_data)) ++ return -EFAULT; ++ if (io_alloc_async_data(req)) ++ return -EAGAIN; ++ ++ switch (req->opcode) { ++ case IORING_OP_READV: ++ return io_rw_prep_async(req, READ); ++ case IORING_OP_WRITEV: ++ return io_rw_prep_async(req, WRITE); ++ case IORING_OP_SENDMSG: ++ return io_sendmsg_prep_async(req); ++ case IORING_OP_RECVMSG: ++ return io_recvmsg_prep_async(req); ++ case IORING_OP_CONNECT: ++ return io_connect_prep_async(req); ++ } ++ printk_once(KERN_WARNING "io_uring: prep_async() bad opcode %d\n", ++ req->opcode); ++ return -EFAULT; ++} ++ ++static u32 io_get_sequence(struct io_kiocb *req) ++{ ++ u32 seq = req->ctx->cached_sq_head; ++ ++ /* need original cached_sq_head, but it was increased for each req */ ++ io_for_each_link(req, req) ++ seq--; ++ return seq; ++} ++ ++static bool io_drain_req(struct io_kiocb *req) ++{ ++ struct io_kiocb *pos; ++ struct io_ring_ctx *ctx = req->ctx; ++ struct io_defer_entry *de; ++ int ret; ++ u32 seq; ++ ++ if (req->flags & REQ_F_FAIL) { ++ io_req_complete_fail_submit(req); ++ return true; ++ } ++ ++ /* ++ * If we need to drain a request in the middle of a link, drain the ++ * head request and the next request/link after the current link. ++ * Considering sequential execution of links, IOSQE_IO_DRAIN will be ++ * maintained for every request of our link. ++ */ ++ if (ctx->drain_next) { ++ req->flags |= REQ_F_IO_DRAIN; ++ ctx->drain_next = false; ++ } ++ /* not interested in head, start from the first linked */ ++ io_for_each_link(pos, req->link) { ++ if (pos->flags & REQ_F_IO_DRAIN) { ++ ctx->drain_next = true; ++ req->flags |= REQ_F_IO_DRAIN; ++ break; ++ } ++ } ++ ++ /* Still need defer if there is pending req in defer list. */ ++ spin_lock(&ctx->completion_lock); ++ if (likely(list_empty_careful(&ctx->defer_list) && ++ !(req->flags & REQ_F_IO_DRAIN))) { ++ spin_unlock(&ctx->completion_lock); ++ ctx->drain_active = false; ++ return false; ++ } ++ spin_unlock(&ctx->completion_lock); ++ ++ seq = io_get_sequence(req); ++ /* Still a chance to pass the sequence check */ ++ if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) ++ return false; ++ ++ ret = io_req_prep_async(req); ++ if (ret) ++ goto fail; ++ io_prep_async_link(req); ++ de = kmalloc(sizeof(*de), GFP_KERNEL); ++ if (!de) { ++ ret = -ENOMEM; ++fail: ++ io_req_complete_failed(req, ret); ++ return true; ++ } ++ ++ spin_lock(&ctx->completion_lock); ++ if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) { ++ spin_unlock(&ctx->completion_lock); ++ kfree(de); ++ io_queue_async_work(req, NULL); ++ return true; ++ } ++ ++ trace_io_uring_defer(ctx, req, req->user_data); ++ de->req = req; ++ de->seq = seq; ++ list_add_tail(&de->list, &ctx->defer_list); ++ spin_unlock(&ctx->completion_lock); ++ return true; ++} ++ ++static void io_clean_op(struct io_kiocb *req) ++{ ++ if (req->flags & REQ_F_BUFFER_SELECTED) { ++ switch (req->opcode) { ++ case IORING_OP_READV: ++ case IORING_OP_READ_FIXED: ++ case IORING_OP_READ: ++ kfree((void *)(unsigned long)req->rw.addr); ++ break; ++ case IORING_OP_RECVMSG: ++ case IORING_OP_RECV: ++ kfree(req->sr_msg.kbuf); ++ break; ++ } ++ } ++ ++ if (req->flags & REQ_F_NEED_CLEANUP) { ++ switch (req->opcode) { ++ case IORING_OP_READV: ++ case IORING_OP_READ_FIXED: ++ case IORING_OP_READ: ++ case IORING_OP_WRITEV: ++ case IORING_OP_WRITE_FIXED: ++ case IORING_OP_WRITE: { ++ struct io_async_rw *io = req->async_data; ++ ++ kfree(io->free_iovec); ++ break; ++ } ++ case IORING_OP_RECVMSG: ++ case IORING_OP_SENDMSG: { ++ struct io_async_msghdr *io = req->async_data; ++ ++ kfree(io->free_iov); ++ break; ++ } ++ case IORING_OP_OPENAT: ++ case IORING_OP_OPENAT2: ++ if (req->open.filename) ++ putname(req->open.filename); ++ break; ++ case IORING_OP_RENAMEAT: ++ putname(req->rename.oldpath); ++ putname(req->rename.newpath); ++ break; ++ case IORING_OP_UNLINKAT: ++ putname(req->unlink.filename); ++ break; ++ } ++ } ++ if ((req->flags & REQ_F_POLLED) && req->apoll) { ++ kfree(req->apoll->double_poll); ++ kfree(req->apoll); ++ req->apoll = NULL; ++ } ++ if (req->flags & REQ_F_INFLIGHT) { ++ struct io_uring_task *tctx = req->task->io_uring; ++ ++ atomic_dec(&tctx->inflight_tracked); ++ } ++ if (req->flags & REQ_F_CREDS) ++ put_cred(req->creds); ++ ++ req->flags &= ~IO_REQ_CLEAN_FLAGS; ++} ++ ++static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ const struct cred *creds = NULL; ++ int ret; ++ ++ if ((req->flags & REQ_F_CREDS) && req->creds != current_cred()) ++ creds = override_creds(req->creds); ++ ++ switch (req->opcode) { ++ case IORING_OP_NOP: ++ ret = io_nop(req, issue_flags); ++ break; ++ case IORING_OP_READV: ++ case IORING_OP_READ_FIXED: ++ case IORING_OP_READ: ++ ret = io_read(req, issue_flags); ++ break; ++ case IORING_OP_WRITEV: ++ case IORING_OP_WRITE_FIXED: ++ case IORING_OP_WRITE: ++ ret = io_write(req, issue_flags); ++ break; ++ case IORING_OP_FSYNC: ++ ret = io_fsync(req, issue_flags); ++ break; ++ case IORING_OP_POLL_ADD: ++ ret = io_poll_add(req, issue_flags); ++ break; ++ case IORING_OP_POLL_REMOVE: ++ ret = io_poll_update(req, issue_flags); ++ break; ++ case IORING_OP_SYNC_FILE_RANGE: ++ ret = io_sync_file_range(req, issue_flags); ++ break; ++ case IORING_OP_SENDMSG: ++ ret = io_sendmsg(req, issue_flags); ++ break; ++ case IORING_OP_SEND: ++ ret = io_send(req, issue_flags); ++ break; ++ case IORING_OP_RECVMSG: ++ ret = io_recvmsg(req, issue_flags); ++ break; ++ case IORING_OP_RECV: ++ ret = io_recv(req, issue_flags); ++ break; ++ case IORING_OP_TIMEOUT: ++ ret = io_timeout(req, issue_flags); ++ break; ++ case IORING_OP_TIMEOUT_REMOVE: ++ ret = io_timeout_remove(req, issue_flags); ++ break; ++ case IORING_OP_ACCEPT: ++ ret = io_accept(req, issue_flags); ++ break; ++ case IORING_OP_CONNECT: ++ ret = io_connect(req, issue_flags); ++ break; ++ case IORING_OP_ASYNC_CANCEL: ++ ret = io_async_cancel(req, issue_flags); ++ break; ++ case IORING_OP_FALLOCATE: ++ ret = io_fallocate(req, issue_flags); ++ break; ++ case IORING_OP_OPENAT: ++ ret = io_openat(req, issue_flags); ++ break; ++ case IORING_OP_CLOSE: ++ ret = io_close(req, issue_flags); ++ break; ++ case IORING_OP_FILES_UPDATE: ++ ret = io_files_update(req, issue_flags); ++ break; ++ case IORING_OP_STATX: ++ ret = io_statx(req, issue_flags); ++ break; ++ case IORING_OP_FADVISE: ++ ret = io_fadvise(req, issue_flags); ++ break; ++ case IORING_OP_MADVISE: ++ ret = io_madvise(req, issue_flags); ++ break; ++ case IORING_OP_OPENAT2: ++ ret = io_openat2(req, issue_flags); ++ break; ++ case IORING_OP_EPOLL_CTL: ++ ret = io_epoll_ctl(req, issue_flags); ++ break; ++ case IORING_OP_SPLICE: ++ ret = io_splice(req, issue_flags); ++ break; ++ case IORING_OP_PROVIDE_BUFFERS: ++ ret = io_provide_buffers(req, issue_flags); ++ break; ++ case IORING_OP_REMOVE_BUFFERS: ++ ret = io_remove_buffers(req, issue_flags); ++ break; ++ case IORING_OP_TEE: ++ ret = io_tee(req, issue_flags); ++ break; ++ case IORING_OP_SHUTDOWN: ++ ret = io_shutdown(req, issue_flags); ++ break; ++ case IORING_OP_RENAMEAT: ++ ret = io_renameat(req, issue_flags); ++ break; ++ case IORING_OP_UNLINKAT: ++ ret = io_unlinkat(req, issue_flags); ++ break; ++ default: ++ ret = -EINVAL; ++ break; ++ } ++ ++ if (creds) ++ revert_creds(creds); ++ if (ret) ++ return ret; ++ /* If the op doesn't have a file, we're not polling for it */ ++ if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) ++ io_iopoll_req_issued(req); ++ ++ return 0; ++} ++ ++static struct io_wq_work *io_wq_free_work(struct io_wq_work *work) ++{ ++ struct io_kiocb *req = container_of(work, struct io_kiocb, work); ++ ++ req = io_put_req_find_next(req); ++ return req ? &req->work : NULL; ++} ++ ++static void io_wq_submit_work(struct io_wq_work *work) ++{ ++ struct io_kiocb *req = container_of(work, struct io_kiocb, work); ++ struct io_kiocb *timeout; ++ int ret = 0; ++ ++ /* one will be dropped by ->io_free_work() after returning to io-wq */ ++ if (!(req->flags & REQ_F_REFCOUNT)) ++ __io_req_set_refcount(req, 2); ++ else ++ req_ref_get(req); ++ ++ timeout = io_prep_linked_timeout(req); ++ if (timeout) ++ io_queue_linked_timeout(timeout); ++ ++ /* either cancelled or io-wq is dying, so don't touch tctx->iowq */ ++ if (work->flags & IO_WQ_WORK_CANCEL) ++ ret = -ECANCELED; ++ ++ if (!ret) { ++ do { ++ ret = io_issue_sqe(req, 0); ++ /* ++ * We can get EAGAIN for polled IO even though we're ++ * forcing a sync submission from here, since we can't ++ * wait for request slots on the block side. ++ */ ++ if (ret != -EAGAIN || !(req->ctx->flags & IORING_SETUP_IOPOLL)) ++ break; ++ cond_resched(); ++ } while (1); ++ } ++ ++ /* avoid locking problems by failing it from a clean context */ ++ if (ret) ++ io_req_task_queue_fail(req, ret); ++} ++ ++static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table, ++ unsigned i) ++{ ++ return &table->files[i]; ++} ++ ++static inline struct file *io_file_from_index(struct io_ring_ctx *ctx, ++ int index) ++{ ++ struct io_fixed_file *slot = io_fixed_file_slot(&ctx->file_table, index); ++ ++ return (struct file *) (slot->file_ptr & FFS_MASK); ++} ++ ++static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file) ++{ ++ unsigned long file_ptr = (unsigned long) file; ++ ++ if (__io_file_supports_nowait(file, READ)) ++ file_ptr |= FFS_ASYNC_READ; ++ if (__io_file_supports_nowait(file, WRITE)) ++ file_ptr |= FFS_ASYNC_WRITE; ++ if (S_ISREG(file_inode(file)->i_mode)) ++ file_ptr |= FFS_ISREG; ++ file_slot->file_ptr = file_ptr; ++} ++ ++static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx, ++ struct io_kiocb *req, int fd) ++{ ++ struct file *file; ++ unsigned long file_ptr; ++ ++ if (unlikely((unsigned int)fd >= ctx->nr_user_files)) ++ return NULL; ++ fd = array_index_nospec(fd, ctx->nr_user_files); ++ file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr; ++ file = (struct file *) (file_ptr & FFS_MASK); ++ file_ptr &= ~FFS_MASK; ++ /* mask in overlapping REQ_F and FFS bits */ ++ req->flags |= (file_ptr << REQ_F_NOWAIT_READ_BIT); ++ io_req_set_rsrc_node(req); ++ return file; ++} ++ ++static struct file *io_file_get_normal(struct io_ring_ctx *ctx, ++ struct io_kiocb *req, int fd) ++{ ++ struct file *file = fget(fd); ++ ++ trace_io_uring_file_get(ctx, fd); ++ ++ /* we don't allow fixed io_uring files */ ++ if (file && unlikely(file->f_op == &io_uring_fops)) ++ io_req_track_inflight(req); ++ return file; ++} ++ ++static inline struct file *io_file_get(struct io_ring_ctx *ctx, ++ struct io_kiocb *req, int fd, bool fixed) ++{ ++ if (fixed) ++ return io_file_get_fixed(ctx, req, fd); ++ else ++ return io_file_get_normal(ctx, req, fd); ++} ++ ++static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked) ++{ ++ struct io_kiocb *prev = req->timeout.prev; ++ int ret = -ENOENT; ++ ++ if (prev) { ++ if (!(req->task->flags & PF_EXITING)) ++ ret = io_try_cancel_userdata(req, prev->user_data); ++ io_req_complete_post(req, ret ?: -ETIME, 0); ++ io_put_req(prev); ++ } else { ++ io_req_complete_post(req, -ETIME, 0); ++ } ++} ++ ++static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) ++{ ++ struct io_timeout_data *data = container_of(timer, ++ struct io_timeout_data, timer); ++ struct io_kiocb *prev, *req = data->req; ++ struct io_ring_ctx *ctx = req->ctx; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&ctx->timeout_lock, flags); ++ prev = req->timeout.head; ++ req->timeout.head = NULL; ++ ++ /* ++ * We don't expect the list to be empty, that will only happen if we ++ * race with the completion of the linked work. ++ */ ++ if (prev) { ++ io_remove_next_linked(prev); ++ if (!req_ref_inc_not_zero(prev)) ++ prev = NULL; ++ } ++ list_del(&req->timeout.list); ++ req->timeout.prev = prev; ++ spin_unlock_irqrestore(&ctx->timeout_lock, flags); ++ ++ req->io_task_work.func = io_req_task_link_timeout; ++ io_req_task_work_add(req); ++ return HRTIMER_NORESTART; ++} ++ ++static void io_queue_linked_timeout(struct io_kiocb *req) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ ++ spin_lock_irq(&ctx->timeout_lock); ++ /* ++ * If the back reference is NULL, then our linked request finished ++ * before we got a chance to setup the timer ++ */ ++ if (req->timeout.head) { ++ struct io_timeout_data *data = req->async_data; ++ ++ data->timer.function = io_link_timeout_fn; ++ hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), ++ data->mode); ++ list_add_tail(&req->timeout.list, &ctx->ltimeout_list); ++ } ++ spin_unlock_irq(&ctx->timeout_lock); ++ /* drop submission reference */ ++ io_put_req(req); ++} ++ ++static void __io_queue_sqe(struct io_kiocb *req) ++ __must_hold(&req->ctx->uring_lock) ++{ ++ struct io_kiocb *linked_timeout; ++ int ret; ++ ++issue_sqe: ++ ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); ++ ++ /* ++ * We async punt it if the file wasn't marked NOWAIT, or if the file ++ * doesn't support non-blocking read/write attempts ++ */ ++ if (likely(!ret)) { ++ if (req->flags & REQ_F_COMPLETE_INLINE) { ++ struct io_ring_ctx *ctx = req->ctx; ++ struct io_submit_state *state = &ctx->submit_state; ++ ++ state->compl_reqs[state->compl_nr++] = req; ++ if (state->compl_nr == ARRAY_SIZE(state->compl_reqs)) ++ io_submit_flush_completions(ctx); ++ return; ++ } ++ ++ linked_timeout = io_prep_linked_timeout(req); ++ if (linked_timeout) ++ io_queue_linked_timeout(linked_timeout); ++ } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) { ++ linked_timeout = io_prep_linked_timeout(req); ++ ++ switch (io_arm_poll_handler(req)) { ++ case IO_APOLL_READY: ++ if (linked_timeout) ++ io_queue_linked_timeout(linked_timeout); ++ goto issue_sqe; ++ case IO_APOLL_ABORTED: ++ /* ++ * Queued up for async execution, worker will release ++ * submit reference when the iocb is actually submitted. ++ */ ++ io_queue_async_work(req, NULL); ++ break; ++ } ++ ++ if (linked_timeout) ++ io_queue_linked_timeout(linked_timeout); ++ } else { ++ io_req_complete_failed(req, ret); ++ } ++} ++ ++static inline void io_queue_sqe(struct io_kiocb *req) ++ __must_hold(&req->ctx->uring_lock) ++{ ++ if (unlikely(req->ctx->drain_active) && io_drain_req(req)) ++ return; ++ ++ if (likely(!(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)))) { ++ __io_queue_sqe(req); ++ } else if (req->flags & REQ_F_FAIL) { ++ io_req_complete_fail_submit(req); ++ } else { ++ int ret = io_req_prep_async(req); ++ ++ if (unlikely(ret)) ++ io_req_complete_failed(req, ret); ++ else ++ io_queue_async_work(req, NULL); ++ } ++} ++ ++/* ++ * Check SQE restrictions (opcode and flags). ++ * ++ * Returns 'true' if SQE is allowed, 'false' otherwise. ++ */ ++static inline bool io_check_restriction(struct io_ring_ctx *ctx, ++ struct io_kiocb *req, ++ unsigned int sqe_flags) ++{ ++ if (likely(!ctx->restricted)) ++ return true; ++ ++ if (!test_bit(req->opcode, ctx->restrictions.sqe_op)) ++ return false; ++ ++ if ((sqe_flags & ctx->restrictions.sqe_flags_required) != ++ ctx->restrictions.sqe_flags_required) ++ return false; ++ ++ if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed | ++ ctx->restrictions.sqe_flags_required)) ++ return false; ++ ++ return true; ++} ++ ++static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++ __must_hold(&ctx->uring_lock) ++{ ++ struct io_submit_state *state; ++ unsigned int sqe_flags; ++ int personality, ret = 0; ++ ++ /* req is partially pre-initialised, see io_preinit_req() */ ++ req->opcode = READ_ONCE(sqe->opcode); ++ /* same numerical values with corresponding REQ_F_*, safe to copy */ ++ req->flags = sqe_flags = READ_ONCE(sqe->flags); ++ req->user_data = READ_ONCE(sqe->user_data); ++ req->file = NULL; ++ req->fixed_rsrc_refs = NULL; ++ req->task = current; ++ ++ /* enforce forwards compatibility on users */ ++ if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) ++ return -EINVAL; ++ if (unlikely(req->opcode >= IORING_OP_LAST)) ++ return -EINVAL; ++ if (!io_check_restriction(ctx, req, sqe_flags)) ++ return -EACCES; ++ ++ if ((sqe_flags & IOSQE_BUFFER_SELECT) && ++ !io_op_defs[req->opcode].buffer_select) ++ return -EOPNOTSUPP; ++ if (unlikely(sqe_flags & IOSQE_IO_DRAIN)) ++ ctx->drain_active = true; ++ ++ personality = READ_ONCE(sqe->personality); ++ if (personality) { ++ req->creds = xa_load(&ctx->personalities, personality); ++ if (!req->creds) ++ return -EINVAL; ++ get_cred(req->creds); ++ req->flags |= REQ_F_CREDS; ++ } ++ state = &ctx->submit_state; ++ ++ /* ++ * Plug now if we have more than 1 IO left after this, and the target ++ * is potentially a read/write to block based storage. ++ */ ++ if (!state->plug_started && state->ios_left > 1 && ++ io_op_defs[req->opcode].plug) { ++ blk_start_plug(&state->plug); ++ state->plug_started = true; ++ } ++ ++ if (io_op_defs[req->opcode].needs_file) { ++ req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd), ++ (sqe_flags & IOSQE_FIXED_FILE)); ++ if (unlikely(!req->file)) ++ ret = -EBADF; ++ } ++ ++ state->ios_left--; ++ return ret; ++} ++ ++static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, ++ const struct io_uring_sqe *sqe) ++ __must_hold(&ctx->uring_lock) ++{ ++ struct io_submit_link *link = &ctx->submit_state.link; ++ int ret; ++ ++ ret = io_init_req(ctx, req, sqe); ++ if (unlikely(ret)) { ++fail_req: ++ /* fail even hard links since we don't submit */ ++ if (link->head) { ++ /* ++ * we can judge a link req is failed or cancelled by if ++ * REQ_F_FAIL is set, but the head is an exception since ++ * it may be set REQ_F_FAIL because of other req's failure ++ * so let's leverage req->result to distinguish if a head ++ * is set REQ_F_FAIL because of its failure or other req's ++ * failure so that we can set the correct ret code for it. ++ * init result here to avoid affecting the normal path. ++ */ ++ if (!(link->head->flags & REQ_F_FAIL)) ++ req_fail_link_node(link->head, -ECANCELED); ++ } else if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) { ++ /* ++ * the current req is a normal req, we should return ++ * error and thus break the submittion loop. ++ */ ++ io_req_complete_failed(req, ret); ++ return ret; ++ } ++ req_fail_link_node(req, ret); ++ } else { ++ ret = io_req_prep(req, sqe); ++ if (unlikely(ret)) ++ goto fail_req; ++ } ++ ++ /* don't need @sqe from now on */ ++ trace_io_uring_submit_sqe(ctx, req, req->opcode, req->user_data, ++ req->flags, true, ++ ctx->flags & IORING_SETUP_SQPOLL); ++ ++ /* ++ * If we already have a head request, queue this one for async ++ * submittal once the head completes. If we don't have a head but ++ * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be ++ * submitted sync once the chain is complete. If none of those ++ * conditions are true (normal request), then just queue it. ++ */ ++ if (link->head) { ++ struct io_kiocb *head = link->head; ++ ++ if (!(req->flags & REQ_F_FAIL)) { ++ ret = io_req_prep_async(req); ++ if (unlikely(ret)) { ++ req_fail_link_node(req, ret); ++ if (!(head->flags & REQ_F_FAIL)) ++ req_fail_link_node(head, -ECANCELED); ++ } ++ } ++ trace_io_uring_link(ctx, req, head); ++ link->last->link = req; ++ link->last = req; ++ ++ /* last request of a link, enqueue the link */ ++ if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) { ++ link->head = NULL; ++ io_queue_sqe(head); ++ } ++ } else { ++ if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { ++ link->head = req; ++ link->last = req; ++ } else { ++ io_queue_sqe(req); ++ } ++ } ++ ++ return 0; ++} ++ ++/* ++ * Batched submission is done, ensure local IO is flushed out. ++ */ ++static void io_submit_state_end(struct io_submit_state *state, ++ struct io_ring_ctx *ctx) ++{ ++ if (state->link.head) ++ io_queue_sqe(state->link.head); ++ if (state->compl_nr) ++ io_submit_flush_completions(ctx); ++ if (state->plug_started) ++ blk_finish_plug(&state->plug); ++} ++ ++/* ++ * Start submission side cache. ++ */ ++static void io_submit_state_start(struct io_submit_state *state, ++ unsigned int max_ios) ++{ ++ state->plug_started = false; ++ state->ios_left = max_ios; ++ /* set only head, no need to init link_last in advance */ ++ state->link.head = NULL; ++} ++ ++static void io_commit_sqring(struct io_ring_ctx *ctx) ++{ ++ struct io_rings *rings = ctx->rings; ++ ++ /* ++ * Ensure any loads from the SQEs are done at this point, ++ * since once we write the new head, the application could ++ * write new data to them. ++ */ ++ smp_store_release(&rings->sq.head, ctx->cached_sq_head); ++} ++ ++/* ++ * Fetch an sqe, if one is available. Note this returns a pointer to memory ++ * that is mapped by userspace. This means that care needs to be taken to ++ * ensure that reads are stable, as we cannot rely on userspace always ++ * being a good citizen. If members of the sqe are validated and then later ++ * used, it's important that those reads are done through READ_ONCE() to ++ * prevent a re-load down the line. ++ */ ++static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) ++{ ++ unsigned head, mask = ctx->sq_entries - 1; ++ unsigned sq_idx = ctx->cached_sq_head++ & mask; ++ ++ /* ++ * The cached sq head (or cq tail) serves two purposes: ++ * ++ * 1) allows us to batch the cost of updating the user visible ++ * head updates. ++ * 2) allows the kernel side to track the head on its own, even ++ * though the application is the one updating it. ++ */ ++ head = READ_ONCE(ctx->sq_array[sq_idx]); ++ if (likely(head < ctx->sq_entries)) ++ return &ctx->sq_sqes[head]; ++ ++ /* drop invalid entries */ ++ ctx->cq_extra--; ++ WRITE_ONCE(ctx->rings->sq_dropped, ++ READ_ONCE(ctx->rings->sq_dropped) + 1); ++ return NULL; ++} ++ ++static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) ++ __must_hold(&ctx->uring_lock) ++{ ++ int submitted = 0; ++ ++ /* make sure SQ entry isn't read before tail */ ++ nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx)); ++ if (!percpu_ref_tryget_many(&ctx->refs, nr)) ++ return -EAGAIN; ++ io_get_task_refs(nr); ++ ++ io_submit_state_start(&ctx->submit_state, nr); ++ while (submitted < nr) { ++ const struct io_uring_sqe *sqe; ++ struct io_kiocb *req; ++ ++ req = io_alloc_req(ctx); ++ if (unlikely(!req)) { ++ if (!submitted) ++ submitted = -EAGAIN; ++ break; ++ } ++ sqe = io_get_sqe(ctx); ++ if (unlikely(!sqe)) { ++ list_add(&req->inflight_entry, &ctx->submit_state.free_list); ++ break; ++ } ++ /* will complete beyond this point, count as submitted */ ++ submitted++; ++ if (io_submit_sqe(ctx, req, sqe)) ++ break; ++ } ++ ++ if (unlikely(submitted != nr)) { ++ int ref_used = (submitted == -EAGAIN) ? 0 : submitted; ++ int unused = nr - ref_used; ++ ++ current->io_uring->cached_refs += unused; ++ percpu_ref_put_many(&ctx->refs, unused); ++ } ++ ++ io_submit_state_end(&ctx->submit_state, ctx); ++ /* Commit SQ ring head once we've consumed and submitted all SQEs */ ++ io_commit_sqring(ctx); ++ ++ return submitted; ++} ++ ++static inline bool io_sqd_events_pending(struct io_sq_data *sqd) ++{ ++ return READ_ONCE(sqd->state); ++} ++ ++static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx) ++{ ++ /* Tell userspace we may need a wakeup call */ ++ spin_lock(&ctx->completion_lock); ++ WRITE_ONCE(ctx->rings->sq_flags, ++ ctx->rings->sq_flags | IORING_SQ_NEED_WAKEUP); ++ spin_unlock(&ctx->completion_lock); ++} ++ ++static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx) ++{ ++ spin_lock(&ctx->completion_lock); ++ WRITE_ONCE(ctx->rings->sq_flags, ++ ctx->rings->sq_flags & ~IORING_SQ_NEED_WAKEUP); ++ spin_unlock(&ctx->completion_lock); ++} ++ ++static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) ++{ ++ unsigned int to_submit; ++ int ret = 0; ++ ++ to_submit = io_sqring_entries(ctx); ++ /* if we're handling multiple rings, cap submit size for fairness */ ++ if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE) ++ to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE; ++ ++ if (!list_empty(&ctx->iopoll_list) || to_submit) { ++ unsigned nr_events = 0; ++ const struct cred *creds = NULL; ++ ++ if (ctx->sq_creds != current_cred()) ++ creds = override_creds(ctx->sq_creds); ++ ++ mutex_lock(&ctx->uring_lock); ++ if (!list_empty(&ctx->iopoll_list)) ++ io_do_iopoll(ctx, &nr_events, 0); ++ ++ /* ++ * Don't submit if refs are dying, good for io_uring_register(), ++ * but also it is relied upon by io_ring_exit_work() ++ */ ++ if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) && ++ !(ctx->flags & IORING_SETUP_R_DISABLED)) ++ ret = io_submit_sqes(ctx, to_submit); ++ mutex_unlock(&ctx->uring_lock); ++ ++ if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait)) ++ wake_up(&ctx->sqo_sq_wait); ++ if (creds) ++ revert_creds(creds); ++ } ++ ++ return ret; ++} ++ ++static void io_sqd_update_thread_idle(struct io_sq_data *sqd) ++{ ++ struct io_ring_ctx *ctx; ++ unsigned sq_thread_idle = 0; ++ ++ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) ++ sq_thread_idle = max(sq_thread_idle, ctx->sq_thread_idle); ++ sqd->sq_thread_idle = sq_thread_idle; ++} ++ ++static bool io_sqd_handle_event(struct io_sq_data *sqd) ++{ ++ bool did_sig = false; ++ struct ksignal ksig; ++ ++ if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) || ++ signal_pending(current)) { ++ mutex_unlock(&sqd->lock); ++ if (signal_pending(current)) ++ did_sig = get_signal(&ksig); ++ cond_resched(); ++ mutex_lock(&sqd->lock); ++ } ++ return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); ++} ++ ++static int io_sq_thread(void *data) ++{ ++ struct io_sq_data *sqd = data; ++ struct io_ring_ctx *ctx; ++ unsigned long timeout = 0; ++ char buf[TASK_COMM_LEN]; ++ DEFINE_WAIT(wait); ++ ++ snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid); ++ set_task_comm(current, buf); ++ ++ if (sqd->sq_cpu != -1) ++ set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu)); ++ else ++ set_cpus_allowed_ptr(current, cpu_online_mask); ++ current->flags |= PF_NO_SETAFFINITY; ++ ++ mutex_lock(&sqd->lock); ++ while (1) { ++ bool cap_entries, sqt_spin = false; ++ ++ if (io_sqd_events_pending(sqd) || signal_pending(current)) { ++ if (io_sqd_handle_event(sqd)) ++ break; ++ timeout = jiffies + sqd->sq_thread_idle; ++ } ++ ++ cap_entries = !list_is_singular(&sqd->ctx_list); ++ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { ++ int ret = __io_sq_thread(ctx, cap_entries); ++ ++ if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list))) ++ sqt_spin = true; ++ } ++ if (io_run_task_work()) ++ sqt_spin = true; ++ ++ if (sqt_spin || !time_after(jiffies, timeout)) { ++ cond_resched(); ++ if (sqt_spin) ++ timeout = jiffies + sqd->sq_thread_idle; ++ continue; ++ } ++ ++ prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); ++ if (!io_sqd_events_pending(sqd) && !current->task_works) { ++ bool needs_sched = true; ++ ++ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { ++ io_ring_set_wakeup_flag(ctx); ++ ++ if ((ctx->flags & IORING_SETUP_IOPOLL) && ++ !list_empty_careful(&ctx->iopoll_list)) { ++ needs_sched = false; ++ break; ++ } ++ if (io_sqring_entries(ctx)) { ++ needs_sched = false; ++ break; ++ } ++ } ++ ++ if (needs_sched) { ++ mutex_unlock(&sqd->lock); ++ schedule(); ++ mutex_lock(&sqd->lock); ++ } ++ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) ++ io_ring_clear_wakeup_flag(ctx); ++ } ++ ++ finish_wait(&sqd->wait, &wait); ++ timeout = jiffies + sqd->sq_thread_idle; ++ } ++ ++ io_uring_cancel_generic(true, sqd); ++ sqd->thread = NULL; ++ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) ++ io_ring_set_wakeup_flag(ctx); ++ io_run_task_work(); ++ mutex_unlock(&sqd->lock); ++ ++ complete(&sqd->exited); ++ do_exit(0); ++} ++ ++struct io_wait_queue { ++ struct wait_queue_entry wq; ++ struct io_ring_ctx *ctx; ++ unsigned cq_tail; ++ unsigned nr_timeouts; ++}; ++ ++static inline bool io_should_wake(struct io_wait_queue *iowq) ++{ ++ struct io_ring_ctx *ctx = iowq->ctx; ++ int dist = ctx->cached_cq_tail - (int) iowq->cq_tail; ++ ++ /* ++ * Wake up if we have enough events, or if a timeout occurred since we ++ * started waiting. For timeouts, we always want to return to userspace, ++ * regardless of event count. ++ */ ++ return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; ++} ++ ++static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, ++ int wake_flags, void *key) ++{ ++ struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue, ++ wq); ++ ++ /* ++ * Cannot safely flush overflowed CQEs from here, ensure we wake up ++ * the task, and the next invocation will do it. ++ */ ++ if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->check_cq_overflow)) ++ return autoremove_wake_function(curr, mode, wake_flags, key); ++ return -1; ++} ++ ++static int io_run_task_work_sig(void) ++{ ++ if (io_run_task_work()) ++ return 1; ++ if (!signal_pending(current)) ++ return 0; ++ if (test_thread_flag(TIF_NOTIFY_SIGNAL)) ++ return -ERESTARTSYS; ++ return -EINTR; ++} ++ ++/* when returns >0, the caller should retry */ ++static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, ++ struct io_wait_queue *iowq, ++ ktime_t timeout) ++{ ++ int ret; ++ ++ /* make sure we run task_work before checking for signals */ ++ ret = io_run_task_work_sig(); ++ if (ret || io_should_wake(iowq)) ++ return ret; ++ /* let the caller flush overflows, retry */ ++ if (test_bit(0, &ctx->check_cq_overflow)) ++ return 1; ++ ++ if (!schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS)) ++ return -ETIME; ++ return 1; ++} ++ ++/* ++ * Wait until events become available, if we don't already have some. The ++ * application must reap them itself, as they reside on the shared cq ring. ++ */ ++static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, ++ const sigset_t __user *sig, size_t sigsz, ++ struct __kernel_timespec __user *uts) ++{ ++ struct io_wait_queue iowq; ++ struct io_rings *rings = ctx->rings; ++ ktime_t timeout = KTIME_MAX; ++ int ret; ++ ++ do { ++ io_cqring_overflow_flush(ctx); ++ if (io_cqring_events(ctx) >= min_events) ++ return 0; ++ if (!io_run_task_work()) ++ break; ++ } while (1); ++ ++ if (uts) { ++ struct timespec64 ts; ++ ++ if (get_timespec64(&ts, uts)) ++ return -EFAULT; ++ timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns()); ++ } ++ ++ if (sig) { ++#ifdef CONFIG_COMPAT ++ if (in_compat_syscall()) ++ ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig, ++ sigsz); ++ else ++#endif ++ ret = set_user_sigmask(sig, sigsz); ++ ++ if (ret) ++ return ret; ++ } ++ ++ init_waitqueue_func_entry(&iowq.wq, io_wake_function); ++ iowq.wq.private = current; ++ INIT_LIST_HEAD(&iowq.wq.entry); ++ iowq.ctx = ctx; ++ iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts); ++ iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events; ++ ++ trace_io_uring_cqring_wait(ctx, min_events); ++ do { ++ /* if we can't even flush overflow, don't wait for more */ ++ if (!io_cqring_overflow_flush(ctx)) { ++ ret = -EBUSY; ++ break; ++ } ++ prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, ++ TASK_INTERRUPTIBLE); ++ ret = io_cqring_wait_schedule(ctx, &iowq, timeout); ++ finish_wait(&ctx->cq_wait, &iowq.wq); ++ cond_resched(); ++ } while (ret > 0); ++ ++ restore_saved_sigmask_unless(ret == -EINTR); ++ ++ return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0; ++} ++ ++static void io_free_page_table(void **table, size_t size) ++{ ++ unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE); ++ ++ for (i = 0; i < nr_tables; i++) ++ kfree(table[i]); ++ kfree(table); ++} ++ ++static void **io_alloc_page_table(size_t size) ++{ ++ unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE); ++ size_t init_size = size; ++ void **table; ++ ++ table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL_ACCOUNT); ++ if (!table) ++ return NULL; ++ ++ for (i = 0; i < nr_tables; i++) { ++ unsigned int this_size = min_t(size_t, size, PAGE_SIZE); ++ ++ table[i] = kzalloc(this_size, GFP_KERNEL_ACCOUNT); ++ if (!table[i]) { ++ io_free_page_table(table, init_size); ++ return NULL; ++ } ++ size -= this_size; ++ } ++ return table; ++} ++ ++static void io_rsrc_node_destroy(struct io_rsrc_node *ref_node) ++{ ++ percpu_ref_exit(&ref_node->refs); ++ kfree(ref_node); ++} ++ ++static void io_rsrc_node_ref_zero(struct percpu_ref *ref) ++{ ++ struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs); ++ struct io_ring_ctx *ctx = node->rsrc_data->ctx; ++ unsigned long flags; ++ bool first_add = false; ++ unsigned long delay = HZ; ++ ++ spin_lock_irqsave(&ctx->rsrc_ref_lock, flags); ++ node->done = true; ++ ++ /* if we are mid-quiesce then do not delay */ ++ if (node->rsrc_data->quiesce) ++ delay = 0; ++ ++ while (!list_empty(&ctx->rsrc_ref_list)) { ++ node = list_first_entry(&ctx->rsrc_ref_list, ++ struct io_rsrc_node, node); ++ /* recycle ref nodes in order */ ++ if (!node->done) ++ break; ++ list_del(&node->node); ++ first_add |= llist_add(&node->llist, &ctx->rsrc_put_llist); ++ } ++ spin_unlock_irqrestore(&ctx->rsrc_ref_lock, flags); ++ ++ if (first_add) ++ mod_delayed_work(system_wq, &ctx->rsrc_put_work, delay); ++} ++ ++static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx) ++{ ++ struct io_rsrc_node *ref_node; ++ ++ ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL); ++ if (!ref_node) ++ return NULL; ++ ++ if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero, ++ 0, GFP_KERNEL)) { ++ kfree(ref_node); ++ return NULL; ++ } ++ INIT_LIST_HEAD(&ref_node->node); ++ INIT_LIST_HEAD(&ref_node->rsrc_list); ++ ref_node->done = false; ++ return ref_node; ++} ++ ++static void io_rsrc_node_switch(struct io_ring_ctx *ctx, ++ struct io_rsrc_data *data_to_kill) ++{ ++ WARN_ON_ONCE(!ctx->rsrc_backup_node); ++ WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node); ++ ++ if (data_to_kill) { ++ struct io_rsrc_node *rsrc_node = ctx->rsrc_node; ++ ++ rsrc_node->rsrc_data = data_to_kill; ++ spin_lock_irq(&ctx->rsrc_ref_lock); ++ list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list); ++ spin_unlock_irq(&ctx->rsrc_ref_lock); ++ ++ atomic_inc(&data_to_kill->refs); ++ percpu_ref_kill(&rsrc_node->refs); ++ ctx->rsrc_node = NULL; ++ } ++ ++ if (!ctx->rsrc_node) { ++ ctx->rsrc_node = ctx->rsrc_backup_node; ++ ctx->rsrc_backup_node = NULL; ++ } ++} ++ ++static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx) ++{ ++ if (ctx->rsrc_backup_node) ++ return 0; ++ ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx); ++ return ctx->rsrc_backup_node ? 0 : -ENOMEM; ++} ++ ++static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx) ++{ ++ int ret; ++ ++ /* As we may drop ->uring_lock, other task may have started quiesce */ ++ if (data->quiesce) ++ return -ENXIO; ++ ++ data->quiesce = true; ++ do { ++ ret = io_rsrc_node_switch_start(ctx); ++ if (ret) ++ break; ++ io_rsrc_node_switch(ctx, data); ++ ++ /* kill initial ref, already quiesced if zero */ ++ if (atomic_dec_and_test(&data->refs)) ++ break; ++ mutex_unlock(&ctx->uring_lock); ++ flush_delayed_work(&ctx->rsrc_put_work); ++ ret = wait_for_completion_interruptible(&data->done); ++ if (!ret) { ++ mutex_lock(&ctx->uring_lock); ++ if (atomic_read(&data->refs) > 0) { ++ /* ++ * it has been revived by another thread while ++ * we were unlocked ++ */ ++ mutex_unlock(&ctx->uring_lock); ++ } else { ++ break; ++ } ++ } ++ ++ atomic_inc(&data->refs); ++ /* wait for all works potentially completing data->done */ ++ flush_delayed_work(&ctx->rsrc_put_work); ++ reinit_completion(&data->done); ++ ++ ret = io_run_task_work_sig(); ++ mutex_lock(&ctx->uring_lock); ++ } while (ret >= 0); ++ data->quiesce = false; ++ ++ return ret; ++} ++ ++static u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx) ++{ ++ unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK; ++ unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT; ++ ++ return &data->tags[table_idx][off]; ++} ++ ++static void io_rsrc_data_free(struct io_rsrc_data *data) ++{ ++ size_t size = data->nr * sizeof(data->tags[0][0]); ++ ++ if (data->tags) ++ io_free_page_table((void **)data->tags, size); ++ kfree(data); ++} ++ ++static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put, ++ u64 __user *utags, unsigned nr, ++ struct io_rsrc_data **pdata) ++{ ++ struct io_rsrc_data *data; ++ int ret = -ENOMEM; ++ unsigned i; ++ ++ data = kzalloc(sizeof(*data), GFP_KERNEL); ++ if (!data) ++ return -ENOMEM; ++ data->tags = (u64 **)io_alloc_page_table(nr * sizeof(data->tags[0][0])); ++ if (!data->tags) { ++ kfree(data); ++ return -ENOMEM; ++ } ++ ++ data->nr = nr; ++ data->ctx = ctx; ++ data->do_put = do_put; ++ if (utags) { ++ ret = -EFAULT; ++ for (i = 0; i < nr; i++) { ++ u64 *tag_slot = io_get_tag_slot(data, i); ++ ++ if (copy_from_user(tag_slot, &utags[i], ++ sizeof(*tag_slot))) ++ goto fail; ++ } ++ } ++ ++ atomic_set(&data->refs, 1); ++ init_completion(&data->done); ++ *pdata = data; ++ return 0; ++fail: ++ io_rsrc_data_free(data); ++ return ret; ++} ++ ++static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) ++{ ++ table->files = kvcalloc(nr_files, sizeof(table->files[0]), ++ GFP_KERNEL_ACCOUNT); ++ return !!table->files; ++} ++ ++static void io_free_file_tables(struct io_file_table *table) ++{ ++ kvfree(table->files); ++ table->files = NULL; ++} ++ ++static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) ++{ ++#if defined(CONFIG_UNIX) ++ if (ctx->ring_sock) { ++ struct sock *sock = ctx->ring_sock->sk; ++ struct sk_buff *skb; ++ ++ while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL) ++ kfree_skb(skb); ++ } ++#else ++ int i; ++ ++ for (i = 0; i < ctx->nr_user_files; i++) { ++ struct file *file; ++ ++ file = io_file_from_index(ctx, i); ++ if (file) ++ fput(file); ++ } ++#endif ++ io_free_file_tables(&ctx->file_table); ++ io_rsrc_data_free(ctx->file_data); ++ ctx->file_data = NULL; ++ ctx->nr_user_files = 0; ++} ++ ++static int io_sqe_files_unregister(struct io_ring_ctx *ctx) ++{ ++ unsigned nr = ctx->nr_user_files; ++ int ret; ++ ++ if (!ctx->file_data) ++ return -ENXIO; ++ ++ /* ++ * Quiesce may unlock ->uring_lock, and while it's not held ++ * prevent new requests using the table. ++ */ ++ ctx->nr_user_files = 0; ++ ret = io_rsrc_ref_quiesce(ctx->file_data, ctx); ++ ctx->nr_user_files = nr; ++ if (!ret) ++ __io_sqe_files_unregister(ctx); ++ return ret; ++} ++ ++static void io_sq_thread_unpark(struct io_sq_data *sqd) ++ __releases(&sqd->lock) ++{ ++ WARN_ON_ONCE(sqd->thread == current); ++ ++ /* ++ * Do the dance but not conditional clear_bit() because it'd race with ++ * other threads incrementing park_pending and setting the bit. ++ */ ++ clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); ++ if (atomic_dec_return(&sqd->park_pending)) ++ set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); ++ mutex_unlock(&sqd->lock); ++} ++ ++static void io_sq_thread_park(struct io_sq_data *sqd) ++ __acquires(&sqd->lock) ++{ ++ WARN_ON_ONCE(sqd->thread == current); ++ ++ atomic_inc(&sqd->park_pending); ++ set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); ++ mutex_lock(&sqd->lock); ++ if (sqd->thread) ++ wake_up_process(sqd->thread); ++} ++ ++static void io_sq_thread_stop(struct io_sq_data *sqd) ++{ ++ WARN_ON_ONCE(sqd->thread == current); ++ WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)); ++ ++ set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); ++ mutex_lock(&sqd->lock); ++ if (sqd->thread) ++ wake_up_process(sqd->thread); ++ mutex_unlock(&sqd->lock); ++ wait_for_completion(&sqd->exited); ++} ++ ++static void io_put_sq_data(struct io_sq_data *sqd) ++{ ++ if (refcount_dec_and_test(&sqd->refs)) { ++ WARN_ON_ONCE(atomic_read(&sqd->park_pending)); ++ ++ io_sq_thread_stop(sqd); ++ kfree(sqd); ++ } ++} ++ ++static void io_sq_thread_finish(struct io_ring_ctx *ctx) ++{ ++ struct io_sq_data *sqd = ctx->sq_data; ++ ++ if (sqd) { ++ io_sq_thread_park(sqd); ++ list_del_init(&ctx->sqd_list); ++ io_sqd_update_thread_idle(sqd); ++ io_sq_thread_unpark(sqd); ++ ++ io_put_sq_data(sqd); ++ ctx->sq_data = NULL; ++ } ++} ++ ++static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p) ++{ ++ struct io_ring_ctx *ctx_attach; ++ struct io_sq_data *sqd; ++ struct fd f; ++ ++ f = fdget(p->wq_fd); ++ if (!f.file) ++ return ERR_PTR(-ENXIO); ++ if (f.file->f_op != &io_uring_fops) { ++ fdput(f); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ ctx_attach = f.file->private_data; ++ sqd = ctx_attach->sq_data; ++ if (!sqd) { ++ fdput(f); ++ return ERR_PTR(-EINVAL); ++ } ++ if (sqd->task_tgid != current->tgid) { ++ fdput(f); ++ return ERR_PTR(-EPERM); ++ } ++ ++ refcount_inc(&sqd->refs); ++ fdput(f); ++ return sqd; ++} ++ ++static struct io_sq_data *io_get_sq_data(struct io_uring_params *p, ++ bool *attached) ++{ ++ struct io_sq_data *sqd; ++ ++ *attached = false; ++ if (p->flags & IORING_SETUP_ATTACH_WQ) { ++ sqd = io_attach_sq_data(p); ++ if (!IS_ERR(sqd)) { ++ *attached = true; ++ return sqd; ++ } ++ /* fall through for EPERM case, setup new sqd/task */ ++ if (PTR_ERR(sqd) != -EPERM) ++ return sqd; ++ } ++ ++ sqd = kzalloc(sizeof(*sqd), GFP_KERNEL); ++ if (!sqd) ++ return ERR_PTR(-ENOMEM); ++ ++ atomic_set(&sqd->park_pending, 0); ++ refcount_set(&sqd->refs, 1); ++ INIT_LIST_HEAD(&sqd->ctx_list); ++ mutex_init(&sqd->lock); ++ init_waitqueue_head(&sqd->wait); ++ init_completion(&sqd->exited); ++ return sqd; ++} ++ ++#if defined(CONFIG_UNIX) ++/* ++ * Ensure the UNIX gc is aware of our file set, so we are certain that ++ * the io_uring can be safely unregistered on process exit, even if we have ++ * loops in the file referencing. ++ */ ++static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset) ++{ ++ struct sock *sk = ctx->ring_sock->sk; ++ struct scm_fp_list *fpl; ++ struct sk_buff *skb; ++ int i, nr_files; ++ ++ fpl = kzalloc(sizeof(*fpl), GFP_KERNEL); ++ if (!fpl) ++ return -ENOMEM; ++ ++ skb = alloc_skb(0, GFP_KERNEL); ++ if (!skb) { ++ kfree(fpl); ++ return -ENOMEM; ++ } ++ ++ skb->sk = sk; ++ skb->scm_io_uring = 1; ++ ++ nr_files = 0; ++ fpl->user = get_uid(current_user()); ++ for (i = 0; i < nr; i++) { ++ struct file *file = io_file_from_index(ctx, i + offset); ++ ++ if (!file) ++ continue; ++ fpl->fp[nr_files] = get_file(file); ++ unix_inflight(fpl->user, fpl->fp[nr_files]); ++ nr_files++; ++ } ++ ++ if (nr_files) { ++ fpl->max = SCM_MAX_FD; ++ fpl->count = nr_files; ++ UNIXCB(skb).fp = fpl; ++ skb->destructor = unix_destruct_scm; ++ refcount_add(skb->truesize, &sk->sk_wmem_alloc); ++ skb_queue_head(&sk->sk_receive_queue, skb); ++ ++ for (i = 0; i < nr; i++) { ++ struct file *file = io_file_from_index(ctx, i + offset); ++ ++ if (file) ++ fput(file); ++ } ++ } else { ++ kfree_skb(skb); ++ free_uid(fpl->user); ++ kfree(fpl); ++ } ++ ++ return 0; ++} ++ ++/* ++ * If UNIX sockets are enabled, fd passing can cause a reference cycle which ++ * causes regular reference counting to break down. We rely on the UNIX ++ * garbage collection to take care of this problem for us. ++ */ ++static int io_sqe_files_scm(struct io_ring_ctx *ctx) ++{ ++ unsigned left, total; ++ int ret = 0; ++ ++ total = 0; ++ left = ctx->nr_user_files; ++ while (left) { ++ unsigned this_files = min_t(unsigned, left, SCM_MAX_FD); ++ ++ ret = __io_sqe_files_scm(ctx, this_files, total); ++ if (ret) ++ break; ++ left -= this_files; ++ total += this_files; ++ } ++ ++ if (!ret) ++ return 0; ++ ++ while (total < ctx->nr_user_files) { ++ struct file *file = io_file_from_index(ctx, total); ++ ++ if (file) ++ fput(file); ++ total++; ++ } ++ ++ return ret; ++} ++#else ++static int io_sqe_files_scm(struct io_ring_ctx *ctx) ++{ ++ return 0; ++} ++#endif ++ ++static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc) ++{ ++ struct file *file = prsrc->file; ++#if defined(CONFIG_UNIX) ++ struct sock *sock = ctx->ring_sock->sk; ++ struct sk_buff_head list, *head = &sock->sk_receive_queue; ++ struct sk_buff *skb; ++ int i; ++ ++ __skb_queue_head_init(&list); ++ ++ /* ++ * Find the skb that holds this file in its SCM_RIGHTS. When found, ++ * remove this entry and rearrange the file array. ++ */ ++ skb = skb_dequeue(head); ++ while (skb) { ++ struct scm_fp_list *fp; ++ ++ fp = UNIXCB(skb).fp; ++ for (i = 0; i < fp->count; i++) { ++ int left; ++ ++ if (fp->fp[i] != file) ++ continue; ++ ++ unix_notinflight(fp->user, fp->fp[i]); ++ left = fp->count - 1 - i; ++ if (left) { ++ memmove(&fp->fp[i], &fp->fp[i + 1], ++ left * sizeof(struct file *)); ++ } ++ fp->count--; ++ if (!fp->count) { ++ kfree_skb(skb); ++ skb = NULL; ++ } else { ++ __skb_queue_tail(&list, skb); ++ } ++ fput(file); ++ file = NULL; ++ break; ++ } ++ ++ if (!file) ++ break; ++ ++ __skb_queue_tail(&list, skb); ++ ++ skb = skb_dequeue(head); ++ } ++ ++ if (skb_peek(&list)) { ++ spin_lock_irq(&head->lock); ++ while ((skb = __skb_dequeue(&list)) != NULL) ++ __skb_queue_tail(head, skb); ++ spin_unlock_irq(&head->lock); ++ } ++#else ++ fput(file); ++#endif ++} ++ ++static void __io_rsrc_put_work(struct io_rsrc_node *ref_node) ++{ ++ struct io_rsrc_data *rsrc_data = ref_node->rsrc_data; ++ struct io_ring_ctx *ctx = rsrc_data->ctx; ++ struct io_rsrc_put *prsrc, *tmp; ++ ++ list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) { ++ list_del(&prsrc->list); ++ ++ if (prsrc->tag) { ++ bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL; ++ ++ io_ring_submit_lock(ctx, lock_ring); ++ spin_lock(&ctx->completion_lock); ++ io_fill_cqe_aux(ctx, prsrc->tag, 0, 0); ++ io_commit_cqring(ctx); ++ spin_unlock(&ctx->completion_lock); ++ io_cqring_ev_posted(ctx); ++ io_ring_submit_unlock(ctx, lock_ring); ++ } ++ ++ rsrc_data->do_put(ctx, prsrc); ++ kfree(prsrc); ++ } ++ ++ io_rsrc_node_destroy(ref_node); ++ if (atomic_dec_and_test(&rsrc_data->refs)) ++ complete(&rsrc_data->done); ++} ++ ++static void io_rsrc_put_work(struct work_struct *work) ++{ ++ struct io_ring_ctx *ctx; ++ struct llist_node *node; ++ ++ ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work); ++ node = llist_del_all(&ctx->rsrc_put_llist); ++ ++ while (node) { ++ struct io_rsrc_node *ref_node; ++ struct llist_node *next = node->next; ++ ++ ref_node = llist_entry(node, struct io_rsrc_node, llist); ++ __io_rsrc_put_work(ref_node); ++ node = next; ++ } ++} ++ ++static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, ++ unsigned nr_args, u64 __user *tags) ++{ ++ __s32 __user *fds = (__s32 __user *) arg; ++ struct file *file; ++ int fd, ret; ++ unsigned i; ++ ++ if (ctx->file_data) ++ return -EBUSY; ++ if (!nr_args) ++ return -EINVAL; ++ if (nr_args > IORING_MAX_FIXED_FILES) ++ return -EMFILE; ++ if (nr_args > rlimit(RLIMIT_NOFILE)) ++ return -EMFILE; ++ ret = io_rsrc_node_switch_start(ctx); ++ if (ret) ++ return ret; ++ ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args, ++ &ctx->file_data); ++ if (ret) ++ return ret; ++ ++ ret = -ENOMEM; ++ if (!io_alloc_file_tables(&ctx->file_table, nr_args)) ++ goto out_free; ++ ++ for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { ++ if (copy_from_user(&fd, &fds[i], sizeof(fd))) { ++ ret = -EFAULT; ++ goto out_fput; ++ } ++ /* allow sparse sets */ ++ if (fd == -1) { ++ ret = -EINVAL; ++ if (unlikely(*io_get_tag_slot(ctx->file_data, i))) ++ goto out_fput; ++ continue; ++ } ++ ++ file = fget(fd); ++ ret = -EBADF; ++ if (unlikely(!file)) ++ goto out_fput; ++ ++ /* ++ * Don't allow io_uring instances to be registered. If UNIX ++ * isn't enabled, then this causes a reference cycle and this ++ * instance can never get freed. If UNIX is enabled we'll ++ * handle it just fine, but there's still no point in allowing ++ * a ring fd as it doesn't support regular read/write anyway. ++ */ ++ if (file->f_op == &io_uring_fops) { ++ fput(file); ++ goto out_fput; ++ } ++ io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file); ++ } ++ ++ ret = io_sqe_files_scm(ctx); ++ if (ret) { ++ __io_sqe_files_unregister(ctx); ++ return ret; ++ } ++ ++ io_rsrc_node_switch(ctx, NULL); ++ return ret; ++out_fput: ++ for (i = 0; i < ctx->nr_user_files; i++) { ++ file = io_file_from_index(ctx, i); ++ if (file) ++ fput(file); ++ } ++ io_free_file_tables(&ctx->file_table); ++ ctx->nr_user_files = 0; ++out_free: ++ io_rsrc_data_free(ctx->file_data); ++ ctx->file_data = NULL; ++ return ret; ++} ++ ++static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file, ++ int index) ++{ ++#if defined(CONFIG_UNIX) ++ struct sock *sock = ctx->ring_sock->sk; ++ struct sk_buff_head *head = &sock->sk_receive_queue; ++ struct sk_buff *skb; ++ ++ /* ++ * See if we can merge this file into an existing skb SCM_RIGHTS ++ * file set. If there's no room, fall back to allocating a new skb ++ * and filling it in. ++ */ ++ spin_lock_irq(&head->lock); ++ skb = skb_peek(head); ++ if (skb) { ++ struct scm_fp_list *fpl = UNIXCB(skb).fp; ++ ++ if (fpl->count < SCM_MAX_FD) { ++ __skb_unlink(skb, head); ++ spin_unlock_irq(&head->lock); ++ fpl->fp[fpl->count] = get_file(file); ++ unix_inflight(fpl->user, fpl->fp[fpl->count]); ++ fpl->count++; ++ spin_lock_irq(&head->lock); ++ __skb_queue_head(head, skb); ++ } else { ++ skb = NULL; ++ } ++ } ++ spin_unlock_irq(&head->lock); ++ ++ if (skb) { ++ fput(file); ++ return 0; ++ } ++ ++ return __io_sqe_files_scm(ctx, 1, index); ++#else ++ return 0; ++#endif ++} ++ ++static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, ++ struct io_rsrc_node *node, void *rsrc) ++{ ++ u64 *tag_slot = io_get_tag_slot(data, idx); ++ struct io_rsrc_put *prsrc; ++ ++ prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL); ++ if (!prsrc) ++ return -ENOMEM; ++ ++ prsrc->tag = *tag_slot; ++ *tag_slot = 0; ++ prsrc->rsrc = rsrc; ++ list_add(&prsrc->list, &node->rsrc_list); ++ return 0; ++} ++ ++static int io_install_fixed_file(struct io_kiocb *req, struct file *file, ++ unsigned int issue_flags, u32 slot_index) ++{ ++ struct io_ring_ctx *ctx = req->ctx; ++ bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ++ bool needs_switch = false; ++ struct io_fixed_file *file_slot; ++ int ret = -EBADF; ++ ++ io_ring_submit_lock(ctx, !force_nonblock); ++ if (file->f_op == &io_uring_fops) ++ goto err; ++ ret = -ENXIO; ++ if (!ctx->file_data) ++ goto err; ++ ret = -EINVAL; ++ if (slot_index >= ctx->nr_user_files) ++ goto err; ++ ++ slot_index = array_index_nospec(slot_index, ctx->nr_user_files); ++ file_slot = io_fixed_file_slot(&ctx->file_table, slot_index); ++ ++ if (file_slot->file_ptr) { ++ struct file *old_file; ++ ++ ret = io_rsrc_node_switch_start(ctx); ++ if (ret) ++ goto err; ++ ++ old_file = (struct file *)(file_slot->file_ptr & FFS_MASK); ++ ret = io_queue_rsrc_removal(ctx->file_data, slot_index, ++ ctx->rsrc_node, old_file); ++ if (ret) ++ goto err; ++ file_slot->file_ptr = 0; ++ needs_switch = true; ++ } ++ ++ *io_get_tag_slot(ctx->file_data, slot_index) = 0; ++ io_fixed_file_set(file_slot, file); ++ ret = io_sqe_file_register(ctx, file, slot_index); ++ if (ret) { ++ file_slot->file_ptr = 0; ++ goto err; ++ } ++ ++ ret = 0; ++err: ++ if (needs_switch) ++ io_rsrc_node_switch(ctx, ctx->file_data); ++ io_ring_submit_unlock(ctx, !force_nonblock); ++ if (ret) ++ fput(file); ++ return ret; ++} ++ ++static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) ++{ ++ unsigned int offset = req->close.file_slot - 1; ++ struct io_ring_ctx *ctx = req->ctx; ++ struct io_fixed_file *file_slot; ++ struct file *file; ++ int ret; ++ ++ io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); ++ ret = -ENXIO; ++ if (unlikely(!ctx->file_data)) ++ goto out; ++ ret = -EINVAL; ++ if (offset >= ctx->nr_user_files) ++ goto out; ++ ret = io_rsrc_node_switch_start(ctx); ++ if (ret) ++ goto out; ++ ++ offset = array_index_nospec(offset, ctx->nr_user_files); ++ file_slot = io_fixed_file_slot(&ctx->file_table, offset); ++ ret = -EBADF; ++ if (!file_slot->file_ptr) ++ goto out; ++ ++ file = (struct file *)(file_slot->file_ptr & FFS_MASK); ++ ret = io_queue_rsrc_removal(ctx->file_data, offset, ctx->rsrc_node, file); ++ if (ret) ++ goto out; ++ ++ file_slot->file_ptr = 0; ++ io_rsrc_node_switch(ctx, ctx->file_data); ++ ret = 0; ++out: ++ io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); ++ return ret; ++} ++ ++static int __io_sqe_files_update(struct io_ring_ctx *ctx, ++ struct io_uring_rsrc_update2 *up, ++ unsigned nr_args) ++{ ++ u64 __user *tags = u64_to_user_ptr(up->tags); ++ __s32 __user *fds = u64_to_user_ptr(up->data); ++ struct io_rsrc_data *data = ctx->file_data; ++ struct io_fixed_file *file_slot; ++ struct file *file; ++ int fd, i, err = 0; ++ unsigned int done; ++ bool needs_switch = false; ++ ++ if (!ctx->file_data) ++ return -ENXIO; ++ if (up->offset + nr_args > ctx->nr_user_files) ++ return -EINVAL; ++ ++ for (done = 0; done < nr_args; done++) { ++ u64 tag = 0; ++ ++ if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) || ++ copy_from_user(&fd, &fds[done], sizeof(fd))) { ++ err = -EFAULT; ++ break; ++ } ++ if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) { ++ err = -EINVAL; ++ break; ++ } ++ if (fd == IORING_REGISTER_FILES_SKIP) ++ continue; ++ ++ i = array_index_nospec(up->offset + done, ctx->nr_user_files); ++ file_slot = io_fixed_file_slot(&ctx->file_table, i); ++ ++ if (file_slot->file_ptr) { ++ file = (struct file *)(file_slot->file_ptr & FFS_MASK); ++ err = io_queue_rsrc_removal(data, i, ctx->rsrc_node, file); ++ if (err) ++ break; ++ file_slot->file_ptr = 0; ++ needs_switch = true; ++ } ++ if (fd != -1) { ++ file = fget(fd); ++ if (!file) { ++ err = -EBADF; ++ break; ++ } ++ /* ++ * Don't allow io_uring instances to be registered. If ++ * UNIX isn't enabled, then this causes a reference ++ * cycle and this instance can never get freed. If UNIX ++ * is enabled we'll handle it just fine, but there's ++ * still no point in allowing a ring fd as it doesn't ++ * support regular read/write anyway. ++ */ ++ if (file->f_op == &io_uring_fops) { ++ fput(file); ++ err = -EBADF; ++ break; ++ } ++ *io_get_tag_slot(data, i) = tag; ++ io_fixed_file_set(file_slot, file); ++ err = io_sqe_file_register(ctx, file, i); ++ if (err) { ++ file_slot->file_ptr = 0; ++ fput(file); ++ break; ++ } ++ } ++ } ++ ++ if (needs_switch) ++ io_rsrc_node_switch(ctx, data); ++ return done ? done : err; ++} ++ ++static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx, ++ struct task_struct *task) ++{ ++ struct io_wq_hash *hash; ++ struct io_wq_data data; ++ unsigned int concurrency; ++ ++ mutex_lock(&ctx->uring_lock); ++ hash = ctx->hash_map; ++ if (!hash) { ++ hash = kzalloc(sizeof(*hash), GFP_KERNEL); ++ if (!hash) { ++ mutex_unlock(&ctx->uring_lock); ++ return ERR_PTR(-ENOMEM); ++ } ++ refcount_set(&hash->refs, 1); ++ init_waitqueue_head(&hash->wait); ++ ctx->hash_map = hash; ++ } ++ mutex_unlock(&ctx->uring_lock); ++ ++ data.hash = hash; ++ data.task = task; ++ data.free_work = io_wq_free_work; ++ data.do_work = io_wq_submit_work; ++ ++ /* Do QD, or 4 * CPUS, whatever is smallest */ ++ concurrency = min(ctx->sq_entries, 4 * num_online_cpus()); ++ ++ return io_wq_create(concurrency, &data); ++} ++ ++static int io_uring_alloc_task_context(struct task_struct *task, ++ struct io_ring_ctx *ctx) ++{ ++ struct io_uring_task *tctx; ++ int ret; ++ ++ tctx = kzalloc(sizeof(*tctx), GFP_KERNEL); ++ if (unlikely(!tctx)) ++ return -ENOMEM; ++ ++ ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL); ++ if (unlikely(ret)) { ++ kfree(tctx); ++ return ret; ++ } ++ ++ tctx->io_wq = io_init_wq_offload(ctx, task); ++ if (IS_ERR(tctx->io_wq)) { ++ ret = PTR_ERR(tctx->io_wq); ++ percpu_counter_destroy(&tctx->inflight); ++ kfree(tctx); ++ return ret; ++ } ++ ++ xa_init(&tctx->xa); ++ init_waitqueue_head(&tctx->wait); ++ atomic_set(&tctx->in_idle, 0); ++ atomic_set(&tctx->inflight_tracked, 0); ++ task->io_uring = tctx; ++ spin_lock_init(&tctx->task_lock); ++ INIT_WQ_LIST(&tctx->task_list); ++ init_task_work(&tctx->task_work, tctx_task_work); ++ return 0; ++} ++ ++void __io_uring_free(struct task_struct *tsk) ++{ ++ struct io_uring_task *tctx = tsk->io_uring; ++ ++ WARN_ON_ONCE(!xa_empty(&tctx->xa)); ++ WARN_ON_ONCE(tctx->io_wq); ++ WARN_ON_ONCE(tctx->cached_refs); ++ ++ percpu_counter_destroy(&tctx->inflight); ++ kfree(tctx); ++ tsk->io_uring = NULL; ++} ++ ++static int io_sq_offload_create(struct io_ring_ctx *ctx, ++ struct io_uring_params *p) ++{ ++ int ret; ++ ++ /* Retain compatibility with failing for an invalid attach attempt */ ++ if ((ctx->flags & (IORING_SETUP_ATTACH_WQ | IORING_SETUP_SQPOLL)) == ++ IORING_SETUP_ATTACH_WQ) { ++ struct fd f; ++ ++ f = fdget(p->wq_fd); ++ if (!f.file) ++ return -ENXIO; ++ if (f.file->f_op != &io_uring_fops) { ++ fdput(f); ++ return -EINVAL; ++ } ++ fdput(f); ++ } ++ if (ctx->flags & IORING_SETUP_SQPOLL) { ++ struct task_struct *tsk; ++ struct io_sq_data *sqd; ++ bool attached; ++ ++ sqd = io_get_sq_data(p, &attached); ++ if (IS_ERR(sqd)) { ++ ret = PTR_ERR(sqd); ++ goto err; ++ } ++ ++ ctx->sq_creds = get_current_cred(); ++ ctx->sq_data = sqd; ++ ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle); ++ if (!ctx->sq_thread_idle) ++ ctx->sq_thread_idle = HZ; ++ ++ io_sq_thread_park(sqd); ++ list_add(&ctx->sqd_list, &sqd->ctx_list); ++ io_sqd_update_thread_idle(sqd); ++ /* don't attach to a dying SQPOLL thread, would be racy */ ++ ret = (attached && !sqd->thread) ? -ENXIO : 0; ++ io_sq_thread_unpark(sqd); ++ ++ if (ret < 0) ++ goto err; ++ if (attached) ++ return 0; ++ ++ if (p->flags & IORING_SETUP_SQ_AFF) { ++ int cpu = p->sq_thread_cpu; ++ ++ ret = -EINVAL; ++ if (cpu >= nr_cpu_ids || !cpu_online(cpu)) ++ goto err_sqpoll; ++ sqd->sq_cpu = cpu; ++ } else { ++ sqd->sq_cpu = -1; ++ } ++ ++ sqd->task_pid = current->pid; ++ sqd->task_tgid = current->tgid; ++ tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE); ++ if (IS_ERR(tsk)) { ++ ret = PTR_ERR(tsk); ++ goto err_sqpoll; ++ } ++ ++ sqd->thread = tsk; ++ ret = io_uring_alloc_task_context(tsk, ctx); ++ wake_up_new_task(tsk); ++ if (ret) ++ goto err; ++ } else if (p->flags & IORING_SETUP_SQ_AFF) { ++ /* Can't have SQ_AFF without SQPOLL */ ++ ret = -EINVAL; ++ goto err; ++ } ++ ++ return 0; ++err_sqpoll: ++ complete(&ctx->sq_data->exited); ++err: ++ io_sq_thread_finish(ctx); ++ return ret; ++} ++ ++static inline void __io_unaccount_mem(struct user_struct *user, ++ unsigned long nr_pages) ++{ ++ atomic_long_sub(nr_pages, &user->locked_vm); ++} ++ ++static inline int __io_account_mem(struct user_struct *user, ++ unsigned long nr_pages) ++{ ++ unsigned long page_limit, cur_pages, new_pages; ++ ++ /* Don't allow more pages than we can safely lock */ ++ page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; ++ ++ do { ++ cur_pages = atomic_long_read(&user->locked_vm); ++ new_pages = cur_pages + nr_pages; ++ if (new_pages > page_limit) ++ return -ENOMEM; ++ } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages, ++ new_pages) != cur_pages); ++ ++ return 0; ++} ++ ++static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages) ++{ ++ if (ctx->user) ++ __io_unaccount_mem(ctx->user, nr_pages); ++ ++ if (ctx->mm_account) ++ atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm); ++} ++ ++static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages) ++{ ++ int ret; ++ ++ if (ctx->user) { ++ ret = __io_account_mem(ctx->user, nr_pages); ++ if (ret) ++ return ret; ++ } ++ ++ if (ctx->mm_account) ++ atomic64_add(nr_pages, &ctx->mm_account->pinned_vm); ++ ++ return 0; ++} ++ ++static void io_mem_free(void *ptr) ++{ ++ struct page *page; ++ ++ if (!ptr) ++ return; ++ ++ page = virt_to_head_page(ptr); ++ if (put_page_testzero(page)) ++ free_compound_page(page); ++} ++ ++static void *io_mem_alloc(size_t size) ++{ ++ gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP; ++ ++ return (void *) __get_free_pages(gfp, get_order(size)); ++} ++ ++static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries, ++ size_t *sq_offset) ++{ ++ struct io_rings *rings; ++ size_t off, sq_array_size; ++ ++ off = struct_size(rings, cqes, cq_entries); ++ if (off == SIZE_MAX) ++ return SIZE_MAX; ++ ++#ifdef CONFIG_SMP ++ off = ALIGN(off, SMP_CACHE_BYTES); ++ if (off == 0) ++ return SIZE_MAX; ++#endif ++ ++ if (sq_offset) ++ *sq_offset = off; ++ ++ sq_array_size = array_size(sizeof(u32), sq_entries); ++ if (sq_array_size == SIZE_MAX) ++ return SIZE_MAX; ++ ++ if (check_add_overflow(off, sq_array_size, &off)) ++ return SIZE_MAX; ++ ++ return off; ++} ++ ++static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot) ++{ ++ struct io_mapped_ubuf *imu = *slot; ++ unsigned int i; ++ ++ if (imu != ctx->dummy_ubuf) { ++ for (i = 0; i < imu->nr_bvecs; i++) ++ unpin_user_page(imu->bvec[i].bv_page); ++ if (imu->acct_pages) ++ io_unaccount_mem(ctx, imu->acct_pages); ++ kvfree(imu); ++ } ++ *slot = NULL; ++} ++ ++static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc) ++{ ++ io_buffer_unmap(ctx, &prsrc->buf); ++ prsrc->buf = NULL; ++} ++ ++static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < ctx->nr_user_bufs; i++) ++ io_buffer_unmap(ctx, &ctx->user_bufs[i]); ++ kfree(ctx->user_bufs); ++ io_rsrc_data_free(ctx->buf_data); ++ ctx->user_bufs = NULL; ++ ctx->buf_data = NULL; ++ ctx->nr_user_bufs = 0; ++} ++ ++static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) ++{ ++ unsigned nr = ctx->nr_user_bufs; ++ int ret; ++ ++ if (!ctx->buf_data) ++ return -ENXIO; ++ ++ /* ++ * Quiesce may unlock ->uring_lock, and while it's not held ++ * prevent new requests using the table. ++ */ ++ ctx->nr_user_bufs = 0; ++ ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx); ++ ctx->nr_user_bufs = nr; ++ if (!ret) ++ __io_sqe_buffers_unregister(ctx); ++ return ret; ++} ++ ++static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst, ++ void __user *arg, unsigned index) ++{ ++ struct iovec __user *src; ++ ++#ifdef CONFIG_COMPAT ++ if (ctx->compat) { ++ struct compat_iovec __user *ciovs; ++ struct compat_iovec ciov; ++ ++ ciovs = (struct compat_iovec __user *) arg; ++ if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov))) ++ return -EFAULT; ++ ++ dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base); ++ dst->iov_len = ciov.iov_len; ++ return 0; ++ } ++#endif ++ src = (struct iovec __user *) arg; ++ if (copy_from_user(dst, &src[index], sizeof(*dst))) ++ return -EFAULT; ++ return 0; ++} ++ ++/* ++ * Not super efficient, but this is just a registration time. And we do cache ++ * the last compound head, so generally we'll only do a full search if we don't ++ * match that one. ++ * ++ * We check if the given compound head page has already been accounted, to ++ * avoid double accounting it. This allows us to account the full size of the ++ * page, not just the constituent pages of a huge page. ++ */ ++static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages, ++ int nr_pages, struct page *hpage) ++{ ++ int i, j; ++ ++ /* check current page array */ ++ for (i = 0; i < nr_pages; i++) { ++ if (!PageCompound(pages[i])) ++ continue; ++ if (compound_head(pages[i]) == hpage) ++ return true; ++ } ++ ++ /* check previously registered pages */ ++ for (i = 0; i < ctx->nr_user_bufs; i++) { ++ struct io_mapped_ubuf *imu = ctx->user_bufs[i]; ++ ++ for (j = 0; j < imu->nr_bvecs; j++) { ++ if (!PageCompound(imu->bvec[j].bv_page)) ++ continue; ++ if (compound_head(imu->bvec[j].bv_page) == hpage) ++ return true; ++ } ++ } ++ ++ return false; ++} ++ ++static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages, ++ int nr_pages, struct io_mapped_ubuf *imu, ++ struct page **last_hpage) ++{ ++ int i, ret; ++ ++ imu->acct_pages = 0; ++ for (i = 0; i < nr_pages; i++) { ++ if (!PageCompound(pages[i])) { ++ imu->acct_pages++; ++ } else { ++ struct page *hpage; ++ ++ hpage = compound_head(pages[i]); ++ if (hpage == *last_hpage) ++ continue; ++ *last_hpage = hpage; ++ if (headpage_already_acct(ctx, pages, i, hpage)) ++ continue; ++ imu->acct_pages += page_size(hpage) >> PAGE_SHIFT; ++ } ++ } ++ ++ if (!imu->acct_pages) ++ return 0; ++ ++ ret = io_account_mem(ctx, imu->acct_pages); ++ if (ret) ++ imu->acct_pages = 0; ++ return ret; ++} ++ ++static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, ++ struct io_mapped_ubuf **pimu, ++ struct page **last_hpage) ++{ ++ struct io_mapped_ubuf *imu = NULL; ++ struct vm_area_struct **vmas = NULL; ++ struct page **pages = NULL; ++ unsigned long off, start, end, ubuf; ++ size_t size; ++ int ret, pret, nr_pages, i; ++ ++ if (!iov->iov_base) { ++ *pimu = ctx->dummy_ubuf; ++ return 0; ++ } ++ ++ ubuf = (unsigned long) iov->iov_base; ++ end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT; ++ start = ubuf >> PAGE_SHIFT; ++ nr_pages = end - start; ++ ++ *pimu = NULL; ++ ret = -ENOMEM; ++ ++ pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL); ++ if (!pages) ++ goto done; ++ ++ vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *), ++ GFP_KERNEL); ++ if (!vmas) ++ goto done; ++ ++ imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL); ++ if (!imu) ++ goto done; ++ ++ ret = 0; ++ mmap_read_lock(current->mm); ++ pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM, ++ pages, vmas); ++ if (pret == nr_pages) { ++ /* don't support file backed memory */ ++ for (i = 0; i < nr_pages; i++) { ++ struct vm_area_struct *vma = vmas[i]; ++ ++ if (vma_is_shmem(vma)) ++ continue; ++ if (vma->vm_file && ++ !is_file_hugepages(vma->vm_file)) { ++ ret = -EOPNOTSUPP; ++ break; ++ } ++ } ++ } else { ++ ret = pret < 0 ? pret : -EFAULT; ++ } ++ mmap_read_unlock(current->mm); ++ if (ret) { ++ /* ++ * if we did partial map, or found file backed vmas, ++ * release any pages we did get ++ */ ++ if (pret > 0) ++ unpin_user_pages(pages, pret); ++ goto done; ++ } ++ ++ ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage); ++ if (ret) { ++ unpin_user_pages(pages, pret); ++ goto done; ++ } ++ ++ off = ubuf & ~PAGE_MASK; ++ size = iov->iov_len; ++ for (i = 0; i < nr_pages; i++) { ++ size_t vec_len; ++ ++ vec_len = min_t(size_t, size, PAGE_SIZE - off); ++ imu->bvec[i].bv_page = pages[i]; ++ imu->bvec[i].bv_len = vec_len; ++ imu->bvec[i].bv_offset = off; ++ off = 0; ++ size -= vec_len; ++ } ++ /* store original address for later verification */ ++ imu->ubuf = ubuf; ++ imu->ubuf_end = ubuf + iov->iov_len; ++ imu->nr_bvecs = nr_pages; ++ *pimu = imu; ++ ret = 0; ++done: ++ if (ret) ++ kvfree(imu); ++ kvfree(pages); ++ kvfree(vmas); ++ return ret; ++} ++ ++static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args) ++{ ++ ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL); ++ return ctx->user_bufs ? 0 : -ENOMEM; ++} ++ ++static int io_buffer_validate(struct iovec *iov) ++{ ++ unsigned long tmp, acct_len = iov->iov_len + (PAGE_SIZE - 1); ++ ++ /* ++ * Don't impose further limits on the size and buffer ++ * constraints here, we'll -EINVAL later when IO is ++ * submitted if they are wrong. ++ */ ++ if (!iov->iov_base) ++ return iov->iov_len ? -EFAULT : 0; ++ if (!iov->iov_len) ++ return -EFAULT; ++ ++ /* arbitrary limit, but we need something */ ++ if (iov->iov_len > SZ_1G) ++ return -EFAULT; ++ ++ if (check_add_overflow((unsigned long)iov->iov_base, acct_len, &tmp)) ++ return -EOVERFLOW; ++ ++ return 0; ++} ++ ++static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, ++ unsigned int nr_args, u64 __user *tags) ++{ ++ struct page *last_hpage = NULL; ++ struct io_rsrc_data *data; ++ int i, ret; ++ struct iovec iov; ++ ++ if (ctx->user_bufs) ++ return -EBUSY; ++ if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS) ++ return -EINVAL; ++ ret = io_rsrc_node_switch_start(ctx); ++ if (ret) ++ return ret; ++ ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data); ++ if (ret) ++ return ret; ++ ret = io_buffers_map_alloc(ctx, nr_args); ++ if (ret) { ++ io_rsrc_data_free(data); ++ return ret; ++ } ++ ++ for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) { ++ ret = io_copy_iov(ctx, &iov, arg, i); ++ if (ret) ++ break; ++ ret = io_buffer_validate(&iov); ++ if (ret) ++ break; ++ if (!iov.iov_base && *io_get_tag_slot(data, i)) { ++ ret = -EINVAL; ++ break; ++ } ++ ++ ret = io_sqe_buffer_register(ctx, &iov, &ctx->user_bufs[i], ++ &last_hpage); ++ if (ret) ++ break; ++ } ++ ++ WARN_ON_ONCE(ctx->buf_data); ++ ++ ctx->buf_data = data; ++ if (ret) ++ __io_sqe_buffers_unregister(ctx); ++ else ++ io_rsrc_node_switch(ctx, NULL); ++ return ret; ++} ++ ++static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, ++ struct io_uring_rsrc_update2 *up, ++ unsigned int nr_args) ++{ ++ u64 __user *tags = u64_to_user_ptr(up->tags); ++ struct iovec iov, __user *iovs = u64_to_user_ptr(up->data); ++ struct page *last_hpage = NULL; ++ bool needs_switch = false; ++ __u32 done; ++ int i, err; ++ ++ if (!ctx->buf_data) ++ return -ENXIO; ++ if (up->offset + nr_args > ctx->nr_user_bufs) ++ return -EINVAL; ++ ++ for (done = 0; done < nr_args; done++) { ++ struct io_mapped_ubuf *imu; ++ int offset = up->offset + done; ++ u64 tag = 0; ++ ++ err = io_copy_iov(ctx, &iov, iovs, done); ++ if (err) ++ break; ++ if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) { ++ err = -EFAULT; ++ break; ++ } ++ err = io_buffer_validate(&iov); ++ if (err) ++ break; ++ if (!iov.iov_base && tag) { ++ err = -EINVAL; ++ break; ++ } ++ err = io_sqe_buffer_register(ctx, &iov, &imu, &last_hpage); ++ if (err) ++ break; ++ ++ i = array_index_nospec(offset, ctx->nr_user_bufs); ++ if (ctx->user_bufs[i] != ctx->dummy_ubuf) { ++ err = io_queue_rsrc_removal(ctx->buf_data, i, ++ ctx->rsrc_node, ctx->user_bufs[i]); ++ if (unlikely(err)) { ++ io_buffer_unmap(ctx, &imu); ++ break; ++ } ++ ctx->user_bufs[i] = NULL; ++ needs_switch = true; ++ } ++ ++ ctx->user_bufs[i] = imu; ++ *io_get_tag_slot(ctx->buf_data, offset) = tag; ++ } ++ ++ if (needs_switch) ++ io_rsrc_node_switch(ctx, ctx->buf_data); ++ return done ? done : err; ++} ++ ++static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg) ++{ ++ __s32 __user *fds = arg; ++ int fd; ++ ++ if (ctx->cq_ev_fd) ++ return -EBUSY; ++ ++ if (copy_from_user(&fd, fds, sizeof(*fds))) ++ return -EFAULT; ++ ++ ctx->cq_ev_fd = eventfd_ctx_fdget(fd); ++ if (IS_ERR(ctx->cq_ev_fd)) { ++ int ret = PTR_ERR(ctx->cq_ev_fd); ++ ++ ctx->cq_ev_fd = NULL; ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int io_eventfd_unregister(struct io_ring_ctx *ctx) ++{ ++ if (ctx->cq_ev_fd) { ++ eventfd_ctx_put(ctx->cq_ev_fd); ++ ctx->cq_ev_fd = NULL; ++ return 0; ++ } ++ ++ return -ENXIO; ++} ++ ++static void io_destroy_buffers(struct io_ring_ctx *ctx) ++{ ++ struct io_buffer *buf; ++ unsigned long index; ++ ++ xa_for_each(&ctx->io_buffers, index, buf) ++ __io_remove_buffers(ctx, buf, index, -1U); ++} ++ ++static void io_req_cache_free(struct list_head *list) ++{ ++ struct io_kiocb *req, *nxt; ++ ++ list_for_each_entry_safe(req, nxt, list, inflight_entry) { ++ list_del(&req->inflight_entry); ++ kmem_cache_free(req_cachep, req); ++ } ++} ++ ++static void io_req_caches_free(struct io_ring_ctx *ctx) ++{ ++ struct io_submit_state *state = &ctx->submit_state; ++ ++ mutex_lock(&ctx->uring_lock); ++ ++ if (state->free_reqs) { ++ kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs); ++ state->free_reqs = 0; ++ } ++ ++ io_flush_cached_locked_reqs(ctx, state); ++ io_req_cache_free(&state->free_list); ++ mutex_unlock(&ctx->uring_lock); ++} ++ ++static void io_wait_rsrc_data(struct io_rsrc_data *data) ++{ ++ if (data && !atomic_dec_and_test(&data->refs)) ++ wait_for_completion(&data->done); ++} ++ ++static void io_ring_ctx_free(struct io_ring_ctx *ctx) ++{ ++ io_sq_thread_finish(ctx); ++ ++ /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */ ++ io_wait_rsrc_data(ctx->buf_data); ++ io_wait_rsrc_data(ctx->file_data); ++ ++ mutex_lock(&ctx->uring_lock); ++ if (ctx->buf_data) ++ __io_sqe_buffers_unregister(ctx); ++ if (ctx->file_data) ++ __io_sqe_files_unregister(ctx); ++ if (ctx->rings) ++ __io_cqring_overflow_flush(ctx, true); ++ mutex_unlock(&ctx->uring_lock); ++ io_eventfd_unregister(ctx); ++ io_destroy_buffers(ctx); ++ if (ctx->sq_creds) ++ put_cred(ctx->sq_creds); ++ ++ /* there are no registered resources left, nobody uses it */ ++ if (ctx->rsrc_node) ++ io_rsrc_node_destroy(ctx->rsrc_node); ++ if (ctx->rsrc_backup_node) ++ io_rsrc_node_destroy(ctx->rsrc_backup_node); ++ flush_delayed_work(&ctx->rsrc_put_work); ++ ++ WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list)); ++ WARN_ON_ONCE(!llist_empty(&ctx->rsrc_put_llist)); ++ ++#if defined(CONFIG_UNIX) ++ if (ctx->ring_sock) { ++ ctx->ring_sock->file = NULL; /* so that iput() is called */ ++ sock_release(ctx->ring_sock); ++ } ++#endif ++ WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list)); ++ ++ if (ctx->mm_account) { ++ mmdrop(ctx->mm_account); ++ ctx->mm_account = NULL; ++ } ++ ++ io_mem_free(ctx->rings); ++ io_mem_free(ctx->sq_sqes); ++ ++ percpu_ref_exit(&ctx->refs); ++ free_uid(ctx->user); ++ io_req_caches_free(ctx); ++ if (ctx->hash_map) ++ io_wq_put_hash(ctx->hash_map); ++ kfree(ctx->cancel_hash); ++ kfree(ctx->dummy_ubuf); ++ kfree(ctx); ++} ++ ++static __poll_t io_uring_poll(struct file *file, poll_table *wait) ++{ ++ struct io_ring_ctx *ctx = file->private_data; ++ __poll_t mask = 0; ++ ++ poll_wait(file, &ctx->poll_wait, wait); ++ /* ++ * synchronizes with barrier from wq_has_sleeper call in ++ * io_commit_cqring ++ */ ++ smp_rmb(); ++ if (!io_sqring_full(ctx)) ++ mask |= EPOLLOUT | EPOLLWRNORM; ++ ++ /* ++ * Don't flush cqring overflow list here, just do a simple check. ++ * Otherwise there could possible be ABBA deadlock: ++ * CPU0 CPU1 ++ * ---- ---- ++ * lock(&ctx->uring_lock); ++ * lock(&ep->mtx); ++ * lock(&ctx->uring_lock); ++ * lock(&ep->mtx); ++ * ++ * Users may get EPOLLIN meanwhile seeing nothing in cqring, this ++ * pushs them to do the flush. ++ */ ++ if (io_cqring_events(ctx) || test_bit(0, &ctx->check_cq_overflow)) ++ mask |= EPOLLIN | EPOLLRDNORM; ++ ++ return mask; ++} ++ ++static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) ++{ ++ const struct cred *creds; ++ ++ creds = xa_erase(&ctx->personalities, id); ++ if (creds) { ++ put_cred(creds); ++ return 0; ++ } ++ ++ return -EINVAL; ++} ++ ++struct io_tctx_exit { ++ struct callback_head task_work; ++ struct completion completion; ++ struct io_ring_ctx *ctx; ++}; ++ ++static void io_tctx_exit_cb(struct callback_head *cb) ++{ ++ struct io_uring_task *tctx = current->io_uring; ++ struct io_tctx_exit *work; ++ ++ work = container_of(cb, struct io_tctx_exit, task_work); ++ /* ++ * When @in_idle, we're in cancellation and it's racy to remove the ++ * node. It'll be removed by the end of cancellation, just ignore it. ++ * tctx can be NULL if the queueing of this task_work raced with ++ * work cancelation off the exec path. ++ */ ++ if (tctx && !atomic_read(&tctx->in_idle)) ++ io_uring_del_tctx_node((unsigned long)work->ctx); ++ complete(&work->completion); ++} ++ ++static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data) ++{ ++ struct io_kiocb *req = container_of(work, struct io_kiocb, work); ++ ++ return req->ctx == data; ++} ++ ++static void io_ring_exit_work(struct work_struct *work) ++{ ++ struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work); ++ unsigned long timeout = jiffies + HZ * 60 * 5; ++ unsigned long interval = HZ / 20; ++ struct io_tctx_exit exit; ++ struct io_tctx_node *node; ++ int ret; ++ ++ /* ++ * If we're doing polled IO and end up having requests being ++ * submitted async (out-of-line), then completions can come in while ++ * we're waiting for refs to drop. We need to reap these manually, ++ * as nobody else will be looking for them. ++ */ ++ do { ++ io_uring_try_cancel_requests(ctx, NULL, true); ++ if (ctx->sq_data) { ++ struct io_sq_data *sqd = ctx->sq_data; ++ struct task_struct *tsk; ++ ++ io_sq_thread_park(sqd); ++ tsk = sqd->thread; ++ if (tsk && tsk->io_uring && tsk->io_uring->io_wq) ++ io_wq_cancel_cb(tsk->io_uring->io_wq, ++ io_cancel_ctx_cb, ctx, true); ++ io_sq_thread_unpark(sqd); ++ } ++ ++ if (WARN_ON_ONCE(time_after(jiffies, timeout))) { ++ /* there is little hope left, don't run it too often */ ++ interval = HZ * 60; ++ } ++ } while (!wait_for_completion_timeout(&ctx->ref_comp, interval)); ++ ++ init_completion(&exit.completion); ++ init_task_work(&exit.task_work, io_tctx_exit_cb); ++ exit.ctx = ctx; ++ /* ++ * Some may use context even when all refs and requests have been put, ++ * and they are free to do so while still holding uring_lock or ++ * completion_lock, see io_req_task_submit(). Apart from other work, ++ * this lock/unlock section also waits them to finish. ++ */ ++ mutex_lock(&ctx->uring_lock); ++ while (!list_empty(&ctx->tctx_list)) { ++ WARN_ON_ONCE(time_after(jiffies, timeout)); ++ ++ node = list_first_entry(&ctx->tctx_list, struct io_tctx_node, ++ ctx_node); ++ /* don't spin on a single task if cancellation failed */ ++ list_rotate_left(&ctx->tctx_list); ++ ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL); ++ if (WARN_ON_ONCE(ret)) ++ continue; ++ wake_up_process(node->task); ++ ++ mutex_unlock(&ctx->uring_lock); ++ wait_for_completion(&exit.completion); ++ mutex_lock(&ctx->uring_lock); ++ } ++ mutex_unlock(&ctx->uring_lock); ++ spin_lock(&ctx->completion_lock); ++ spin_unlock(&ctx->completion_lock); ++ ++ io_ring_ctx_free(ctx); ++} ++ ++/* Returns true if we found and killed one or more timeouts */ ++static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, ++ bool cancel_all) ++{ ++ struct io_kiocb *req, *tmp; ++ int canceled = 0; ++ ++ spin_lock(&ctx->completion_lock); ++ spin_lock_irq(&ctx->timeout_lock); ++ list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { ++ if (io_match_task(req, tsk, cancel_all)) { ++ io_kill_timeout(req, -ECANCELED); ++ canceled++; ++ } ++ } ++ spin_unlock_irq(&ctx->timeout_lock); ++ if (canceled != 0) ++ io_commit_cqring(ctx); ++ spin_unlock(&ctx->completion_lock); ++ if (canceled != 0) ++ io_cqring_ev_posted(ctx); ++ return canceled != 0; ++} ++ ++static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) ++{ ++ unsigned long index; ++ struct creds *creds; ++ ++ mutex_lock(&ctx->uring_lock); ++ percpu_ref_kill(&ctx->refs); ++ if (ctx->rings) ++ __io_cqring_overflow_flush(ctx, true); ++ xa_for_each(&ctx->personalities, index, creds) ++ io_unregister_personality(ctx, index); ++ mutex_unlock(&ctx->uring_lock); ++ ++ io_kill_timeouts(ctx, NULL, true); ++ io_poll_remove_all(ctx, NULL, true); ++ ++ /* if we failed setting up the ctx, we might not have any rings */ ++ io_iopoll_try_reap_events(ctx); ++ ++ INIT_WORK(&ctx->exit_work, io_ring_exit_work); ++ /* ++ * Use system_unbound_wq to avoid spawning tons of event kworkers ++ * if we're exiting a ton of rings at the same time. It just adds ++ * noise and overhead, there's no discernable change in runtime ++ * over using system_wq. ++ */ ++ queue_work(system_unbound_wq, &ctx->exit_work); ++} ++ ++static int io_uring_release(struct inode *inode, struct file *file) ++{ ++ struct io_ring_ctx *ctx = file->private_data; ++ ++ file->private_data = NULL; ++ io_ring_ctx_wait_and_kill(ctx); ++ return 0; ++} ++ ++struct io_task_cancel { ++ struct task_struct *task; ++ bool all; ++}; ++ ++static bool io_cancel_task_cb(struct io_wq_work *work, void *data) ++{ ++ struct io_kiocb *req = container_of(work, struct io_kiocb, work); ++ struct io_task_cancel *cancel = data; ++ ++ return io_match_task_safe(req, cancel->task, cancel->all); ++} ++ ++static bool io_cancel_defer_files(struct io_ring_ctx *ctx, ++ struct task_struct *task, bool cancel_all) ++{ ++ struct io_defer_entry *de; ++ LIST_HEAD(list); ++ ++ spin_lock(&ctx->completion_lock); ++ list_for_each_entry_reverse(de, &ctx->defer_list, list) { ++ if (io_match_task_safe(de->req, task, cancel_all)) { ++ list_cut_position(&list, &ctx->defer_list, &de->list); ++ break; ++ } ++ } ++ spin_unlock(&ctx->completion_lock); ++ if (list_empty(&list)) ++ return false; ++ ++ while (!list_empty(&list)) { ++ de = list_first_entry(&list, struct io_defer_entry, list); ++ list_del_init(&de->list); ++ io_req_complete_failed(de->req, -ECANCELED); ++ kfree(de); ++ } ++ return true; ++} ++ ++static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx) ++{ ++ struct io_tctx_node *node; ++ enum io_wq_cancel cret; ++ bool ret = false; ++ ++ mutex_lock(&ctx->uring_lock); ++ list_for_each_entry(node, &ctx->tctx_list, ctx_node) { ++ struct io_uring_task *tctx = node->task->io_uring; ++ ++ /* ++ * io_wq will stay alive while we hold uring_lock, because it's ++ * killed after ctx nodes, which requires to take the lock. ++ */ ++ if (!tctx || !tctx->io_wq) ++ continue; ++ cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true); ++ ret |= (cret != IO_WQ_CANCEL_NOTFOUND); ++ } ++ mutex_unlock(&ctx->uring_lock); ++ ++ return ret; ++} ++ ++static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, ++ struct task_struct *task, ++ bool cancel_all) ++{ ++ struct io_task_cancel cancel = { .task = task, .all = cancel_all, }; ++ struct io_uring_task *tctx = task ? task->io_uring : NULL; ++ ++ while (1) { ++ enum io_wq_cancel cret; ++ bool ret = false; ++ ++ if (!task) { ++ ret |= io_uring_try_cancel_iowq(ctx); ++ } else if (tctx && tctx->io_wq) { ++ /* ++ * Cancels requests of all rings, not only @ctx, but ++ * it's fine as the task is in exit/exec. ++ */ ++ cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb, ++ &cancel, true); ++ ret |= (cret != IO_WQ_CANCEL_NOTFOUND); ++ } ++ ++ /* SQPOLL thread does its own polling */ ++ if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) || ++ (ctx->sq_data && ctx->sq_data->thread == current)) { ++ while (!list_empty_careful(&ctx->iopoll_list)) { ++ io_iopoll_try_reap_events(ctx); ++ ret = true; ++ } ++ } ++ ++ ret |= io_cancel_defer_files(ctx, task, cancel_all); ++ ret |= io_poll_remove_all(ctx, task, cancel_all); ++ ret |= io_kill_timeouts(ctx, task, cancel_all); ++ if (task) ++ ret |= io_run_task_work(); ++ if (!ret) ++ break; ++ cond_resched(); ++ } ++} ++ ++static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) ++{ ++ struct io_uring_task *tctx = current->io_uring; ++ struct io_tctx_node *node; ++ int ret; ++ ++ if (unlikely(!tctx)) { ++ ret = io_uring_alloc_task_context(current, ctx); ++ if (unlikely(ret)) ++ return ret; ++ ++ tctx = current->io_uring; ++ if (ctx->iowq_limits_set) { ++ unsigned int limits[2] = { ctx->iowq_limits[0], ++ ctx->iowq_limits[1], }; ++ ++ ret = io_wq_max_workers(tctx->io_wq, limits); ++ if (ret) ++ return ret; ++ } ++ } ++ if (!xa_load(&tctx->xa, (unsigned long)ctx)) { ++ node = kmalloc(sizeof(*node), GFP_KERNEL); ++ if (!node) ++ return -ENOMEM; ++ node->ctx = ctx; ++ node->task = current; ++ ++ ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx, ++ node, GFP_KERNEL)); ++ if (ret) { ++ kfree(node); ++ return ret; ++ } ++ ++ mutex_lock(&ctx->uring_lock); ++ list_add(&node->ctx_node, &ctx->tctx_list); ++ mutex_unlock(&ctx->uring_lock); ++ } ++ tctx->last = ctx; ++ return 0; ++} ++ ++/* ++ * Note that this task has used io_uring. We use it for cancelation purposes. ++ */ ++static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx) ++{ ++ struct io_uring_task *tctx = current->io_uring; ++ ++ if (likely(tctx && tctx->last == ctx)) ++ return 0; ++ return __io_uring_add_tctx_node(ctx); ++} ++ ++/* ++ * Remove this io_uring_file -> task mapping. ++ */ ++static void io_uring_del_tctx_node(unsigned long index) ++{ ++ struct io_uring_task *tctx = current->io_uring; ++ struct io_tctx_node *node; ++ ++ if (!tctx) ++ return; ++ node = xa_erase(&tctx->xa, index); ++ if (!node) ++ return; ++ ++ WARN_ON_ONCE(current != node->task); ++ WARN_ON_ONCE(list_empty(&node->ctx_node)); ++ ++ mutex_lock(&node->ctx->uring_lock); ++ list_del(&node->ctx_node); ++ mutex_unlock(&node->ctx->uring_lock); ++ ++ if (tctx->last == node->ctx) ++ tctx->last = NULL; ++ kfree(node); ++} ++ ++static void io_uring_clean_tctx(struct io_uring_task *tctx) ++{ ++ struct io_wq *wq = tctx->io_wq; ++ struct io_tctx_node *node; ++ unsigned long index; ++ ++ xa_for_each(&tctx->xa, index, node) { ++ io_uring_del_tctx_node(index); ++ cond_resched(); ++ } ++ if (wq) { ++ /* ++ * Must be after io_uring_del_task_file() (removes nodes under ++ * uring_lock) to avoid race with io_uring_try_cancel_iowq(). ++ */ ++ io_wq_put_and_exit(wq); ++ tctx->io_wq = NULL; ++ } ++} ++ ++static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked) ++{ ++ if (tracked) ++ return atomic_read(&tctx->inflight_tracked); ++ return percpu_counter_sum(&tctx->inflight); ++} ++ ++/* ++ * Find any io_uring ctx that this task has registered or done IO on, and cancel ++ * requests. @sqd should be not-null IFF it's an SQPOLL thread cancellation. ++ */ ++static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd) ++{ ++ struct io_uring_task *tctx = current->io_uring; ++ struct io_ring_ctx *ctx; ++ s64 inflight; ++ DEFINE_WAIT(wait); ++ ++ WARN_ON_ONCE(sqd && sqd->thread != current); ++ ++ if (!current->io_uring) ++ return; ++ if (tctx->io_wq) ++ io_wq_exit_start(tctx->io_wq); ++ ++ atomic_inc(&tctx->in_idle); ++ do { ++ io_uring_drop_tctx_refs(current); ++ /* read completions before cancelations */ ++ inflight = tctx_inflight(tctx, !cancel_all); ++ if (!inflight) ++ break; ++ ++ if (!sqd) { ++ struct io_tctx_node *node; ++ unsigned long index; ++ ++ xa_for_each(&tctx->xa, index, node) { ++ /* sqpoll task will cancel all its requests */ ++ if (node->ctx->sq_data) ++ continue; ++ io_uring_try_cancel_requests(node->ctx, current, ++ cancel_all); ++ } ++ } else { ++ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) ++ io_uring_try_cancel_requests(ctx, current, ++ cancel_all); ++ } ++ ++ prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE); ++ io_run_task_work(); ++ io_uring_drop_tctx_refs(current); ++ ++ /* ++ * If we've seen completions, retry without waiting. This ++ * avoids a race where a completion comes in before we did ++ * prepare_to_wait(). ++ */ ++ if (inflight == tctx_inflight(tctx, !cancel_all)) ++ schedule(); ++ finish_wait(&tctx->wait, &wait); ++ } while (1); ++ ++ io_uring_clean_tctx(tctx); ++ if (cancel_all) { ++ /* ++ * We shouldn't run task_works after cancel, so just leave ++ * ->in_idle set for normal exit. ++ */ ++ atomic_dec(&tctx->in_idle); ++ /* for exec all current's requests should be gone, kill tctx */ ++ __io_uring_free(current); ++ } ++} ++ ++void __io_uring_cancel(bool cancel_all) ++{ ++ io_uring_cancel_generic(cancel_all, NULL); ++} ++ ++static void *io_uring_validate_mmap_request(struct file *file, ++ loff_t pgoff, size_t sz) ++{ ++ struct io_ring_ctx *ctx = file->private_data; ++ loff_t offset = pgoff << PAGE_SHIFT; ++ struct page *page; ++ void *ptr; ++ ++ switch (offset) { ++ case IORING_OFF_SQ_RING: ++ case IORING_OFF_CQ_RING: ++ ptr = ctx->rings; ++ break; ++ case IORING_OFF_SQES: ++ ptr = ctx->sq_sqes; ++ break; ++ default: ++ return ERR_PTR(-EINVAL); ++ } ++ ++ page = virt_to_head_page(ptr); ++ if (sz > page_size(page)) ++ return ERR_PTR(-EINVAL); ++ ++ return ptr; ++} ++ ++#ifdef CONFIG_MMU ++ ++static int io_uring_mmap(struct file *file, struct vm_area_struct *vma) ++{ ++ size_t sz = vma->vm_end - vma->vm_start; ++ unsigned long pfn; ++ void *ptr; ++ ++ ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz); ++ if (IS_ERR(ptr)) ++ return PTR_ERR(ptr); ++ ++ pfn = virt_to_phys(ptr) >> PAGE_SHIFT; ++ return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot); ++} ++ ++#else /* !CONFIG_MMU */ ++ ++static int io_uring_mmap(struct file *file, struct vm_area_struct *vma) ++{ ++ return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL; ++} ++ ++static unsigned int io_uring_nommu_mmap_capabilities(struct file *file) ++{ ++ return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE; ++} ++ ++static unsigned long io_uring_nommu_get_unmapped_area(struct file *file, ++ unsigned long addr, unsigned long len, ++ unsigned long pgoff, unsigned long flags) ++{ ++ void *ptr; ++ ++ ptr = io_uring_validate_mmap_request(file, pgoff, len); ++ if (IS_ERR(ptr)) ++ return PTR_ERR(ptr); ++ ++ return (unsigned long) ptr; ++} ++ ++#endif /* !CONFIG_MMU */ ++ ++static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx) ++{ ++ DEFINE_WAIT(wait); ++ ++ do { ++ if (!io_sqring_full(ctx)) ++ break; ++ prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE); ++ ++ if (!io_sqring_full(ctx)) ++ break; ++ schedule(); ++ } while (!signal_pending(current)); ++ ++ finish_wait(&ctx->sqo_sq_wait, &wait); ++ return 0; ++} ++ ++static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz, ++ struct __kernel_timespec __user **ts, ++ const sigset_t __user **sig) ++{ ++ struct io_uring_getevents_arg arg; ++ ++ /* ++ * If EXT_ARG isn't set, then we have no timespec and the argp pointer ++ * is just a pointer to the sigset_t. ++ */ ++ if (!(flags & IORING_ENTER_EXT_ARG)) { ++ *sig = (const sigset_t __user *) argp; ++ *ts = NULL; ++ return 0; ++ } ++ ++ /* ++ * EXT_ARG is set - ensure we agree on the size of it and copy in our ++ * timespec and sigset_t pointers if good. ++ */ ++ if (*argsz != sizeof(arg)) ++ return -EINVAL; ++ if (copy_from_user(&arg, argp, sizeof(arg))) ++ return -EFAULT; ++ if (arg.pad) ++ return -EINVAL; ++ *sig = u64_to_user_ptr(arg.sigmask); ++ *argsz = arg.sigmask_sz; ++ *ts = u64_to_user_ptr(arg.ts); ++ return 0; ++} ++ ++SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, ++ u32, min_complete, u32, flags, const void __user *, argp, ++ size_t, argsz) ++{ ++ struct io_ring_ctx *ctx; ++ int submitted = 0; ++ struct fd f; ++ long ret; ++ ++ io_run_task_work(); ++ ++ if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP | ++ IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG))) ++ return -EINVAL; ++ ++ f = fdget(fd); ++ if (unlikely(!f.file)) ++ return -EBADF; ++ ++ ret = -EOPNOTSUPP; ++ if (unlikely(f.file->f_op != &io_uring_fops)) ++ goto out_fput; ++ ++ ret = -ENXIO; ++ ctx = f.file->private_data; ++ if (unlikely(!percpu_ref_tryget(&ctx->refs))) ++ goto out_fput; ++ ++ ret = -EBADFD; ++ if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED)) ++ goto out; ++ ++ /* ++ * For SQ polling, the thread will do all submissions and completions. ++ * Just return the requested submit count, and wake the thread if ++ * we were asked to. ++ */ ++ ret = 0; ++ if (ctx->flags & IORING_SETUP_SQPOLL) { ++ io_cqring_overflow_flush(ctx); ++ ++ if (unlikely(ctx->sq_data->thread == NULL)) { ++ ret = -EOWNERDEAD; ++ goto out; ++ } ++ if (flags & IORING_ENTER_SQ_WAKEUP) ++ wake_up(&ctx->sq_data->wait); ++ if (flags & IORING_ENTER_SQ_WAIT) { ++ ret = io_sqpoll_wait_sq(ctx); ++ if (ret) ++ goto out; ++ } ++ submitted = to_submit; ++ } else if (to_submit) { ++ ret = io_uring_add_tctx_node(ctx); ++ if (unlikely(ret)) ++ goto out; ++ mutex_lock(&ctx->uring_lock); ++ submitted = io_submit_sqes(ctx, to_submit); ++ mutex_unlock(&ctx->uring_lock); ++ ++ if (submitted != to_submit) ++ goto out; ++ } ++ if (flags & IORING_ENTER_GETEVENTS) { ++ const sigset_t __user *sig; ++ struct __kernel_timespec __user *ts; ++ ++ ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig); ++ if (unlikely(ret)) ++ goto out; ++ ++ min_complete = min(min_complete, ctx->cq_entries); ++ ++ /* ++ * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user ++ * space applications don't need to do io completion events ++ * polling again, they can rely on io_sq_thread to do polling ++ * work, which can reduce cpu usage and uring_lock contention. ++ */ ++ if (ctx->flags & IORING_SETUP_IOPOLL && ++ !(ctx->flags & IORING_SETUP_SQPOLL)) { ++ ret = io_iopoll_check(ctx, min_complete); ++ } else { ++ ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts); ++ } ++ } ++ ++out: ++ percpu_ref_put(&ctx->refs); ++out_fput: ++ fdput(f); ++ return submitted ? submitted : ret; ++} ++ ++#ifdef CONFIG_PROC_FS ++static int io_uring_show_cred(struct seq_file *m, unsigned int id, ++ const struct cred *cred) ++{ ++ struct user_namespace *uns = seq_user_ns(m); ++ struct group_info *gi; ++ kernel_cap_t cap; ++ unsigned __capi; ++ int g; ++ ++ seq_printf(m, "%5d\n", id); ++ seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid)); ++ seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid)); ++ seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid)); ++ seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid)); ++ seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid)); ++ seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid)); ++ seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid)); ++ seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid)); ++ seq_puts(m, "\n\tGroups:\t"); ++ gi = cred->group_info; ++ for (g = 0; g < gi->ngroups; g++) { ++ seq_put_decimal_ull(m, g ? " " : "", ++ from_kgid_munged(uns, gi->gid[g])); ++ } ++ seq_puts(m, "\n\tCapEff:\t"); ++ cap = cred->cap_effective; ++ CAP_FOR_EACH_U32(__capi) ++ seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8); ++ seq_putc(m, '\n'); ++ return 0; ++} ++ ++static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) ++{ ++ struct io_sq_data *sq = NULL; ++ bool has_lock; ++ int i; ++ ++ /* ++ * Avoid ABBA deadlock between the seq lock and the io_uring mutex, ++ * since fdinfo case grabs it in the opposite direction of normal use ++ * cases. If we fail to get the lock, we just don't iterate any ++ * structures that could be going away outside the io_uring mutex. ++ */ ++ has_lock = mutex_trylock(&ctx->uring_lock); ++ ++ if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) { ++ sq = ctx->sq_data; ++ if (!sq->thread) ++ sq = NULL; ++ } ++ ++ seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1); ++ seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1); ++ seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files); ++ for (i = 0; has_lock && i < ctx->nr_user_files; i++) { ++ struct file *f = io_file_from_index(ctx, i); ++ ++ if (f) ++ seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname); ++ else ++ seq_printf(m, "%5u: \n", i); ++ } ++ seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs); ++ for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) { ++ struct io_mapped_ubuf *buf = ctx->user_bufs[i]; ++ unsigned int len = buf->ubuf_end - buf->ubuf; ++ ++ seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, len); ++ } ++ if (has_lock && !xa_empty(&ctx->personalities)) { ++ unsigned long index; ++ const struct cred *cred; ++ ++ seq_printf(m, "Personalities:\n"); ++ xa_for_each(&ctx->personalities, index, cred) ++ io_uring_show_cred(m, index, cred); ++ } ++ seq_printf(m, "PollList:\n"); ++ spin_lock(&ctx->completion_lock); ++ for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { ++ struct hlist_head *list = &ctx->cancel_hash[i]; ++ struct io_kiocb *req; ++ ++ hlist_for_each_entry(req, list, hash_node) ++ seq_printf(m, " op=%d, task_works=%d\n", req->opcode, ++ req->task->task_works != NULL); ++ } ++ spin_unlock(&ctx->completion_lock); ++ if (has_lock) ++ mutex_unlock(&ctx->uring_lock); ++} ++ ++static void io_uring_show_fdinfo(struct seq_file *m, struct file *f) ++{ ++ struct io_ring_ctx *ctx = f->private_data; ++ ++ if (percpu_ref_tryget(&ctx->refs)) { ++ __io_uring_show_fdinfo(ctx, m); ++ percpu_ref_put(&ctx->refs); ++ } ++} ++#endif ++ ++static const struct file_operations io_uring_fops = { ++ .release = io_uring_release, ++ .mmap = io_uring_mmap, ++#ifndef CONFIG_MMU ++ .get_unmapped_area = io_uring_nommu_get_unmapped_area, ++ .mmap_capabilities = io_uring_nommu_mmap_capabilities, ++#endif ++ .poll = io_uring_poll, ++#ifdef CONFIG_PROC_FS ++ .show_fdinfo = io_uring_show_fdinfo, ++#endif ++}; ++ ++static int io_allocate_scq_urings(struct io_ring_ctx *ctx, ++ struct io_uring_params *p) ++{ ++ struct io_rings *rings; ++ size_t size, sq_array_offset; ++ ++ /* make sure these are sane, as we already accounted them */ ++ ctx->sq_entries = p->sq_entries; ++ ctx->cq_entries = p->cq_entries; ++ ++ size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset); ++ if (size == SIZE_MAX) ++ return -EOVERFLOW; ++ ++ rings = io_mem_alloc(size); ++ if (!rings) ++ return -ENOMEM; ++ ++ ctx->rings = rings; ++ ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); ++ rings->sq_ring_mask = p->sq_entries - 1; ++ rings->cq_ring_mask = p->cq_entries - 1; ++ rings->sq_ring_entries = p->sq_entries; ++ rings->cq_ring_entries = p->cq_entries; ++ ++ size = array_size(sizeof(struct io_uring_sqe), p->sq_entries); ++ if (size == SIZE_MAX) { ++ io_mem_free(ctx->rings); ++ ctx->rings = NULL; ++ return -EOVERFLOW; ++ } ++ ++ ctx->sq_sqes = io_mem_alloc(size); ++ if (!ctx->sq_sqes) { ++ io_mem_free(ctx->rings); ++ ctx->rings = NULL; ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++ ++static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file) ++{ ++ int ret, fd; ++ ++ fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); ++ if (fd < 0) ++ return fd; ++ ++ ret = io_uring_add_tctx_node(ctx); ++ if (ret) { ++ put_unused_fd(fd); ++ return ret; ++ } ++ fd_install(fd, file); ++ return fd; ++} ++ ++/* ++ * Allocate an anonymous fd, this is what constitutes the application ++ * visible backing of an io_uring instance. The application mmaps this ++ * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled, ++ * we have to tie this fd to a socket for file garbage collection purposes. ++ */ ++static struct file *io_uring_get_file(struct io_ring_ctx *ctx) ++{ ++ struct file *file; ++#if defined(CONFIG_UNIX) ++ int ret; ++ ++ ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP, ++ &ctx->ring_sock); ++ if (ret) ++ return ERR_PTR(ret); ++#endif ++ ++ file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx, ++ O_RDWR | O_CLOEXEC); ++#if defined(CONFIG_UNIX) ++ if (IS_ERR(file)) { ++ sock_release(ctx->ring_sock); ++ ctx->ring_sock = NULL; ++ } else { ++ ctx->ring_sock->file = file; ++ } ++#endif ++ return file; ++} ++ ++static int io_uring_create(unsigned entries, struct io_uring_params *p, ++ struct io_uring_params __user *params) ++{ ++ struct io_ring_ctx *ctx; ++ struct file *file; ++ int ret; ++ ++ if (!entries) ++ return -EINVAL; ++ if (entries > IORING_MAX_ENTRIES) { ++ if (!(p->flags & IORING_SETUP_CLAMP)) ++ return -EINVAL; ++ entries = IORING_MAX_ENTRIES; ++ } ++ ++ /* ++ * Use twice as many entries for the CQ ring. It's possible for the ++ * application to drive a higher depth than the size of the SQ ring, ++ * since the sqes are only used at submission time. This allows for ++ * some flexibility in overcommitting a bit. If the application has ++ * set IORING_SETUP_CQSIZE, it will have passed in the desired number ++ * of CQ ring entries manually. ++ */ ++ p->sq_entries = roundup_pow_of_two(entries); ++ if (p->flags & IORING_SETUP_CQSIZE) { ++ /* ++ * If IORING_SETUP_CQSIZE is set, we do the same roundup ++ * to a power-of-two, if it isn't already. We do NOT impose ++ * any cq vs sq ring sizing. ++ */ ++ if (!p->cq_entries) ++ return -EINVAL; ++ if (p->cq_entries > IORING_MAX_CQ_ENTRIES) { ++ if (!(p->flags & IORING_SETUP_CLAMP)) ++ return -EINVAL; ++ p->cq_entries = IORING_MAX_CQ_ENTRIES; ++ } ++ p->cq_entries = roundup_pow_of_two(p->cq_entries); ++ if (p->cq_entries < p->sq_entries) ++ return -EINVAL; ++ } else { ++ p->cq_entries = 2 * p->sq_entries; ++ } ++ ++ ctx = io_ring_ctx_alloc(p); ++ if (!ctx) ++ return -ENOMEM; ++ ctx->compat = in_compat_syscall(); ++ if (!capable(CAP_IPC_LOCK)) ++ ctx->user = get_uid(current_user()); ++ ++ /* ++ * This is just grabbed for accounting purposes. When a process exits, ++ * the mm is exited and dropped before the files, hence we need to hang ++ * on to this mm purely for the purposes of being able to unaccount ++ * memory (locked/pinned vm). It's not used for anything else. ++ */ ++ mmgrab(current->mm); ++ ctx->mm_account = current->mm; ++ ++ ret = io_allocate_scq_urings(ctx, p); ++ if (ret) ++ goto err; ++ ++ ret = io_sq_offload_create(ctx, p); ++ if (ret) ++ goto err; ++ /* always set a rsrc node */ ++ ret = io_rsrc_node_switch_start(ctx); ++ if (ret) ++ goto err; ++ io_rsrc_node_switch(ctx, NULL); ++ ++ memset(&p->sq_off, 0, sizeof(p->sq_off)); ++ p->sq_off.head = offsetof(struct io_rings, sq.head); ++ p->sq_off.tail = offsetof(struct io_rings, sq.tail); ++ p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask); ++ p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries); ++ p->sq_off.flags = offsetof(struct io_rings, sq_flags); ++ p->sq_off.dropped = offsetof(struct io_rings, sq_dropped); ++ p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; ++ ++ memset(&p->cq_off, 0, sizeof(p->cq_off)); ++ p->cq_off.head = offsetof(struct io_rings, cq.head); ++ p->cq_off.tail = offsetof(struct io_rings, cq.tail); ++ p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask); ++ p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries); ++ p->cq_off.overflow = offsetof(struct io_rings, cq_overflow); ++ p->cq_off.cqes = offsetof(struct io_rings, cqes); ++ p->cq_off.flags = offsetof(struct io_rings, cq_flags); ++ ++ p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP | ++ IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS | ++ IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL | ++ IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED | ++ IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS | ++ IORING_FEAT_RSRC_TAGS; ++ ++ if (copy_to_user(params, p, sizeof(*p))) { ++ ret = -EFAULT; ++ goto err; ++ } ++ ++ file = io_uring_get_file(ctx); ++ if (IS_ERR(file)) { ++ ret = PTR_ERR(file); ++ goto err; ++ } ++ ++ /* ++ * Install ring fd as the very last thing, so we don't risk someone ++ * having closed it before we finish setup ++ */ ++ ret = io_uring_install_fd(ctx, file); ++ if (ret < 0) { ++ /* fput will clean it up */ ++ fput(file); ++ return ret; ++ } ++ ++ trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags); ++ return ret; ++err: ++ io_ring_ctx_wait_and_kill(ctx); ++ return ret; ++} ++ ++/* ++ * Sets up an aio uring context, and returns the fd. Applications asks for a ++ * ring size, we return the actual sq/cq ring sizes (among other things) in the ++ * params structure passed in. ++ */ ++static long io_uring_setup(u32 entries, struct io_uring_params __user *params) ++{ ++ struct io_uring_params p; ++ int i; ++ ++ if (copy_from_user(&p, params, sizeof(p))) ++ return -EFAULT; ++ for (i = 0; i < ARRAY_SIZE(p.resv); i++) { ++ if (p.resv[i]) ++ return -EINVAL; ++ } ++ ++ if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL | ++ IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE | ++ IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ | ++ IORING_SETUP_R_DISABLED)) ++ return -EINVAL; ++ ++ return io_uring_create(entries, &p, params); ++} ++ ++SYSCALL_DEFINE2(io_uring_setup, u32, entries, ++ struct io_uring_params __user *, params) ++{ ++ return io_uring_setup(entries, params); ++} ++ ++static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) ++{ ++ struct io_uring_probe *p; ++ size_t size; ++ int i, ret; ++ ++ size = struct_size(p, ops, nr_args); ++ if (size == SIZE_MAX) ++ return -EOVERFLOW; ++ p = kzalloc(size, GFP_KERNEL); ++ if (!p) ++ return -ENOMEM; ++ ++ ret = -EFAULT; ++ if (copy_from_user(p, arg, size)) ++ goto out; ++ ret = -EINVAL; ++ if (memchr_inv(p, 0, size)) ++ goto out; ++ ++ p->last_op = IORING_OP_LAST - 1; ++ if (nr_args > IORING_OP_LAST) ++ nr_args = IORING_OP_LAST; ++ ++ for (i = 0; i < nr_args; i++) { ++ p->ops[i].op = i; ++ if (!io_op_defs[i].not_supported) ++ p->ops[i].flags = IO_URING_OP_SUPPORTED; ++ } ++ p->ops_len = i; ++ ++ ret = 0; ++ if (copy_to_user(arg, p, size)) ++ ret = -EFAULT; ++out: ++ kfree(p); ++ return ret; ++} ++ ++static int io_register_personality(struct io_ring_ctx *ctx) ++{ ++ const struct cred *creds; ++ u32 id; ++ int ret; ++ ++ creds = get_current_cred(); ++ ++ ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds, ++ XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL); ++ if (ret < 0) { ++ put_cred(creds); ++ return ret; ++ } ++ return id; ++} ++ ++static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg, ++ unsigned int nr_args) ++{ ++ struct io_uring_restriction *res; ++ size_t size; ++ int i, ret; ++ ++ /* Restrictions allowed only if rings started disabled */ ++ if (!(ctx->flags & IORING_SETUP_R_DISABLED)) ++ return -EBADFD; ++ ++ /* We allow only a single restrictions registration */ ++ if (ctx->restrictions.registered) ++ return -EBUSY; ++ ++ if (!arg || nr_args > IORING_MAX_RESTRICTIONS) ++ return -EINVAL; ++ ++ size = array_size(nr_args, sizeof(*res)); ++ if (size == SIZE_MAX) ++ return -EOVERFLOW; ++ ++ res = memdup_user(arg, size); ++ if (IS_ERR(res)) ++ return PTR_ERR(res); ++ ++ ret = 0; ++ ++ for (i = 0; i < nr_args; i++) { ++ switch (res[i].opcode) { ++ case IORING_RESTRICTION_REGISTER_OP: ++ if (res[i].register_op >= IORING_REGISTER_LAST) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ __set_bit(res[i].register_op, ++ ctx->restrictions.register_op); ++ break; ++ case IORING_RESTRICTION_SQE_OP: ++ if (res[i].sqe_op >= IORING_OP_LAST) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op); ++ break; ++ case IORING_RESTRICTION_SQE_FLAGS_ALLOWED: ++ ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags; ++ break; ++ case IORING_RESTRICTION_SQE_FLAGS_REQUIRED: ++ ctx->restrictions.sqe_flags_required = res[i].sqe_flags; ++ break; ++ default: ++ ret = -EINVAL; ++ goto out; ++ } ++ } ++ ++out: ++ /* Reset all restrictions if an error happened */ ++ if (ret != 0) ++ memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); ++ else ++ ctx->restrictions.registered = true; ++ ++ kfree(res); ++ return ret; ++} ++ ++static int io_register_enable_rings(struct io_ring_ctx *ctx) ++{ ++ if (!(ctx->flags & IORING_SETUP_R_DISABLED)) ++ return -EBADFD; ++ ++ if (ctx->restrictions.registered) ++ ctx->restricted = 1; ++ ++ ctx->flags &= ~IORING_SETUP_R_DISABLED; ++ if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) ++ wake_up(&ctx->sq_data->wait); ++ return 0; ++} ++ ++static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, ++ struct io_uring_rsrc_update2 *up, ++ unsigned nr_args) ++{ ++ __u32 tmp; ++ int err; ++ ++ if (check_add_overflow(up->offset, nr_args, &tmp)) ++ return -EOVERFLOW; ++ err = io_rsrc_node_switch_start(ctx); ++ if (err) ++ return err; ++ ++ switch (type) { ++ case IORING_RSRC_FILE: ++ return __io_sqe_files_update(ctx, up, nr_args); ++ case IORING_RSRC_BUFFER: ++ return __io_sqe_buffers_update(ctx, up, nr_args); ++ } ++ return -EINVAL; ++} ++ ++static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg, ++ unsigned nr_args) ++{ ++ struct io_uring_rsrc_update2 up; ++ ++ if (!nr_args) ++ return -EINVAL; ++ memset(&up, 0, sizeof(up)); ++ if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update))) ++ return -EFAULT; ++ if (up.resv || up.resv2) ++ return -EINVAL; ++ return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args); ++} ++ ++static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, ++ unsigned size, unsigned type) ++{ ++ struct io_uring_rsrc_update2 up; ++ ++ if (size != sizeof(up)) ++ return -EINVAL; ++ if (copy_from_user(&up, arg, sizeof(up))) ++ return -EFAULT; ++ if (!up.nr || up.resv || up.resv2) ++ return -EINVAL; ++ return __io_register_rsrc_update(ctx, type, &up, up.nr); ++} ++ ++static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, ++ unsigned int size, unsigned int type) ++{ ++ struct io_uring_rsrc_register rr; ++ ++ /* keep it extendible */ ++ if (size != sizeof(rr)) ++ return -EINVAL; ++ ++ memset(&rr, 0, sizeof(rr)); ++ if (copy_from_user(&rr, arg, size)) ++ return -EFAULT; ++ if (!rr.nr || rr.resv || rr.resv2) ++ return -EINVAL; ++ ++ switch (type) { ++ case IORING_RSRC_FILE: ++ return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data), ++ rr.nr, u64_to_user_ptr(rr.tags)); ++ case IORING_RSRC_BUFFER: ++ return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data), ++ rr.nr, u64_to_user_ptr(rr.tags)); ++ } ++ return -EINVAL; ++} ++ ++static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg, ++ unsigned len) ++{ ++ struct io_uring_task *tctx = current->io_uring; ++ cpumask_var_t new_mask; ++ int ret; ++ ++ if (!tctx || !tctx->io_wq) ++ return -EINVAL; ++ ++ if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) ++ return -ENOMEM; ++ ++ cpumask_clear(new_mask); ++ if (len > cpumask_size()) ++ len = cpumask_size(); ++ ++#ifdef CONFIG_COMPAT ++ if (in_compat_syscall()) { ++ ret = compat_get_bitmap(cpumask_bits(new_mask), ++ (const compat_ulong_t __user *)arg, ++ len * 8 /* CHAR_BIT */); ++ } else { ++ ret = copy_from_user(new_mask, arg, len); ++ } ++#else ++ ret = copy_from_user(new_mask, arg, len); ++#endif ++ ++ if (ret) { ++ free_cpumask_var(new_mask); ++ return -EFAULT; ++ } ++ ++ ret = io_wq_cpu_affinity(tctx->io_wq, new_mask); ++ free_cpumask_var(new_mask); ++ return ret; ++} ++ ++static int io_unregister_iowq_aff(struct io_ring_ctx *ctx) ++{ ++ struct io_uring_task *tctx = current->io_uring; ++ ++ if (!tctx || !tctx->io_wq) ++ return -EINVAL; ++ ++ return io_wq_cpu_affinity(tctx->io_wq, NULL); ++} ++ ++static int io_register_iowq_max_workers(struct io_ring_ctx *ctx, ++ void __user *arg) ++ __must_hold(&ctx->uring_lock) ++{ ++ struct io_tctx_node *node; ++ struct io_uring_task *tctx = NULL; ++ struct io_sq_data *sqd = NULL; ++ __u32 new_count[2]; ++ int i, ret; ++ ++ if (copy_from_user(new_count, arg, sizeof(new_count))) ++ return -EFAULT; ++ for (i = 0; i < ARRAY_SIZE(new_count); i++) ++ if (new_count[i] > INT_MAX) ++ return -EINVAL; ++ ++ if (ctx->flags & IORING_SETUP_SQPOLL) { ++ sqd = ctx->sq_data; ++ if (sqd) { ++ /* ++ * Observe the correct sqd->lock -> ctx->uring_lock ++ * ordering. Fine to drop uring_lock here, we hold ++ * a ref to the ctx. ++ */ ++ refcount_inc(&sqd->refs); ++ mutex_unlock(&ctx->uring_lock); ++ mutex_lock(&sqd->lock); ++ mutex_lock(&ctx->uring_lock); ++ if (sqd->thread) ++ tctx = sqd->thread->io_uring; ++ } ++ } else { ++ tctx = current->io_uring; ++ } ++ ++ BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits)); ++ ++ for (i = 0; i < ARRAY_SIZE(new_count); i++) ++ if (new_count[i]) ++ ctx->iowq_limits[i] = new_count[i]; ++ ctx->iowq_limits_set = true; ++ ++ ret = -EINVAL; ++ if (tctx && tctx->io_wq) { ++ ret = io_wq_max_workers(tctx->io_wq, new_count); ++ if (ret) ++ goto err; ++ } else { ++ memset(new_count, 0, sizeof(new_count)); ++ } ++ ++ if (sqd) { ++ mutex_unlock(&sqd->lock); ++ io_put_sq_data(sqd); ++ } ++ ++ if (copy_to_user(arg, new_count, sizeof(new_count))) ++ return -EFAULT; ++ ++ /* that's it for SQPOLL, only the SQPOLL task creates requests */ ++ if (sqd) ++ return 0; ++ ++ /* now propagate the restriction to all registered users */ ++ list_for_each_entry(node, &ctx->tctx_list, ctx_node) { ++ struct io_uring_task *tctx = node->task->io_uring; ++ ++ if (WARN_ON_ONCE(!tctx->io_wq)) ++ continue; ++ ++ for (i = 0; i < ARRAY_SIZE(new_count); i++) ++ new_count[i] = ctx->iowq_limits[i]; ++ /* ignore errors, it always returns zero anyway */ ++ (void)io_wq_max_workers(tctx->io_wq, new_count); ++ } ++ return 0; ++err: ++ if (sqd) { ++ mutex_unlock(&sqd->lock); ++ io_put_sq_data(sqd); ++ } ++ return ret; ++} ++ ++static bool io_register_op_must_quiesce(int op) ++{ ++ switch (op) { ++ case IORING_REGISTER_BUFFERS: ++ case IORING_UNREGISTER_BUFFERS: ++ case IORING_REGISTER_FILES: ++ case IORING_UNREGISTER_FILES: ++ case IORING_REGISTER_FILES_UPDATE: ++ case IORING_REGISTER_PROBE: ++ case IORING_REGISTER_PERSONALITY: ++ case IORING_UNREGISTER_PERSONALITY: ++ case IORING_REGISTER_FILES2: ++ case IORING_REGISTER_FILES_UPDATE2: ++ case IORING_REGISTER_BUFFERS2: ++ case IORING_REGISTER_BUFFERS_UPDATE: ++ case IORING_REGISTER_IOWQ_AFF: ++ case IORING_UNREGISTER_IOWQ_AFF: ++ case IORING_REGISTER_IOWQ_MAX_WORKERS: ++ return false; ++ default: ++ return true; ++ } ++} ++ ++static int io_ctx_quiesce(struct io_ring_ctx *ctx) ++{ ++ long ret; ++ ++ percpu_ref_kill(&ctx->refs); ++ ++ /* ++ * Drop uring mutex before waiting for references to exit. If another ++ * thread is currently inside io_uring_enter() it might need to grab the ++ * uring_lock to make progress. If we hold it here across the drain ++ * wait, then we can deadlock. It's safe to drop the mutex here, since ++ * no new references will come in after we've killed the percpu ref. ++ */ ++ mutex_unlock(&ctx->uring_lock); ++ do { ++ ret = wait_for_completion_interruptible(&ctx->ref_comp); ++ if (!ret) ++ break; ++ ret = io_run_task_work_sig(); ++ } while (ret >= 0); ++ mutex_lock(&ctx->uring_lock); ++ ++ if (ret) ++ io_refs_resurrect(&ctx->refs, &ctx->ref_comp); ++ return ret; ++} ++ ++static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, ++ void __user *arg, unsigned nr_args) ++ __releases(ctx->uring_lock) ++ __acquires(ctx->uring_lock) ++{ ++ int ret; ++ ++ /* ++ * We're inside the ring mutex, if the ref is already dying, then ++ * someone else killed the ctx or is already going through ++ * io_uring_register(). ++ */ ++ if (percpu_ref_is_dying(&ctx->refs)) ++ return -ENXIO; ++ ++ if (ctx->restricted) { ++ if (opcode >= IORING_REGISTER_LAST) ++ return -EINVAL; ++ opcode = array_index_nospec(opcode, IORING_REGISTER_LAST); ++ if (!test_bit(opcode, ctx->restrictions.register_op)) ++ return -EACCES; ++ } ++ ++ if (io_register_op_must_quiesce(opcode)) { ++ ret = io_ctx_quiesce(ctx); ++ if (ret) ++ return ret; ++ } ++ ++ switch (opcode) { ++ case IORING_REGISTER_BUFFERS: ++ ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL); ++ break; ++ case IORING_UNREGISTER_BUFFERS: ++ ret = -EINVAL; ++ if (arg || nr_args) ++ break; ++ ret = io_sqe_buffers_unregister(ctx); ++ break; ++ case IORING_REGISTER_FILES: ++ ret = io_sqe_files_register(ctx, arg, nr_args, NULL); ++ break; ++ case IORING_UNREGISTER_FILES: ++ ret = -EINVAL; ++ if (arg || nr_args) ++ break; ++ ret = io_sqe_files_unregister(ctx); ++ break; ++ case IORING_REGISTER_FILES_UPDATE: ++ ret = io_register_files_update(ctx, arg, nr_args); ++ break; ++ case IORING_REGISTER_EVENTFD: ++ case IORING_REGISTER_EVENTFD_ASYNC: ++ ret = -EINVAL; ++ if (nr_args != 1) ++ break; ++ ret = io_eventfd_register(ctx, arg); ++ if (ret) ++ break; ++ if (opcode == IORING_REGISTER_EVENTFD_ASYNC) ++ ctx->eventfd_async = 1; ++ else ++ ctx->eventfd_async = 0; ++ break; ++ case IORING_UNREGISTER_EVENTFD: ++ ret = -EINVAL; ++ if (arg || nr_args) ++ break; ++ ret = io_eventfd_unregister(ctx); ++ break; ++ case IORING_REGISTER_PROBE: ++ ret = -EINVAL; ++ if (!arg || nr_args > 256) ++ break; ++ ret = io_probe(ctx, arg, nr_args); ++ break; ++ case IORING_REGISTER_PERSONALITY: ++ ret = -EINVAL; ++ if (arg || nr_args) ++ break; ++ ret = io_register_personality(ctx); ++ break; ++ case IORING_UNREGISTER_PERSONALITY: ++ ret = -EINVAL; ++ if (arg) ++ break; ++ ret = io_unregister_personality(ctx, nr_args); ++ break; ++ case IORING_REGISTER_ENABLE_RINGS: ++ ret = -EINVAL; ++ if (arg || nr_args) ++ break; ++ ret = io_register_enable_rings(ctx); ++ break; ++ case IORING_REGISTER_RESTRICTIONS: ++ ret = io_register_restrictions(ctx, arg, nr_args); ++ break; ++ case IORING_REGISTER_FILES2: ++ ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE); ++ break; ++ case IORING_REGISTER_FILES_UPDATE2: ++ ret = io_register_rsrc_update(ctx, arg, nr_args, ++ IORING_RSRC_FILE); ++ break; ++ case IORING_REGISTER_BUFFERS2: ++ ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER); ++ break; ++ case IORING_REGISTER_BUFFERS_UPDATE: ++ ret = io_register_rsrc_update(ctx, arg, nr_args, ++ IORING_RSRC_BUFFER); ++ break; ++ case IORING_REGISTER_IOWQ_AFF: ++ ret = -EINVAL; ++ if (!arg || !nr_args) ++ break; ++ ret = io_register_iowq_aff(ctx, arg, nr_args); ++ break; ++ case IORING_UNREGISTER_IOWQ_AFF: ++ ret = -EINVAL; ++ if (arg || nr_args) ++ break; ++ ret = io_unregister_iowq_aff(ctx); ++ break; ++ case IORING_REGISTER_IOWQ_MAX_WORKERS: ++ ret = -EINVAL; ++ if (!arg || nr_args != 2) ++ break; ++ ret = io_register_iowq_max_workers(ctx, arg); ++ break; ++ default: ++ ret = -EINVAL; ++ break; ++ } ++ ++ if (io_register_op_must_quiesce(opcode)) { ++ /* bring the ctx back to life */ ++ percpu_ref_reinit(&ctx->refs); ++ reinit_completion(&ctx->ref_comp); ++ } ++ return ret; ++} ++ ++SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, ++ void __user *, arg, unsigned int, nr_args) ++{ ++ struct io_ring_ctx *ctx; ++ long ret = -EBADF; ++ struct fd f; ++ ++ f = fdget(fd); ++ if (!f.file) ++ return -EBADF; ++ ++ ret = -EOPNOTSUPP; ++ if (f.file->f_op != &io_uring_fops) ++ goto out_fput; ++ ++ ctx = f.file->private_data; ++ ++ io_run_task_work(); ++ ++ mutex_lock(&ctx->uring_lock); ++ ret = __io_uring_register(ctx, opcode, arg, nr_args); ++ mutex_unlock(&ctx->uring_lock); ++ trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, ++ ctx->cq_ev_fd != NULL, ret); ++out_fput: ++ fdput(f); ++ return ret; ++} ++ ++static int __init io_uring_init(void) ++{ ++#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \ ++ BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \ ++ BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \ ++} while (0) ++ ++#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \ ++ __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename) ++ BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64); ++ BUILD_BUG_SQE_ELEM(0, __u8, opcode); ++ BUILD_BUG_SQE_ELEM(1, __u8, flags); ++ BUILD_BUG_SQE_ELEM(2, __u16, ioprio); ++ BUILD_BUG_SQE_ELEM(4, __s32, fd); ++ BUILD_BUG_SQE_ELEM(8, __u64, off); ++ BUILD_BUG_SQE_ELEM(8, __u64, addr2); ++ BUILD_BUG_SQE_ELEM(16, __u64, addr); ++ BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in); ++ BUILD_BUG_SQE_ELEM(24, __u32, len); ++ BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags); ++ BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags); ++ BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags); ++ BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags); ++ BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events); ++ BUILD_BUG_SQE_ELEM(28, __u32, poll32_events); ++ BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags); ++ BUILD_BUG_SQE_ELEM(28, __u32, msg_flags); ++ BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags); ++ BUILD_BUG_SQE_ELEM(28, __u32, accept_flags); ++ BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags); ++ BUILD_BUG_SQE_ELEM(28, __u32, open_flags); ++ BUILD_BUG_SQE_ELEM(28, __u32, statx_flags); ++ BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice); ++ BUILD_BUG_SQE_ELEM(28, __u32, splice_flags); ++ BUILD_BUG_SQE_ELEM(32, __u64, user_data); ++ BUILD_BUG_SQE_ELEM(40, __u16, buf_index); ++ BUILD_BUG_SQE_ELEM(40, __u16, buf_group); ++ BUILD_BUG_SQE_ELEM(42, __u16, personality); ++ BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in); ++ BUILD_BUG_SQE_ELEM(44, __u32, file_index); ++ ++ BUILD_BUG_ON(sizeof(struct io_uring_files_update) != ++ sizeof(struct io_uring_rsrc_update)); ++ BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) > ++ sizeof(struct io_uring_rsrc_update2)); ++ ++ /* ->buf_index is u16 */ ++ BUILD_BUG_ON(IORING_MAX_REG_BUFFERS >= (1u << 16)); ++ ++ /* should fit into one byte */ ++ BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8)); ++ ++ BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); ++ BUILD_BUG_ON(__REQ_F_LAST_BIT > 8 * sizeof(int)); ++ ++ req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC | ++ SLAB_ACCOUNT); ++ return 0; ++}; ++__initcall(io_uring_init); +--- a/kernel/exit.c ++++ b/kernel/exit.c +@@ -763,7 +763,7 @@ void __noreturn do_exit(long code) + schedule(); + } + +- io_uring_files_cancel(tsk->files); ++ io_uring_files_cancel(); + exit_signals(tsk); /* sets PF_EXITING */ + + /* sync mm's RSS info before statistics gathering */ +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -926,6 +926,7 @@ static struct task_struct *dup_task_stru + tsk->splice_pipe = NULL; + tsk->task_frag.page = NULL; + tsk->wake_q.next = NULL; ++ tsk->pf_io_worker = NULL; + + account_kernel_stack(tsk, 1); + +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -21,7 +21,7 @@ + #include + + #include "../workqueue_internal.h" +-#include "../../fs/io-wq.h" ++#include "../../io_uring/io-wq.h" + #include "../smpboot.h" + + #include "pelt.h" diff --git a/next-5.10/io_uring-pass-in-epoll_uring_wake-for-eventfd-signaling-and-wakeups.patch b/next-5.10/io_uring-pass-in-epoll_uring_wake-for-eventfd-signaling-and-wakeups.patch new file mode 100644 index 00000000000..f651bebcea7 --- /dev/null +++ b/next-5.10/io_uring-pass-in-epoll_uring_wake-for-eventfd-signaling-and-wakeups.patch @@ -0,0 +1,82 @@ +From 4029c28b8d6792778bdadc4e431be30302ff4a3b Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 23 Dec 2022 07:04:49 -0700 +Subject: io_uring: pass in EPOLL_URING_WAKE for eventfd signaling and wakeups + +From: Jens Axboe + +[ Upstream commit 4464853277d0ccdb9914608dd1332f0fa2f9846f ] + +Pass in EPOLL_URING_WAKE when signaling eventfd or doing poll related +wakups, so that we can check for a circular event dependency between +eventfd and epoll. If this flag is set when our wakeup handlers are +called, then we know we have a dependency that needs to terminate +multishot requests. + +eventfd and epoll are the only such possible dependencies. + +Cc: stable@vger.kernel.org # 6.0 +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + io_uring/io_uring.c | 27 ++++++++++++++++++++------- + 1 file changed, 20 insertions(+), 7 deletions(-) + +--- a/io_uring/io_uring.c ++++ b/io_uring/io_uring.c +@@ -1626,13 +1626,15 @@ static void io_cqring_ev_posted(struct i + * wake as many waiters as we need to. + */ + if (wq_has_sleeper(&ctx->cq_wait)) +- wake_up_all(&ctx->cq_wait); ++ __wake_up(&ctx->cq_wait, TASK_NORMAL, 0, ++ poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); + if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait)) + wake_up(&ctx->sq_data->wait); + if (io_should_trigger_evfd(ctx)) +- eventfd_signal(ctx->cq_ev_fd, 1); ++ eventfd_signal_mask(ctx->cq_ev_fd, 1, EPOLL_URING_WAKE); + if (waitqueue_active(&ctx->poll_wait)) +- wake_up_interruptible(&ctx->poll_wait); ++ __wake_up(&ctx->poll_wait, TASK_INTERRUPTIBLE, 0, ++ poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); + } + + static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) +@@ -1642,12 +1644,14 @@ static void io_cqring_ev_posted_iopoll(s + + if (ctx->flags & IORING_SETUP_SQPOLL) { + if (waitqueue_active(&ctx->cq_wait)) +- wake_up_all(&ctx->cq_wait); ++ __wake_up(&ctx->cq_wait, TASK_NORMAL, 0, ++ poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); + } + if (io_should_trigger_evfd(ctx)) +- eventfd_signal(ctx->cq_ev_fd, 1); ++ eventfd_signal_mask(ctx->cq_ev_fd, 1, EPOLL_URING_WAKE); + if (waitqueue_active(&ctx->poll_wait)) +- wake_up_interruptible(&ctx->poll_wait); ++ __wake_up(&ctx->poll_wait, TASK_INTERRUPTIBLE, 0, ++ poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); + } + + /* Returns true if there are no backlogged entries after the flush */ +@@ -5477,8 +5481,17 @@ static int io_poll_wake(struct wait_queu + if (mask && !(mask & poll->events)) + return 0; + +- if (io_poll_get_ownership(req)) ++ if (io_poll_get_ownership(req)) { ++ /* ++ * If we trigger a multishot poll off our own wakeup path, ++ * disable multishot as there is a circular dependency between ++ * CQ posting and triggering the event. ++ */ ++ if (mask & EPOLL_URING_WAKE) ++ poll->events |= EPOLLONESHOT; ++ + __io_poll_execute(req, mask); ++ } + return 1; + } + diff --git a/queue-5.10/iommu-amd-fix-pci-device-refcount-leak-in-ppr_notifi.patch b/next-5.10/iommu-amd-fix-pci-device-refcount-leak-in-ppr_notifi.patch similarity index 100% rename from queue-5.10/iommu-amd-fix-pci-device-refcount-leak-in-ppr_notifi.patch rename to next-5.10/iommu-amd-fix-pci-device-refcount-leak-in-ppr_notifi.patch diff --git a/queue-5.10/iommu-fsl_pamu-fix-resource-leak-in-fsl_pamu_probe.patch b/next-5.10/iommu-fsl_pamu-fix-resource-leak-in-fsl_pamu_probe.patch similarity index 100% rename from queue-5.10/iommu-fsl_pamu-fix-resource-leak-in-fsl_pamu_probe.patch rename to next-5.10/iommu-fsl_pamu-fix-resource-leak-in-fsl_pamu_probe.patch diff --git a/queue-5.10/iommu-sun50i-consider-all-fault-sources-for-reset.patch b/next-5.10/iommu-sun50i-consider-all-fault-sources-for-reset.patch similarity index 100% rename from queue-5.10/iommu-sun50i-consider-all-fault-sources-for-reset.patch rename to next-5.10/iommu-sun50i-consider-all-fault-sources-for-reset.patch diff --git a/queue-5.10/iommu-sun50i-fix-flush-size.patch b/next-5.10/iommu-sun50i-fix-flush-size.patch similarity index 100% rename from queue-5.10/iommu-sun50i-fix-flush-size.patch rename to next-5.10/iommu-sun50i-fix-flush-size.patch diff --git a/queue-5.10/iommu-sun50i-fix-r-w-permission-check.patch b/next-5.10/iommu-sun50i-fix-r-w-permission-check.patch similarity index 100% rename from queue-5.10/iommu-sun50i-fix-r-w-permission-check.patch rename to next-5.10/iommu-sun50i-fix-r-w-permission-check.patch diff --git a/queue-5.10/iommu-sun50i-fix-reset-release.patch b/next-5.10/iommu-sun50i-fix-reset-release.patch similarity index 100% rename from queue-5.10/iommu-sun50i-fix-reset-release.patch rename to next-5.10/iommu-sun50i-fix-reset-release.patch diff --git a/queue-5.10/iommu-sun50i-remove-iommu_domain_identity.patch b/next-5.10/iommu-sun50i-remove-iommu_domain_identity.patch similarity index 100% rename from queue-5.10/iommu-sun50i-remove-iommu_domain_identity.patch rename to next-5.10/iommu-sun50i-remove-iommu_domain_identity.patch diff --git a/next-5.10/iov_iter-add-helper-to-save-iov_iter-state.patch b/next-5.10/iov_iter-add-helper-to-save-iov_iter-state.patch new file mode 100644 index 00000000000..f5d5927a7dc --- /dev/null +++ b/next-5.10/iov_iter-add-helper-to-save-iov_iter-state.patch @@ -0,0 +1,131 @@ +From 4d95aa07b6f1059990366f88112c48f217fbbc5d Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 10 Sep 2021 11:18:36 -0600 +Subject: iov_iter: add helper to save iov_iter state + +From: Jens Axboe + +[ Upstream commit 8fb0f47a9d7acf620d0fd97831b69da9bc5e22ed ] + +In an ideal world, when someone is passed an iov_iter and returns X bytes, +then X bytes would have been consumed/advanced from the iov_iter. But we +have use cases that always consume the entire iterator, a few examples +of that are iomap and bdev O_DIRECT. This means we cannot rely on the +state of the iov_iter once we've called ->read_iter() or ->write_iter(). + +This would be easier if we didn't always have to deal with truncate of +the iov_iter, as rewinding would be trivial without that. We recently +added a commit to track the truncate state, but that grew the iov_iter +by 8 bytes and wasn't the best solution. + +Implement a helper to save enough of the iov_iter state to sanely restore +it after we've called the read/write iterator helpers. This currently +only works for IOVEC/BVEC/KVEC as that's all we need, support for other +iterator types are left as an exercise for the reader. + +Link: https://lore.kernel.org/linux-fsdevel/CAHk-=wiacKV4Gh-MYjteU0LwNBSGpWrK-Ov25HdqB1ewinrFPg@mail.gmail.com/ +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/uio.h | 15 +++++++++++++++ + lib/iov_iter.c | 52 +++++++++++++++++++++++++++++++++------------------- + 2 files changed, 48 insertions(+), 19 deletions(-) + +--- a/include/linux/uio.h ++++ b/include/linux/uio.h +@@ -26,6 +26,12 @@ enum iter_type { + ITER_DISCARD = 64, + }; + ++struct iov_iter_state { ++ size_t iov_offset; ++ size_t count; ++ unsigned long nr_segs; ++}; ++ + struct iov_iter { + /* + * Bit 0 is the read/write bit, set if we're writing. +@@ -55,6 +61,14 @@ static inline enum iter_type iov_iter_ty + return i->type & ~(READ | WRITE); + } + ++static inline void iov_iter_save_state(struct iov_iter *iter, ++ struct iov_iter_state *state) ++{ ++ state->iov_offset = iter->iov_offset; ++ state->count = iter->count; ++ state->nr_segs = iter->nr_segs; ++} ++ + static inline bool iter_is_iovec(const struct iov_iter *i) + { + return iov_iter_type(i) == ITER_IOVEC; +@@ -226,6 +240,7 @@ ssize_t iov_iter_get_pages(struct iov_it + ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, struct page ***pages, + size_t maxsize, size_t *start); + int iov_iter_npages(const struct iov_iter *i, int maxpages); ++void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state); + + const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags); + +--- a/lib/iov_iter.c ++++ b/lib/iov_iter.c +@@ -1836,24 +1836,38 @@ int import_single_range(int rw, void __u + } + EXPORT_SYMBOL(import_single_range); + +-int iov_iter_for_each_range(struct iov_iter *i, size_t bytes, +- int (*f)(struct kvec *vec, void *context), +- void *context) ++/** ++ * iov_iter_restore() - Restore a &struct iov_iter to the same state as when ++ * iov_iter_save_state() was called. ++ * ++ * @i: &struct iov_iter to restore ++ * @state: state to restore from ++ * ++ * Used after iov_iter_save_state() to bring restore @i, if operations may ++ * have advanced it. ++ * ++ * Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC ++ */ ++void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) + { +- struct kvec w; +- int err = -EINVAL; +- if (!bytes) +- return 0; +- +- iterate_all_kinds(i, bytes, v, -EINVAL, ({ +- w.iov_base = kmap(v.bv_page) + v.bv_offset; +- w.iov_len = v.bv_len; +- err = f(&w, context); +- kunmap(v.bv_page); +- err;}), ({ +- w = v; +- err = f(&w, context);}) +- ) +- return err; ++ if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) && ++ !iov_iter_is_kvec(i)) ++ return; ++ i->iov_offset = state->iov_offset; ++ i->count = state->count; ++ /* ++ * For the *vec iters, nr_segs + iov is constant - if we increment ++ * the vec, then we also decrement the nr_segs count. Hence we don't ++ * need to track both of these, just one is enough and we can deduct ++ * the other from that. ITER_KVEC and ITER_IOVEC are the same struct ++ * size, so we can just increment the iov pointer as they are unionzed. ++ * ITER_BVEC _may_ be the same size on some archs, but on others it is ++ * not. Be safe and handle it separately. ++ */ ++ BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec)); ++ if (iov_iter_is_bvec(i)) ++ i->bvec -= state->nr_segs - i->nr_segs; ++ else ++ i->iov -= state->nr_segs - i->nr_segs; ++ i->nr_segs = state->nr_segs; + } +-EXPORT_SYMBOL(iov_iter_for_each_range); diff --git a/queue-5.10/ipmi-fix-memleak-when-unload-ipmi-driver.patch b/next-5.10/ipmi-fix-memleak-when-unload-ipmi-driver.patch similarity index 100% rename from queue-5.10/ipmi-fix-memleak-when-unload-ipmi-driver.patch rename to next-5.10/ipmi-fix-memleak-when-unload-ipmi-driver.patch diff --git a/queue-5.10/irqchip-gic-pm-use-pm_runtime_resume_and_get-in-gic_.patch b/next-5.10/irqchip-gic-pm-use-pm_runtime_resume_and_get-in-gic_.patch similarity index 100% rename from queue-5.10/irqchip-gic-pm-use-pm_runtime_resume_and_get-in-gic_.patch rename to next-5.10/irqchip-gic-pm-use-pm_runtime_resume_and_get-in-gic_.patch diff --git a/queue-5.10/kbuild-refactor-single-builds-of-.ko.patch b/next-5.10/kbuild-refactor-single-builds-of-.ko.patch similarity index 100% rename from queue-5.10/kbuild-refactor-single-builds-of-.ko.patch rename to next-5.10/kbuild-refactor-single-builds-of-.ko.patch diff --git a/queue-5.10/kbuild-remove-unneeded-mkdir-for-external-modules_in.patch b/next-5.10/kbuild-remove-unneeded-mkdir-for-external-modules_in.patch similarity index 100% rename from queue-5.10/kbuild-remove-unneeded-mkdir-for-external-modules_in.patch rename to next-5.10/kbuild-remove-unneeded-mkdir-for-external-modules_in.patch diff --git a/queue-5.10/kbuild-unify-modules-_install-for-in-tree-and-extern.patch b/next-5.10/kbuild-unify-modules-_install-for-in-tree-and-extern.patch similarity index 100% rename from queue-5.10/kbuild-unify-modules-_install-for-in-tree-and-extern.patch rename to next-5.10/kbuild-unify-modules-_install-for-in-tree-and-extern.patch diff --git a/queue-5.10/kcsan-instrument-memcpy-memset-memmove-with-newer-clang.patch b/next-5.10/kcsan-instrument-memcpy-memset-memmove-with-newer-clang.patch similarity index 100% rename from queue-5.10/kcsan-instrument-memcpy-memset-memmove-with-newer-clang.patch rename to next-5.10/kcsan-instrument-memcpy-memset-memmove-with-newer-clang.patch diff --git a/next-5.10/kernel-allow-fork-with-tif_notify_signal-pending.patch b/next-5.10/kernel-allow-fork-with-tif_notify_signal-pending.patch new file mode 100644 index 00000000000..894afb5d84f --- /dev/null +++ b/next-5.10/kernel-allow-fork-with-tif_notify_signal-pending.patch @@ -0,0 +1,38 @@ +From b6ea49a1293ab63ef65553c576b8d919bdd3d42e Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Mon, 22 Mar 2021 09:39:12 -0600 +Subject: kernel: allow fork with TIF_NOTIFY_SIGNAL pending + +From: Jens Axboe + +[ Upstream commit 66ae0d1e2d9fe6ec70e73fcfdcf4b390e271c1ac ] + +fork() fails if signal_pending() is true, but there are two conditions +that can lead to that: + +1) An actual signal is pending. We want fork to fail for that one, like + we always have. + +2) TIF_NOTIFY_SIGNAL is pending, because the task has pending task_work. + We don't need to make it fail for that case. + +Allow fork() to proceed if just task_work is pending, by changing the +signal_pending() check to task_sigpending(). + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + kernel/fork.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -1942,7 +1942,7 @@ static __latent_entropy struct task_stru + recalc_sigpending(); + spin_unlock_irq(¤t->sighand->siglock); + retval = -ERESTARTNOINTR; +- if (signal_pending(current)) ++ if (task_sigpending(current)) + goto fork_out; + + retval = -ENOMEM; diff --git a/next-5.10/kernel-don-t-call-do_exit-for-pf_io_worker-threads.patch b/next-5.10/kernel-don-t-call-do_exit-for-pf_io_worker-threads.patch new file mode 100644 index 00000000000..56ab9362794 --- /dev/null +++ b/next-5.10/kernel-don-t-call-do_exit-for-pf_io_worker-threads.patch @@ -0,0 +1,46 @@ +From e11f9f61752f252ee19e9c0dee22218f9078a75b Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 26 Mar 2021 08:57:10 -0600 +Subject: kernel: don't call do_exit() for PF_IO_WORKER threads + +From: Jens Axboe + +[ Upstream commit 10442994ba195efef6fdcc0c3699e4633cb5161b ] + +Right now we're never calling get_signal() from PF_IO_WORKER threads, but +in preparation for doing so, don't handle a fatal signal for them. The +workers have state they need to cleanup when exiting, so just return +instead of calling do_exit() on their behalf. The threads themselves will +detect a fatal signal and do proper shutdown. + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + kernel/signal.c | 10 +++++++++- + 1 file changed, 9 insertions(+), 1 deletion(-) + +--- a/kernel/signal.c ++++ b/kernel/signal.c +@@ -2755,13 +2755,21 @@ relock: + } + + /* ++ * PF_IO_WORKER threads will catch and exit on fatal signals ++ * themselves. They have cleanup that must be performed, so ++ * we cannot call do_exit() on their behalf. ++ */ ++ if (current->flags & PF_IO_WORKER) ++ goto out; ++ ++ /* + * Death signals, no core dump. + */ + do_group_exit(ksig->info.si_signo); + /* NOTREACHED */ + } + spin_unlock_irq(&sighand->siglock); +- ++out: + ksig->sig = signr; + return ksig->sig > 0; + } diff --git a/next-5.10/kernel-provide-create_io_thread-helper.patch b/next-5.10/kernel-provide-create_io_thread-helper.patch new file mode 100644 index 00000000000..bfb70a9300b --- /dev/null +++ b/next-5.10/kernel-provide-create_io_thread-helper.patch @@ -0,0 +1,87 @@ +From a3db03dc7a4f33de20188638d5b193c2946c5dfe Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Thu, 4 Mar 2021 12:21:05 -0700 +Subject: kernel: provide create_io_thread() helper + +From: Jens Axboe + +[ Upstream commit cc440e8738e5c875297ac0e90316745093be7e28 ] + +Provide a generic helper for setting up an io_uring worker. Returns a +task_struct so that the caller can do whatever setup is needed, then call +wake_up_new_task() to kick it into gear. + +Add a kernel_clone_args member, io_thread, which tells copy_process() to +mark the task with PF_IO_WORKER. + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/sched/task.h | 2 ++ + kernel/fork.c | 30 ++++++++++++++++++++++++++++++ + 2 files changed, 32 insertions(+) + +--- a/include/linux/sched/task.h ++++ b/include/linux/sched/task.h +@@ -31,6 +31,7 @@ struct kernel_clone_args { + /* Number of elements in *set_tid */ + size_t set_tid_size; + int cgroup; ++ int io_thread; + struct cgroup *cgrp; + struct css_set *cset; + }; +@@ -85,6 +86,7 @@ extern void exit_files(struct task_struc + extern void exit_itimers(struct task_struct *); + + extern pid_t kernel_clone(struct kernel_clone_args *kargs); ++struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node); + struct task_struct *fork_idle(int); + struct mm_struct *copy_init_mm(void); + extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -1949,6 +1949,8 @@ static __latent_entropy struct task_stru + p = dup_task_struct(current, node); + if (!p) + goto fork_out; ++ if (args->io_thread) ++ p->flags |= PF_IO_WORKER; + + /* + * This _must_ happen before we call free_task(), i.e. before we jump +@@ -2416,6 +2418,34 @@ struct mm_struct *copy_init_mm(void) + } + + /* ++ * This is like kernel_clone(), but shaved down and tailored to just ++ * creating io_uring workers. It returns a created task, or an error pointer. ++ * The returned task is inactive, and the caller must fire it up through ++ * wake_up_new_task(p). All signals are blocked in the created task. ++ */ ++struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) ++{ ++ unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD| ++ CLONE_IO; ++ struct kernel_clone_args args = { ++ .flags = ((lower_32_bits(flags) | CLONE_VM | ++ CLONE_UNTRACED) & ~CSIGNAL), ++ .exit_signal = (lower_32_bits(flags) & CSIGNAL), ++ .stack = (unsigned long)fn, ++ .stack_size = (unsigned long)arg, ++ .io_thread = 1, ++ }; ++ struct task_struct *tsk; ++ ++ tsk = copy_process(NULL, 0, node, &args); ++ if (!IS_ERR(tsk)) { ++ sigfillset(&tsk->blocked); ++ sigdelsetmask(&tsk->blocked, sigmask(SIGKILL)); ++ } ++ return tsk; ++} ++ ++/* + * Ok, this is the main fork-routine. + * + * It copies the process, and if successful kick-starts diff --git a/next-5.10/kernel-remove-checking-for-tif_notify_signal.patch b/next-5.10/kernel-remove-checking-for-tif_notify_signal.patch new file mode 100644 index 00000000000..0fe2ab88679 --- /dev/null +++ b/next-5.10/kernel-remove-checking-for-tif_notify_signal.patch @@ -0,0 +1,95 @@ +From 16021acf95cec0ada90dae2b8409ac234c678f3b Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 16:04:39 -0600 +Subject: kernel: remove checking for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit e296dc4996b8094ccde45d19090d804c4103513e ] + +It's available everywhere now, no need to check or add dummy defines. + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/entry-common.h | 4 ---- + include/linux/sched/signal.h | 2 -- + include/linux/tracehook.h | 4 ---- + kernel/signal.c | 2 -- + 4 files changed, 12 deletions(-) + +--- a/include/linux/entry-common.h ++++ b/include/linux/entry-common.h +@@ -37,10 +37,6 @@ + # define _TIF_UPROBE (0) + #endif + +-#ifndef _TIF_NOTIFY_SIGNAL +-# define _TIF_NOTIFY_SIGNAL (0) +-#endif +- + /* + * TIF flags handled in syscall_enter_from_user_mode() + */ +--- a/include/linux/sched/signal.h ++++ b/include/linux/sched/signal.h +@@ -361,7 +361,6 @@ static inline int task_sigpending(struct + + static inline int signal_pending(struct task_struct *p) + { +-#if defined(TIF_NOTIFY_SIGNAL) + /* + * TIF_NOTIFY_SIGNAL isn't really a signal, but it requires the same + * behavior in terms of ensuring that we break out of wait loops +@@ -369,7 +368,6 @@ static inline int signal_pending(struct + */ + if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL))) + return 1; +-#endif + return task_sigpending(p); + } + +--- a/include/linux/tracehook.h ++++ b/include/linux/tracehook.h +@@ -205,12 +205,10 @@ static inline void tracehook_notify_resu + */ + static inline void tracehook_notify_signal(void) + { +-#if defined(TIF_NOTIFY_SIGNAL) + clear_thread_flag(TIF_NOTIFY_SIGNAL); + smp_mb__after_atomic(); + if (current->task_works) + task_work_run(); +-#endif + } + + /* +@@ -218,11 +216,9 @@ static inline void tracehook_notify_sign + */ + static inline void set_notify_signal(struct task_struct *task) + { +-#if defined(TIF_NOTIFY_SIGNAL) + if (!test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) && + !wake_up_state(task, TASK_INTERRUPTIBLE)) + kick_process(task); +-#endif + } + + #endif /* */ +--- a/kernel/signal.c ++++ b/kernel/signal.c +@@ -2525,14 +2525,12 @@ bool get_signal(struct ksignal *ksig) + * that the arch handlers don't all have to do it. If we get here + * without TIF_SIGPENDING, just exit after running signal work. + */ +-#ifdef TIF_NOTIFY_SIGNAL + if (!IS_ENABLED(CONFIG_GENERIC_ENTRY)) { + if (test_thread_flag(TIF_NOTIFY_SIGNAL)) + tracehook_notify_signal(); + if (!task_sigpending(current)) + return false; + } +-#endif + + if (unlikely(uprobe_deny_signal())) + return false; diff --git a/next-5.10/kernel-stop-masking-signals-in-create_io_thread.patch b/next-5.10/kernel-stop-masking-signals-in-create_io_thread.patch new file mode 100644 index 00000000000..27d9661c8e2 --- /dev/null +++ b/next-5.10/kernel-stop-masking-signals-in-create_io_thread.patch @@ -0,0 +1,56 @@ +From 3ba2de08db37474986cabdd4e675b8e3bfc5df5a Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 26 Mar 2021 09:05:22 -0600 +Subject: kernel: stop masking signals in create_io_thread() + +From: Jens Axboe + +[ Upstream commit b16b3855d89fba640996fefdd3a113c0aa0e380d ] + +This is racy - move the blocking into when the task is created and +we're marking it as PF_IO_WORKER anyway. The IO threads are now +prepared to handle signals like SIGSTOP as well, so clear that from +the mask to allow proper stopping of IO threads. + +Acked-by: "Eric W. Biederman" +Reported-by: Oleg Nesterov +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + kernel/fork.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -1949,8 +1949,14 @@ static __latent_entropy struct task_stru + p = dup_task_struct(current, node); + if (!p) + goto fork_out; +- if (args->io_thread) ++ if (args->io_thread) { ++ /* ++ * Mark us an IO worker, and block any signal that isn't ++ * fatal or STOP ++ */ + p->flags |= PF_IO_WORKER; ++ siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP)); ++ } + + /* + * This _must_ happen before we call free_task(), i.e. before we jump +@@ -2435,14 +2441,8 @@ struct task_struct *create_io_thread(int + .stack_size = (unsigned long)arg, + .io_thread = 1, + }; +- struct task_struct *tsk; + +- tsk = copy_process(NULL, 0, node, &args); +- if (!IS_ERR(tsk)) { +- sigfillset(&tsk->blocked); +- sigdelsetmask(&tsk->blocked, sigmask(SIGKILL)); +- } +- return tsk; ++ return copy_process(NULL, 0, node, &args); + } + + /* diff --git a/queue-5.10/lib-debugobjects-fix-stat-count-and-optimize-debug_o.patch b/next-5.10/lib-debugobjects-fix-stat-count-and-optimize-debug_o.patch similarity index 100% rename from queue-5.10/lib-debugobjects-fix-stat-count-and-optimize-debug_o.patch rename to next-5.10/lib-debugobjects-fix-stat-count-and-optimize-debug_o.patch diff --git a/queue-5.10/lib-fonts-fix-undefined-behavior-in-bit-shift-for-ge.patch b/next-5.10/lib-fonts-fix-undefined-behavior-in-bit-shift-for-ge.patch similarity index 100% rename from queue-5.10/lib-fonts-fix-undefined-behavior-in-bit-shift-for-ge.patch rename to next-5.10/lib-fonts-fix-undefined-behavior-in-bit-shift-for-ge.patch diff --git a/queue-5.10/lib-notifier-error-inject-fix-error-when-writing-err.patch b/next-5.10/lib-notifier-error-inject-fix-error-when-writing-err.patch similarity index 100% rename from queue-5.10/lib-notifier-error-inject-fix-error-when-writing-err.patch rename to next-5.10/lib-notifier-error-inject-fix-error-when-writing-err.patch diff --git a/queue-5.10/libbpf-avoid-enum-forward-declarations-in-public-api.patch b/next-5.10/libbpf-avoid-enum-forward-declarations-in-public-api.patch similarity index 100% rename from queue-5.10/libbpf-avoid-enum-forward-declarations-in-public-api.patch rename to next-5.10/libbpf-avoid-enum-forward-declarations-in-public-api.patch diff --git a/queue-5.10/libbpf-fix-null-pointer-dereference-in-find_prog_by_.patch b/next-5.10/libbpf-fix-null-pointer-dereference-in-find_prog_by_.patch similarity index 100% rename from queue-5.10/libbpf-fix-null-pointer-dereference-in-find_prog_by_.patch rename to next-5.10/libbpf-fix-null-pointer-dereference-in-find_prog_by_.patch diff --git a/queue-5.10/libbpf-fix-use-after-free-in-btf_dump_name_dups.patch b/next-5.10/libbpf-fix-use-after-free-in-btf_dump_name_dups.patch similarity index 100% rename from queue-5.10/libbpf-fix-use-after-free-in-btf_dump_name_dups.patch rename to next-5.10/libbpf-fix-use-after-free-in-btf_dump_name_dups.patch diff --git a/queue-5.10/libfs-add-define_simple_attribute_signed-for-signed-.patch b/next-5.10/libfs-add-define_simple_attribute_signed-for-signed-.patch similarity index 100% rename from queue-5.10/libfs-add-define_simple_attribute_signed-for-signed-.patch rename to next-5.10/libfs-add-define_simple_attribute_signed-for-signed-.patch diff --git a/queue-5.10/loadpin-ignore-the-contents-argument-of-the-lsm-hook.patch b/next-5.10/loadpin-ignore-the-contents-argument-of-the-lsm-hook.patch similarity index 100% rename from queue-5.10/loadpin-ignore-the-contents-argument-of-the-lsm-hook.patch rename to next-5.10/loadpin-ignore-the-contents-argument-of-the-lsm-hook.patch diff --git a/next-5.10/m68k-add-support-for-tif_notify_signal.patch b/next-5.10/m68k-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..fcd9445d222 --- /dev/null +++ b/next-5.10/m68k-add-support-for-tif_notify_signal.patch @@ -0,0 +1,42 @@ +From da81a523c0a6278891e020596edc7a32cb593922 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 15:10:55 -0600 +Subject: m68k: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit e660653cd9f2df470d156c249631f68b9dee51ee ] + +Wire up TIF_NOTIFY_SIGNAL handling for m68k. + +Cc: linux-m68k@lists.linux-m68k.org +Acked-by: Geert Uytterhoeven +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/m68k/include/asm/thread_info.h | 1 + + arch/m68k/kernel/signal.c | 3 ++- + 2 files changed, 3 insertions(+), 1 deletion(-) + +--- a/arch/m68k/include/asm/thread_info.h ++++ b/arch/m68k/include/asm/thread_info.h +@@ -60,6 +60,7 @@ static inline struct thread_info *curren + * bits 0-7 are tested at every exception exit + * bits 8-15 are also tested at syscall exit + */ ++#define TIF_NOTIFY_SIGNAL 4 + #define TIF_NOTIFY_RESUME 5 /* callback before returning to user */ + #define TIF_SIGPENDING 6 /* signal pending */ + #define TIF_NEED_RESCHED 7 /* rescheduling necessary */ +--- a/arch/m68k/kernel/signal.c ++++ b/arch/m68k/kernel/signal.c +@@ -1129,7 +1129,8 @@ static void do_signal(struct pt_regs *re + + void do_notify_resume(struct pt_regs *regs) + { +- if (test_thread_flag(TIF_SIGPENDING)) ++ if (test_thread_flag(TIF_NOTIFY_SIGNAL) || ++ test_thread_flag(TIF_SIGPENDING)) + do_signal(regs); + + if (test_thread_flag(TIF_NOTIFY_RESUME)) diff --git a/queue-5.10/macintosh-fix-possible-memory-leak-in-macio_add_one_.patch b/next-5.10/macintosh-fix-possible-memory-leak-in-macio_add_one_.patch similarity index 100% rename from queue-5.10/macintosh-fix-possible-memory-leak-in-macio_add_one_.patch rename to next-5.10/macintosh-fix-possible-memory-leak-in-macio_add_one_.patch diff --git a/queue-5.10/macintosh-macio-adb-check-the-return-value-of-iorema.patch b/next-5.10/macintosh-macio-adb-check-the-return-value-of-iorema.patch similarity index 100% rename from queue-5.10/macintosh-macio-adb-check-the-return-value-of-iorema.patch rename to next-5.10/macintosh-macio-adb-check-the-return-value-of-iorema.patch diff --git a/queue-5.10/mailbox-zynq-ipi-fix-error-handling-while-device_reg.patch b/next-5.10/mailbox-zynq-ipi-fix-error-handling-while-device_reg.patch similarity index 100% rename from queue-5.10/mailbox-zynq-ipi-fix-error-handling-while-device_reg.patch rename to next-5.10/mailbox-zynq-ipi-fix-error-handling-while-device_reg.patch diff --git a/next-5.10/make-sure-nd-path.mnt-and-nd-path.dentry-are-always-valid-pointers.patch b/next-5.10/make-sure-nd-path.mnt-and-nd-path.dentry-are-always-valid-pointers.patch new file mode 100644 index 00000000000..e75795ad5c9 --- /dev/null +++ b/next-5.10/make-sure-nd-path.mnt-and-nd-path.dentry-are-always-valid-pointers.patch @@ -0,0 +1,59 @@ +From 597ed7633bde8abc24c0cea5fbbfd824200d3251 Mon Sep 17 00:00:00 2001 +From: Al Viro +Date: Tue, 6 Apr 2021 12:33:07 -0400 +Subject: Make sure nd->path.mnt and nd->path.dentry are always valid pointers + +From: Al Viro + +[ Upstream commit 7d01ef7585c07afaf487759a48486228cd065726 ] + +Initialize them in set_nameidata() and make sure that terminate_walk() clears them +once the pointers become potentially invalid (i.e. we leave RCU mode or drop them +in non-RCU one). Currently we have "path_init() always initializes them and nobody +accesses them outside of path_init()/terminate_walk() segments", which is asking +for trouble. + +With that change we would have nd->path.{mnt,dentry} + 1) always valid - NULL or pointing to currently allocated objects. + 2) non-NULL while we are successfully walking + 3) NULL when we are not walking at all + 4) contributing to refcounts whenever non-NULL outside of RCU mode. + +Fixes: 6c6ec2b0a3e0 ("fs: add support for LOOKUP_CACHED") +Reported-by: syzbot+c88a7030da47945a3cc3@syzkaller.appspotmail.com +Tested-by: Christian Brauner +Signed-off-by: Al Viro +Signed-off-by: Greg Kroah-Hartman +--- + fs/namei.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +--- a/fs/namei.c ++++ b/fs/namei.c +@@ -529,6 +529,8 @@ static void set_nameidata(struct nameida + p->stack = p->internal; + p->dfd = dfd; + p->name = name; ++ p->path.mnt = NULL; ++ p->path.dentry = NULL; + p->total_link_count = old ? old->total_link_count : 0; + p->saved = old; + current->nameidata = p; +@@ -602,6 +604,8 @@ static void terminate_walk(struct nameid + rcu_read_unlock(); + } + nd->depth = 0; ++ nd->path.mnt = NULL; ++ nd->path.dentry = NULL; + } + + /* path_put is needed afterwards regardless of success or failure */ +@@ -2243,8 +2247,6 @@ static const char *path_init(struct name + } + + nd->root.mnt = NULL; +- nd->path.mnt = NULL; +- nd->path.dentry = NULL; + + /* Absolute pathname -- fetch the root (LOOKUP_IN_ROOT uses nd->dfd). */ + if (*s == '/' && !(flags & LOOKUP_IN_ROOT)) { diff --git a/queue-5.10/mcb-mcb-parse-fix-error-handing-in-chameleon_parse_g.patch b/next-5.10/mcb-mcb-parse-fix-error-handing-in-chameleon_parse_g.patch similarity index 100% rename from queue-5.10/mcb-mcb-parse-fix-error-handing-in-chameleon_parse_g.patch rename to next-5.10/mcb-mcb-parse-fix-error-handing-in-chameleon_parse_g.patch diff --git a/queue-5.10/md-fix-a-crash-in-mempool_free.patch b/next-5.10/md-fix-a-crash-in-mempool_free.patch similarity index 100% rename from queue-5.10/md-fix-a-crash-in-mempool_free.patch rename to next-5.10/md-fix-a-crash-in-mempool_free.patch diff --git a/queue-5.10/md-raid1-stop-mdx_raid1-thread-when-raid1-array-run-.patch b/next-5.10/md-raid1-stop-mdx_raid1-thread-when-raid1-array-run-.patch similarity index 100% rename from queue-5.10/md-raid1-stop-mdx_raid1-thread-when-raid1-array-run-.patch rename to next-5.10/md-raid1-stop-mdx_raid1-thread-when-raid1-array-run-.patch diff --git a/queue-5.10/media-c8sectpfe-add-of_node_put-when-breaking-out-of.patch b/next-5.10/media-c8sectpfe-add-of_node_put-when-breaking-out-of.patch similarity index 100% rename from queue-5.10/media-c8sectpfe-add-of_node_put-when-breaking-out-of.patch rename to next-5.10/media-c8sectpfe-add-of_node_put-when-breaking-out-of.patch diff --git a/queue-5.10/media-camss-clean-up-received-buffers-on-failed-star.patch b/next-5.10/media-camss-clean-up-received-buffers-on-failed-star.patch similarity index 100% rename from queue-5.10/media-camss-clean-up-received-buffers-on-failed-star.patch rename to next-5.10/media-camss-clean-up-received-buffers-on-failed-star.patch diff --git a/queue-5.10/media-coda-add-check-for-dcoda_iram_alloc.patch b/next-5.10/media-coda-add-check-for-dcoda_iram_alloc.patch similarity index 100% rename from queue-5.10/media-coda-add-check-for-dcoda_iram_alloc.patch rename to next-5.10/media-coda-add-check-for-dcoda_iram_alloc.patch diff --git a/queue-5.10/media-coda-add-check-for-kmalloc.patch b/next-5.10/media-coda-add-check-for-kmalloc.patch similarity index 100% rename from queue-5.10/media-coda-add-check-for-kmalloc.patch rename to next-5.10/media-coda-add-check-for-kmalloc.patch diff --git a/queue-5.10/media-coda-jpeg-add-check-for-kmalloc.patch b/next-5.10/media-coda-jpeg-add-check-for-kmalloc.patch similarity index 100% rename from queue-5.10/media-coda-jpeg-add-check-for-kmalloc.patch rename to next-5.10/media-coda-jpeg-add-check-for-kmalloc.patch diff --git a/queue-5.10/media-dvb-core-fix-ignored-return-value-in-dvb_regis.patch b/next-5.10/media-dvb-core-fix-ignored-return-value-in-dvb_regis.patch similarity index 100% rename from queue-5.10/media-dvb-core-fix-ignored-return-value-in-dvb_regis.patch rename to next-5.10/media-dvb-core-fix-ignored-return-value-in-dvb_regis.patch diff --git a/queue-5.10/media-dvb-frontends-fix-leak-of-memory-fw.patch b/next-5.10/media-dvb-frontends-fix-leak-of-memory-fw.patch similarity index 100% rename from queue-5.10/media-dvb-frontends-fix-leak-of-memory-fw.patch rename to next-5.10/media-dvb-frontends-fix-leak-of-memory-fw.patch diff --git a/queue-5.10/media-dvb-usb-az6027-fix-null-ptr-deref-in-az6027_i2.patch b/next-5.10/media-dvb-usb-az6027-fix-null-ptr-deref-in-az6027_i2.patch similarity index 100% rename from queue-5.10/media-dvb-usb-az6027-fix-null-ptr-deref-in-az6027_i2.patch rename to next-5.10/media-dvb-usb-az6027-fix-null-ptr-deref-in-az6027_i2.patch diff --git a/queue-5.10/media-dvb-usb-fix-memory-leak-in-dvb_usb_adapter_ini.patch b/next-5.10/media-dvb-usb-fix-memory-leak-in-dvb_usb_adapter_ini.patch similarity index 100% rename from queue-5.10/media-dvb-usb-fix-memory-leak-in-dvb_usb_adapter_ini.patch rename to next-5.10/media-dvb-usb-fix-memory-leak-in-dvb_usb_adapter_ini.patch diff --git a/queue-5.10/media-dvbdev-adopts-refcnt-to-avoid-uaf.patch b/next-5.10/media-dvbdev-adopts-refcnt-to-avoid-uaf.patch similarity index 100% rename from queue-5.10/media-dvbdev-adopts-refcnt-to-avoid-uaf.patch rename to next-5.10/media-dvbdev-adopts-refcnt-to-avoid-uaf.patch diff --git a/queue-5.10/media-dvbdev-fix-build-warning-due-to-comments.patch b/next-5.10/media-dvbdev-fix-build-warning-due-to-comments.patch similarity index 100% rename from queue-5.10/media-dvbdev-fix-build-warning-due-to-comments.patch rename to next-5.10/media-dvbdev-fix-build-warning-due-to-comments.patch diff --git a/queue-5.10/media-dvbdev-fix-refcnt-bug.patch b/next-5.10/media-dvbdev-fix-refcnt-bug.patch similarity index 100% rename from queue-5.10/media-dvbdev-fix-refcnt-bug.patch rename to next-5.10/media-dvbdev-fix-refcnt-bug.patch diff --git a/queue-5.10/media-exynos4-is-don-t-rely-on-the-v4l2_async_subdev.patch b/next-5.10/media-exynos4-is-don-t-rely-on-the-v4l2_async_subdev.patch similarity index 100% rename from queue-5.10/media-exynos4-is-don-t-rely-on-the-v4l2_async_subdev.patch rename to next-5.10/media-exynos4-is-don-t-rely-on-the-v4l2_async_subdev.patch diff --git a/queue-5.10/media-exynos4-is-use-v4l2_async_notifier_add_fwnode_.patch b/next-5.10/media-exynos4-is-use-v4l2_async_notifier_add_fwnode_.patch similarity index 100% rename from queue-5.10/media-exynos4-is-use-v4l2_async_notifier_add_fwnode_.patch rename to next-5.10/media-exynos4-is-use-v4l2_async_notifier_add_fwnode_.patch diff --git a/queue-5.10/media-i2c-ad5820-fix-error-path.patch b/next-5.10/media-i2c-ad5820-fix-error-path.patch similarity index 100% rename from queue-5.10/media-i2c-ad5820-fix-error-path.patch rename to next-5.10/media-i2c-ad5820-fix-error-path.patch diff --git a/queue-5.10/media-imon-fix-a-race-condition-in-send_packet.patch b/next-5.10/media-imon-fix-a-race-condition-in-send_packet.patch similarity index 100% rename from queue-5.10/media-imon-fix-a-race-condition-in-send_packet.patch rename to next-5.10/media-imon-fix-a-race-condition-in-send_packet.patch diff --git a/queue-5.10/media-platform-exynos4-is-fix-error-handling-in-fimc.patch b/next-5.10/media-platform-exynos4-is-fix-error-handling-in-fimc.patch similarity index 100% rename from queue-5.10/media-platform-exynos4-is-fix-error-handling-in-fimc.patch rename to next-5.10/media-platform-exynos4-is-fix-error-handling-in-fimc.patch diff --git a/queue-5.10/media-platform-exynos4-is-fix-return-value-check-in-.patch b/next-5.10/media-platform-exynos4-is-fix-return-value-check-in-.patch similarity index 100% rename from queue-5.10/media-platform-exynos4-is-fix-return-value-check-in-.patch rename to next-5.10/media-platform-exynos4-is-fix-return-value-check-in-.patch diff --git a/queue-5.10/media-s5p-mfc-add-variant-data-for-mfc-v7-hardware-f.patch b/next-5.10/media-s5p-mfc-add-variant-data-for-mfc-v7-hardware-f.patch similarity index 100% rename from queue-5.10/media-s5p-mfc-add-variant-data-for-mfc-v7-hardware-f.patch rename to next-5.10/media-s5p-mfc-add-variant-data-for-mfc-v7-hardware-f.patch diff --git a/queue-5.10/media-saa7164-fix-missing-pci_disable_device.patch b/next-5.10/media-saa7164-fix-missing-pci_disable_device.patch similarity index 100% rename from queue-5.10/media-saa7164-fix-missing-pci_disable_device.patch rename to next-5.10/media-saa7164-fix-missing-pci_disable_device.patch diff --git a/queue-5.10/media-si470x-fix-use-after-free-in-si470x_int_in_cal.patch b/next-5.10/media-si470x-fix-use-after-free-in-si470x_int_in_cal.patch similarity index 100% rename from queue-5.10/media-si470x-fix-use-after-free-in-si470x_int_in_cal.patch rename to next-5.10/media-si470x-fix-use-after-free-in-si470x_int_in_cal.patch diff --git a/queue-5.10/media-solo6x10-fix-possible-memory-leak-in-solo_sysf.patch b/next-5.10/media-solo6x10-fix-possible-memory-leak-in-solo_sysf.patch similarity index 100% rename from queue-5.10/media-solo6x10-fix-possible-memory-leak-in-solo_sysf.patch rename to next-5.10/media-solo6x10-fix-possible-memory-leak-in-solo_sysf.patch diff --git a/queue-5.10/media-videobuf-dma-contig-use-dma_mmap_coherent.patch b/next-5.10/media-videobuf-dma-contig-use-dma_mmap_coherent.patch similarity index 100% rename from queue-5.10/media-videobuf-dma-contig-use-dma_mmap_coherent.patch rename to next-5.10/media-videobuf-dma-contig-use-dma_mmap_coherent.patch diff --git a/queue-5.10/media-vidtv-fix-use-after-free-in-vidtv_bridge_dvb_i.patch b/next-5.10/media-vidtv-fix-use-after-free-in-vidtv_bridge_dvb_i.patch similarity index 100% rename from queue-5.10/media-vidtv-fix-use-after-free-in-vidtv_bridge_dvb_i.patch rename to next-5.10/media-vidtv-fix-use-after-free-in-vidtv_bridge_dvb_i.patch diff --git a/queue-5.10/media-vimc-fix-wrong-function-called-when-vimc_init-.patch b/next-5.10/media-vimc-fix-wrong-function-called-when-vimc_init-.patch similarity index 100% rename from queue-5.10/media-vimc-fix-wrong-function-called-when-vimc_init-.patch rename to next-5.10/media-vimc-fix-wrong-function-called-when-vimc_init-.patch diff --git a/queue-5.10/media-vivid-fix-compose-size-exceed-boundary.patch b/next-5.10/media-vivid-fix-compose-size-exceed-boundary.patch similarity index 100% rename from queue-5.10/media-vivid-fix-compose-size-exceed-boundary.patch rename to next-5.10/media-vivid-fix-compose-size-exceed-boundary.patch diff --git a/next-5.10/microblaze-add-support-for-tif_notify_signal.patch b/next-5.10/microblaze-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..d6d09495221 --- /dev/null +++ b/next-5.10/microblaze-add-support-for-tif_notify_signal.patch @@ -0,0 +1,49 @@ +From ab74702d525c52cbb5dea9fa5169f3f6f5eae2d7 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 15:13:57 -0600 +Subject: microblaze: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit f4ea089e429e0d366cd1a34a2cbe3c7b13d98d75 ] + +Wire up TIF_NOTIFY_SIGNAL handling for microblaze. + +Acked-by: Michal Simek +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/microblaze/include/asm/thread_info.h | 2 ++ + arch/microblaze/kernel/signal.c | 3 ++- + 2 files changed, 4 insertions(+), 1 deletion(-) + +--- a/arch/microblaze/include/asm/thread_info.h ++++ b/arch/microblaze/include/asm/thread_info.h +@@ -107,6 +107,7 @@ static inline struct thread_info *curren + #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ + /* restore singlestep on return to user mode */ + #define TIF_SINGLESTEP 4 ++#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ + #define TIF_MEMDIE 6 /* is terminating due to OOM killer */ + #define TIF_SYSCALL_AUDIT 9 /* syscall auditing active */ + #define TIF_SECCOMP 10 /* secure computing */ +@@ -119,6 +120,7 @@ static inline struct thread_info *curren + #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) + #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) + #define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) + #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) + #define _TIF_SECCOMP (1 << TIF_SECCOMP) +--- a/arch/microblaze/kernel/signal.c ++++ b/arch/microblaze/kernel/signal.c +@@ -313,7 +313,8 @@ static void do_signal(struct pt_regs *re + + asmlinkage void do_notify_resume(struct pt_regs *regs, int in_syscall) + { +- if (test_thread_flag(TIF_SIGPENDING)) ++ if (test_thread_flag(TIF_SIGPENDING) || ++ test_thread_flag(TIF_NOTIFY_SIGNAL)) + do_signal(regs, in_syscall); + + if (test_thread_flag(TIF_NOTIFY_RESUME)) diff --git a/next-5.10/mips-add-support-for-tif_notify_signal.patch b/next-5.10/mips-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..65f197fb496 --- /dev/null +++ b/next-5.10/mips-add-support-for-tif_notify_signal.patch @@ -0,0 +1,58 @@ +From 60559f867bc926e5595c13e4b85ea2fea168ecf8 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 15:16:02 -0600 +Subject: mips: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit f45c184bce15f4a314c0210519bc3b4aab408838 ] + +Wire up TIF_NOTIFY_SIGNAL handling for mips. + +Cc: linux-mips@vger.kernel.org +Acked-By: Thomas Bogendoerfer +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/mips/include/asm/thread_info.h | 4 +++- + arch/mips/kernel/signal.c | 2 +- + 2 files changed, 4 insertions(+), 2 deletions(-) + +--- a/arch/mips/include/asm/thread_info.h ++++ b/arch/mips/include/asm/thread_info.h +@@ -115,6 +115,7 @@ static inline struct thread_info *curren + #define TIF_SECCOMP 4 /* secure computing */ + #define TIF_NOTIFY_RESUME 5 /* callback before returning to user */ + #define TIF_UPROBE 6 /* breakpointed or singlestepping */ ++#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ + #define TIF_RESTORE_SIGMASK 9 /* restore signal mask in do_signal() */ + #define TIF_USEDFPU 16 /* FPU was used by this task this quantum (SMP) */ + #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ +@@ -139,6 +140,7 @@ static inline struct thread_info *curren + #define _TIF_SECCOMP (1< +Date: Fri, 9 Oct 2020 15:18:43 -0600 +Subject: nds32: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit b13e8bf615fe26fb6a6dfe1b5a1c65e1624dfee2 ] + +Wire up TIF_NOTIFY_SIGNAL handling for nds32. + +Cc: Nick Hu +Cc: Greentime Hu +Cc: Vincent Chen +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/nds32/include/asm/thread_info.h | 2 ++ + arch/nds32/kernel/ex-exit.S | 2 +- + arch/nds32/kernel/signal.c | 2 +- + 3 files changed, 4 insertions(+), 2 deletions(-) + +--- a/arch/nds32/include/asm/thread_info.h ++++ b/arch/nds32/include/asm/thread_info.h +@@ -48,6 +48,7 @@ struct thread_info { + #define TIF_NEED_RESCHED 2 + #define TIF_SINGLESTEP 3 + #define TIF_NOTIFY_RESUME 4 /* callback before returning to user */ ++#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ + #define TIF_SYSCALL_TRACE 8 + #define TIF_POLLING_NRFLAG 17 + #define TIF_MEMDIE 18 +@@ -57,6 +58,7 @@ struct thread_info { + #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) + #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) + #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + #define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) + #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) + #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) +--- a/arch/nds32/kernel/ex-exit.S ++++ b/arch/nds32/kernel/ex-exit.S +@@ -120,7 +120,7 @@ work_pending: + andi $p1, $r1, #_TIF_NEED_RESCHED + bnez $p1, work_resched + +- andi $p1, $r1, #_TIF_SIGPENDING|#_TIF_NOTIFY_RESUME ++ andi $p1, $r1, #_TIF_SIGPENDING|#_TIF_NOTIFY_RESUME|#_TIF_NOTIFY_SIGNAL + beqz $p1, no_work_pending + + move $r0, $sp ! 'regs' +--- a/arch/nds32/kernel/signal.c ++++ b/arch/nds32/kernel/signal.c +@@ -376,7 +376,7 @@ static void do_signal(struct pt_regs *re + asmlinkage void + do_notify_resume(struct pt_regs *regs, unsigned int thread_flags) + { +- if (thread_flags & _TIF_SIGPENDING) ++ if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs); + + if (thread_flags & _TIF_NOTIFY_RESUME) diff --git a/queue-5.10/net-add-a-helper-to-avoid-issues-with-hw-tx-timestam.patch b/next-5.10/net-add-a-helper-to-avoid-issues-with-hw-tx-timestam.patch similarity index 100% rename from queue-5.10/net-add-a-helper-to-avoid-issues-with-hw-tx-timestam.patch rename to next-5.10/net-add-a-helper-to-avoid-issues-with-hw-tx-timestam.patch diff --git a/next-5.10/net-add-accept-helper-not-installing-fd.patch b/next-5.10/net-add-accept-helper-not-installing-fd.patch new file mode 100644 index 00000000000..898eae8e5ff --- /dev/null +++ b/next-5.10/net-add-accept-helper-not-installing-fd.patch @@ -0,0 +1,142 @@ +From ab96e6863470dd3f13f9a894455b802cde9bcefc Mon Sep 17 00:00:00 2001 +From: Pavel Begunkov +Date: Wed, 25 Aug 2021 12:25:44 +0100 +Subject: net: add accept helper not installing fd + +From: Pavel Begunkov + +[ Upstream commit d32f89da7fa8ccc8b3fb8f909d61e42b9bc39329 ] + +Introduce and reuse a helper that acts similarly to __sys_accept4_file() +but returns struct file instead of installing file descriptor. Will be +used by io_uring. + +Signed-off-by: Pavel Begunkov +Acked-by: Jakub Kicinski +Signed-off-by: Jens Axboe +Acked-by: David S. Miller +Link: https://lore.kernel.org/r/c57b9e8e818d93683a3d24f8ca50ca038d1da8c4.1629888991.git.asml.silence@gmail.com +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/socket.h | 3 ++ + net/socket.c | 67 ++++++++++++++++++++++++++----------------------- + 2 files changed, 39 insertions(+), 31 deletions(-) + +--- a/include/linux/socket.h ++++ b/include/linux/socket.h +@@ -421,6 +421,9 @@ extern int __sys_accept4_file(struct fil + struct sockaddr __user *upeer_sockaddr, + int __user *upeer_addrlen, int flags, + unsigned long nofile); ++extern struct file *do_accept(struct file *file, unsigned file_flags, ++ struct sockaddr __user *upeer_sockaddr, ++ int __user *upeer_addrlen, int flags); + extern int __sys_accept4(int fd, struct sockaddr __user *upeer_sockaddr, + int __user *upeer_addrlen, int flags); + extern int __sys_socket(int family, int type, int protocol); +--- a/net/socket.c ++++ b/net/socket.c +@@ -1688,30 +1688,22 @@ SYSCALL_DEFINE2(listen, int, fd, int, ba + return __sys_listen(fd, backlog); + } + +-int __sys_accept4_file(struct file *file, unsigned file_flags, ++struct file *do_accept(struct file *file, unsigned file_flags, + struct sockaddr __user *upeer_sockaddr, +- int __user *upeer_addrlen, int flags, +- unsigned long nofile) ++ int __user *upeer_addrlen, int flags) + { + struct socket *sock, *newsock; + struct file *newfile; +- int err, len, newfd; ++ int err, len; + struct sockaddr_storage address; + +- if (flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) +- return -EINVAL; +- +- if (SOCK_NONBLOCK != O_NONBLOCK && (flags & SOCK_NONBLOCK)) +- flags = (flags & ~SOCK_NONBLOCK) | O_NONBLOCK; +- + sock = sock_from_file(file, &err); + if (!sock) +- goto out; ++ return ERR_PTR(err); + +- err = -ENFILE; + newsock = sock_alloc(); + if (!newsock) +- goto out; ++ return ERR_PTR(-ENFILE); + + newsock->type = sock->type; + newsock->ops = sock->ops; +@@ -1722,18 +1714,9 @@ int __sys_accept4_file(struct file *file + */ + __module_get(newsock->ops->owner); + +- newfd = __get_unused_fd_flags(flags, nofile); +- if (unlikely(newfd < 0)) { +- err = newfd; +- sock_release(newsock); +- goto out; +- } + newfile = sock_alloc_file(newsock, flags, sock->sk->sk_prot_creator->name); +- if (IS_ERR(newfile)) { +- err = PTR_ERR(newfile); +- put_unused_fd(newfd); +- goto out; +- } ++ if (IS_ERR(newfile)) ++ return newfile; + + err = security_socket_accept(sock, newsock); + if (err) +@@ -1758,16 +1741,38 @@ int __sys_accept4_file(struct file *file + } + + /* File flags are not inherited via accept() unlike another OSes. */ +- +- fd_install(newfd, newfile); +- err = newfd; +-out: +- return err; ++ return newfile; + out_fd: + fput(newfile); +- put_unused_fd(newfd); +- goto out; ++ return ERR_PTR(err); ++} ++ ++int __sys_accept4_file(struct file *file, unsigned file_flags, ++ struct sockaddr __user *upeer_sockaddr, ++ int __user *upeer_addrlen, int flags, ++ unsigned long nofile) ++{ ++ struct file *newfile; ++ int newfd; ++ ++ if (flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) ++ return -EINVAL; + ++ if (SOCK_NONBLOCK != O_NONBLOCK && (flags & SOCK_NONBLOCK)) ++ flags = (flags & ~SOCK_NONBLOCK) | O_NONBLOCK; ++ ++ newfd = __get_unused_fd_flags(flags, nofile); ++ if (unlikely(newfd < 0)) ++ return newfd; ++ ++ newfile = do_accept(file, file_flags, upeer_sockaddr, upeer_addrlen, ++ flags); ++ if (IS_ERR(newfile)) { ++ put_unused_fd(newfd); ++ return PTR_ERR(newfile); ++ } ++ fd_install(newfd, newfile); ++ return newfd; + } + + /* diff --git a/queue-5.10/net-add-atomic_long_t-to-net_device_stats-fields.patch b/next-5.10/net-add-atomic_long_t-to-net_device_stats-fields.patch similarity index 100% rename from queue-5.10/net-add-atomic_long_t-to-net_device_stats-fields.patch rename to next-5.10/net-add-atomic_long_t-to-net_device_stats-fields.patch diff --git a/queue-5.10/net-add-inline-function-skb_csum_is_sctp.patch b/next-5.10/net-add-inline-function-skb_csum_is_sctp.patch similarity index 100% rename from queue-5.10/net-add-inline-function-skb_csum_is_sctp.patch rename to next-5.10/net-add-inline-function-skb_csum_is_sctp.patch diff --git a/queue-5.10/net-amd-lance-don-t-call-dev_kfree_skb-under-spin_lo.patch b/next-5.10/net-amd-lance-don-t-call-dev_kfree_skb-under-spin_lo.patch similarity index 100% rename from queue-5.10/net-amd-lance-don-t-call-dev_kfree_skb-under-spin_lo.patch rename to next-5.10/net-amd-lance-don-t-call-dev_kfree_skb-under-spin_lo.patch diff --git a/queue-5.10/net-amd-xgbe-check-only-the-minimum-speed-for-active.patch b/next-5.10/net-amd-xgbe-check-only-the-minimum-speed-for-active.patch similarity index 100% rename from queue-5.10/net-amd-xgbe-check-only-the-minimum-speed-for-active.patch rename to next-5.10/net-amd-xgbe-check-only-the-minimum-speed-for-active.patch diff --git a/queue-5.10/net-amd-xgbe-fix-logic-around-active-and-passive-cab.patch b/next-5.10/net-amd-xgbe-fix-logic-around-active-and-passive-cab.patch similarity index 100% rename from queue-5.10/net-amd-xgbe-fix-logic-around-active-and-passive-cab.patch rename to next-5.10/net-amd-xgbe-fix-logic-around-active-and-passive-cab.patch diff --git a/queue-5.10/net-apple-bmac-don-t-call-dev_kfree_skb-under-spin_l.patch b/next-5.10/net-apple-bmac-don-t-call-dev_kfree_skb-under-spin_l.patch similarity index 100% rename from queue-5.10/net-apple-bmac-don-t-call-dev_kfree_skb-under-spin_l.patch rename to next-5.10/net-apple-bmac-don-t-call-dev_kfree_skb-under-spin_l.patch diff --git a/queue-5.10/net-apple-mace-don-t-call-dev_kfree_skb-under-spin_l.patch b/next-5.10/net-apple-mace-don-t-call-dev_kfree_skb-under-spin_l.patch similarity index 100% rename from queue-5.10/net-apple-mace-don-t-call-dev_kfree_skb-under-spin_l.patch rename to next-5.10/net-apple-mace-don-t-call-dev_kfree_skb-under-spin_l.patch diff --git a/queue-5.10/net-defxx-fix-missing-err-handling-in-dfx_init.patch b/next-5.10/net-defxx-fix-missing-err-handling-in-dfx_init.patch similarity index 100% rename from queue-5.10/net-defxx-fix-missing-err-handling-in-dfx_init.patch rename to next-5.10/net-defxx-fix-missing-err-handling-in-dfx_init.patch diff --git a/queue-5.10/net-emaclite-don-t-call-dev_kfree_skb-under-spin_loc.patch b/next-5.10/net-emaclite-don-t-call-dev_kfree_skb-under-spin_loc.patch similarity index 100% rename from queue-5.10/net-emaclite-don-t-call-dev_kfree_skb-under-spin_loc.patch rename to next-5.10/net-emaclite-don-t-call-dev_kfree_skb-under-spin_loc.patch diff --git a/queue-5.10/net-ethernet-dnet-don-t-call-dev_kfree_skb-under-spi.patch b/next-5.10/net-ethernet-dnet-don-t-call-dev_kfree_skb-under-spi.patch similarity index 100% rename from queue-5.10/net-ethernet-dnet-don-t-call-dev_kfree_skb-under-spi.patch rename to next-5.10/net-ethernet-dnet-don-t-call-dev_kfree_skb-under-spi.patch diff --git a/queue-5.10/net-ethernet-ti-fix-return-type-of-netcp_ndo_start_x.patch b/next-5.10/net-ethernet-ti-fix-return-type-of-netcp_ndo_start_x.patch similarity index 100% rename from queue-5.10/net-ethernet-ti-fix-return-type-of-netcp_ndo_start_x.patch rename to next-5.10/net-ethernet-ti-fix-return-type-of-netcp_ndo_start_x.patch diff --git a/queue-5.10/net-farsync-fix-kmemleak-when-rmmods-farsync.patch b/next-5.10/net-farsync-fix-kmemleak-when-rmmods-farsync.patch similarity index 100% rename from queue-5.10/net-farsync-fix-kmemleak-when-rmmods-farsync.patch rename to next-5.10/net-farsync-fix-kmemleak-when-rmmods-farsync.patch diff --git a/queue-5.10/net-hsr-generate-supervision-frame-without-hsr-prp-t.patch b/next-5.10/net-hsr-generate-supervision-frame-without-hsr-prp-t.patch similarity index 100% rename from queue-5.10/net-hsr-generate-supervision-frame-without-hsr-prp-t.patch rename to next-5.10/net-hsr-generate-supervision-frame-without-hsr-prp-t.patch diff --git a/queue-5.10/net-igc-use-skb_csum_is_sctp-instead-of-protocol-che.patch b/next-5.10/net-igc-use-skb_csum_is_sctp-instead-of-protocol-che.patch similarity index 100% rename from queue-5.10/net-igc-use-skb_csum_is_sctp-instead-of-protocol-che.patch rename to next-5.10/net-igc-use-skb_csum_is_sctp-instead-of-protocol-che.patch diff --git a/queue-5.10/net-lan9303-fix-read-error-execution-path.patch b/next-5.10/net-lan9303-fix-read-error-execution-path.patch similarity index 100% rename from queue-5.10/net-lan9303-fix-read-error-execution-path.patch rename to next-5.10/net-lan9303-fix-read-error-execution-path.patch diff --git a/queue-5.10/net-macsec-fix-net-device-access-prior-to-holding-a-.patch b/next-5.10/net-macsec-fix-net-device-access-prior-to-holding-a-.patch similarity index 100% rename from queue-5.10/net-macsec-fix-net-device-access-prior-to-holding-a-.patch rename to next-5.10/net-macsec-fix-net-device-access-prior-to-holding-a-.patch diff --git a/queue-5.10/net-proc-provide-proc_fs-n-fallback-for-proc_create_.patch b/next-5.10/net-proc-provide-proc_fs-n-fallback-for-proc_create_.patch similarity index 100% rename from queue-5.10/net-proc-provide-proc_fs-n-fallback-for-proc_create_.patch rename to next-5.10/net-proc-provide-proc_fs-n-fallback-for-proc_create_.patch diff --git a/next-5.10/net-provide-__sys_shutdown_sock-that-takes-a-socket.patch b/next-5.10/net-provide-__sys_shutdown_sock-that-takes-a-socket.patch new file mode 100644 index 00000000000..2d2a56c0e06 --- /dev/null +++ b/next-5.10/net-provide-__sys_shutdown_sock-that-takes-a-socket.patch @@ -0,0 +1,62 @@ +From b7a66ad26bc4d7ba3285345a3bfef88214164df3 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Sat, 5 Sep 2020 11:13:35 -0600 +Subject: net: provide __sys_shutdown_sock() that takes a socket + +From: Jens Axboe + +[ Upstream commit b713c195d59332277a31a59c91f755e53b5b302b ] + +No functional changes in this patch, needed to provide io_uring support +for shutdown(2). + +Cc: netdev@vger.kernel.org +Cc: David S. Miller +Acked-by: Jakub Kicinski +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/socket.h | 1 + + net/socket.c | 15 ++++++++++++--- + 2 files changed, 13 insertions(+), 3 deletions(-) + +--- a/include/linux/socket.h ++++ b/include/linux/socket.h +@@ -436,5 +436,6 @@ extern int __sys_getpeername(int fd, str + int __user *usockaddr_len); + extern int __sys_socketpair(int family, int type, int protocol, + int __user *usockvec); ++extern int __sys_shutdown_sock(struct socket *sock, int how); + extern int __sys_shutdown(int fd, int how); + #endif /* _LINUX_SOCKET_H */ +--- a/net/socket.c ++++ b/net/socket.c +@@ -2181,6 +2181,17 @@ SYSCALL_DEFINE5(getsockopt, int, fd, int + * Shutdown a socket. + */ + ++int __sys_shutdown_sock(struct socket *sock, int how) ++{ ++ int err; ++ ++ err = security_socket_shutdown(sock, how); ++ if (!err) ++ err = sock->ops->shutdown(sock, how); ++ ++ return err; ++} ++ + int __sys_shutdown(int fd, int how) + { + int err, fput_needed; +@@ -2188,9 +2199,7 @@ int __sys_shutdown(int fd, int how) + + sock = sockfd_lookup_light(fd, &err, &fput_needed); + if (sock != NULL) { +- err = security_socket_shutdown(sock, how); +- if (!err) +- err = sock->ops->shutdown(sock, how); ++ err = __sys_shutdown_sock(sock, how); + fput_light(sock->file, fput_needed); + } + return err; diff --git a/next-5.10/net-remove-cmsg-restriction-from-io_uring-based-send-recvmsg-calls.patch b/next-5.10/net-remove-cmsg-restriction-from-io_uring-based-send-recvmsg-calls.patch new file mode 100644 index 00000000000..a2f9e7957ea --- /dev/null +++ b/next-5.10/net-remove-cmsg-restriction-from-io_uring-based-send-recvmsg-calls.patch @@ -0,0 +1,90 @@ +From 89a29005f595d929d8d784f3422fb6a350f4b05f Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Wed, 17 Feb 2021 10:14:21 -0700 +Subject: net: remove cmsg restriction from io_uring based send/recvmsg calls + +From: Jens Axboe + +[ Upstream commit e54937963fa249595824439dc839c948188dea83 ] + +No need to restrict these anymore, as the worker threads are direct +clones of the original task. Hence we know for a fact that we can +support anything that the regular task can. + +Since the only user of proto_ops->flags was to flag PROTO_CMSG_DATA_ONLY, +kill the member and the flag definition too. + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/net.h | 3 --- + net/ipv4/af_inet.c | 1 - + net/ipv6/af_inet6.c | 1 - + net/socket.c | 10 ---------- + 4 files changed, 15 deletions(-) + +--- a/include/linux/net.h ++++ b/include/linux/net.h +@@ -42,8 +42,6 @@ struct net; + #define SOCK_PASSCRED 3 + #define SOCK_PASSSEC 4 + +-#define PROTO_CMSG_DATA_ONLY 0x0001 +- + #ifndef ARCH_HAS_SOCKET_TYPES + /** + * enum sock_type - Socket types +@@ -138,7 +136,6 @@ typedef int (*sk_read_actor_t)(read_desc + + struct proto_ops { + int family; +- unsigned int flags; + struct module *owner; + int (*release) (struct socket *sock); + int (*bind) (struct socket *sock, +--- a/net/ipv4/af_inet.c ++++ b/net/ipv4/af_inet.c +@@ -1017,7 +1017,6 @@ static int inet_compat_ioctl(struct sock + + const struct proto_ops inet_stream_ops = { + .family = PF_INET, +- .flags = PROTO_CMSG_DATA_ONLY, + .owner = THIS_MODULE, + .release = inet_release, + .bind = inet_bind, +--- a/net/ipv6/af_inet6.c ++++ b/net/ipv6/af_inet6.c +@@ -661,7 +661,6 @@ int inet6_recvmsg(struct socket *sock, s + + const struct proto_ops inet6_stream_ops = { + .family = PF_INET6, +- .flags = PROTO_CMSG_DATA_ONLY, + .owner = THIS_MODULE, + .release = inet6_release, + .bind = inet6_bind, +--- a/net/socket.c ++++ b/net/socket.c +@@ -2419,10 +2419,6 @@ static int ___sys_sendmsg(struct socket + long __sys_sendmsg_sock(struct socket *sock, struct msghdr *msg, + unsigned int flags) + { +- /* disallow ancillary data requests from this path */ +- if (msg->msg_control || msg->msg_controllen) +- return -EINVAL; +- + return ____sys_sendmsg(sock, msg, flags, NULL, 0); + } + +@@ -2631,12 +2627,6 @@ long __sys_recvmsg_sock(struct socket *s + struct user_msghdr __user *umsg, + struct sockaddr __user *uaddr, unsigned int flags) + { +- if (msg->msg_control || msg->msg_controllen) { +- /* disallow ancillary data reqs unless cmsg is plain data */ +- if (!(sock->ops->flags & PROTO_CMSG_DATA_ONLY)) +- return -EINVAL; +- } +- + return ____sys_recvmsg(sock, msg, umsg, uaddr, flags, 0); + } + diff --git a/queue-5.10/net-stmmac-selftests-fix-potential-memleak-in-stmmac.patch b/next-5.10/net-stmmac-selftests-fix-potential-memleak-in-stmmac.patch similarity index 100% rename from queue-5.10/net-stmmac-selftests-fix-potential-memleak-in-stmmac.patch rename to next-5.10/net-stmmac-selftests-fix-potential-memleak-in-stmmac.patch diff --git a/queue-5.10/net-stream-purge-sk_error_queue-in-sk_stream_kill_qu.patch b/next-5.10/net-stream-purge-sk_error_queue-in-sk_stream_kill_qu.patch similarity index 100% rename from queue-5.10/net-stream-purge-sk_error_queue-in-sk_stream_kill_qu.patch rename to next-5.10/net-stream-purge-sk_error_queue-in-sk_stream_kill_qu.patch diff --git a/queue-5.10/net-switch-to-storing-kcov-handle-directly-in-sk_buf.patch b/next-5.10/net-switch-to-storing-kcov-handle-directly-in-sk_buf.patch similarity index 100% rename from queue-5.10/net-switch-to-storing-kcov-handle-directly-in-sk_buf.patch rename to next-5.10/net-switch-to-storing-kcov-handle-directly-in-sk_buf.patch diff --git a/queue-5.10/net-tunnel-wait-until-all-sk_user_data-reader-finish.patch b/next-5.10/net-tunnel-wait-until-all-sk_user_data-reader-finish.patch similarity index 100% rename from queue-5.10/net-tunnel-wait-until-all-sk_user_data-reader-finish.patch rename to next-5.10/net-tunnel-wait-until-all-sk_user_data-reader-finish.patch diff --git a/queue-5.10/net-vmw_vsock-vmci-check-memcpy_from_msg.patch b/next-5.10/net-vmw_vsock-vmci-check-memcpy_from_msg.patch similarity index 100% rename from queue-5.10/net-vmw_vsock-vmci-check-memcpy_from_msg.patch rename to next-5.10/net-vmw_vsock-vmci-check-memcpy_from_msg.patch diff --git a/queue-5.10/net_sched-reject-tcf_em_simple-case-for-complex-emat.patch b/next-5.10/net_sched-reject-tcf_em_simple-case-for-complex-emat.patch similarity index 100% rename from queue-5.10/net_sched-reject-tcf_em_simple-case-for-complex-emat.patch rename to next-5.10/net_sched-reject-tcf_em_simple-case-for-complex-emat.patch diff --git a/queue-5.10/netfilter-conntrack-set-icmpv6-redirects-as-related.patch b/next-5.10/netfilter-conntrack-set-icmpv6-redirects-as-related.patch similarity index 100% rename from queue-5.10/netfilter-conntrack-set-icmpv6-redirects-as-related.patch rename to next-5.10/netfilter-conntrack-set-icmpv6-redirects-as-related.patch diff --git a/queue-5.10/netfilter-flowtable-really-fix-nat-ipv6-offload.patch b/next-5.10/netfilter-flowtable-really-fix-nat-ipv6-offload.patch similarity index 100% rename from queue-5.10/netfilter-flowtable-really-fix-nat-ipv6-offload.patch rename to next-5.10/netfilter-flowtable-really-fix-nat-ipv6-offload.patch diff --git a/queue-5.10/nfc-pn533-clear-nfc_target-before-being-used.patch b/next-5.10/nfc-pn533-clear-nfc_target-before-being-used.patch similarity index 100% rename from queue-5.10/nfc-pn533-clear-nfc_target-before-being-used.patch rename to next-5.10/nfc-pn533-clear-nfc_target-before-being-used.patch diff --git a/queue-5.10/nfs-fix-an-oops-in-nfs_d_automount.patch b/next-5.10/nfs-fix-an-oops-in-nfs_d_automount.patch similarity index 100% rename from queue-5.10/nfs-fix-an-oops-in-nfs_d_automount.patch rename to next-5.10/nfs-fix-an-oops-in-nfs_d_automount.patch diff --git a/queue-5.10/nfsd-don-t-call-nfsd_file_put-from-client-states-seq.patch b/next-5.10/nfsd-don-t-call-nfsd_file_put-from-client-states-seq.patch similarity index 100% rename from queue-5.10/nfsd-don-t-call-nfsd_file_put-from-client-states-seq.patch rename to next-5.10/nfsd-don-t-call-nfsd_file_put-from-client-states-seq.patch diff --git a/queue-5.10/nfsd-remove-spurious-cb_setup_err-tracepoint.patch b/next-5.10/nfsd-remove-spurious-cb_setup_err-tracepoint.patch similarity index 100% rename from queue-5.10/nfsd-remove-spurious-cb_setup_err-tracepoint.patch rename to next-5.10/nfsd-remove-spurious-cb_setup_err-tracepoint.patch diff --git a/queue-5.10/nfsd-under-nfsv4.1-fix-double-svc_xprt_put-on-rpc_cr.patch b/next-5.10/nfsd-under-nfsv4.1-fix-double-svc_xprt_put-on-rpc_cr.patch similarity index 100% rename from queue-5.10/nfsd-under-nfsv4.1-fix-double-svc_xprt_put-on-rpc_cr.patch rename to next-5.10/nfsd-under-nfsv4.1-fix-double-svc_xprt_put-on-rpc_cr.patch diff --git a/queue-5.10/nfsv4-fix-a-deadlock-between-nfs4_open_recover_helpe.patch b/next-5.10/nfsv4-fix-a-deadlock-between-nfs4_open_recover_helpe.patch similarity index 100% rename from queue-5.10/nfsv4-fix-a-deadlock-between-nfs4_open_recover_helpe.patch rename to next-5.10/nfsv4-fix-a-deadlock-between-nfs4_open_recover_helpe.patch diff --git a/queue-5.10/nfsv4.2-clear-fattr4_word2_security_label-when-done-.patch b/next-5.10/nfsv4.2-clear-fattr4_word2_security_label-when-done-.patch similarity index 100% rename from queue-5.10/nfsv4.2-clear-fattr4_word2_security_label-when-done-.patch rename to next-5.10/nfsv4.2-clear-fattr4_word2_security_label-when-done-.patch diff --git a/queue-5.10/nfsv4.2-fix-a-memory-stomp-in-decode_attr_security_l.patch b/next-5.10/nfsv4.2-fix-a-memory-stomp-in-decode_attr_security_l.patch similarity index 100% rename from queue-5.10/nfsv4.2-fix-a-memory-stomp-in-decode_attr_security_l.patch rename to next-5.10/nfsv4.2-fix-a-memory-stomp-in-decode_attr_security_l.patch diff --git a/queue-5.10/nfsv4.2-fix-initialisation-of-struct-nfs4_label.patch b/next-5.10/nfsv4.2-fix-initialisation-of-struct-nfs4_label.patch similarity index 100% rename from queue-5.10/nfsv4.2-fix-initialisation-of-struct-nfs4_label.patch rename to next-5.10/nfsv4.2-fix-initialisation-of-struct-nfs4_label.patch diff --git a/queue-5.10/nfsv4.x-fail-client-initialisation-if-state-manager-.patch b/next-5.10/nfsv4.x-fail-client-initialisation-if-state-manager-.patch similarity index 100% rename from queue-5.10/nfsv4.x-fail-client-initialisation-if-state-manager-.patch rename to next-5.10/nfsv4.x-fail-client-initialisation-if-state-manager-.patch diff --git a/queue-5.10/nilfs2-fix-shift-out-of-bounds-due-to-too-large-expo.patch b/next-5.10/nilfs2-fix-shift-out-of-bounds-due-to-too-large-expo.patch similarity index 100% rename from queue-5.10/nilfs2-fix-shift-out-of-bounds-due-to-too-large-expo.patch rename to next-5.10/nilfs2-fix-shift-out-of-bounds-due-to-too-large-expo.patch diff --git a/queue-5.10/nilfs2-fix-shift-out-of-bounds-overflow-in-nilfs_sb2.patch b/next-5.10/nilfs2-fix-shift-out-of-bounds-overflow-in-nilfs_sb2.patch similarity index 100% rename from queue-5.10/nilfs2-fix-shift-out-of-bounds-overflow-in-nilfs_sb2.patch rename to next-5.10/nilfs2-fix-shift-out-of-bounds-overflow-in-nilfs_sb2.patch diff --git a/next-5.10/nios32-add-support-for-tif_notify_signal.patch b/next-5.10/nios32-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..5b025ddce89 --- /dev/null +++ b/next-5.10/nios32-add-support-for-tif_notify_signal.patch @@ -0,0 +1,50 @@ +From ad18cba50dbe1fbc8e57c4e96f4a952efe212eb3 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 15:21:21 -0600 +Subject: nios32: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 42020064274c235d720d9c4b7d9a678b133e59cf ] + +Wire up TIF_NOTIFY_SIGNAL handling for nios32. + +Cc: Ley Foon Tan +Acked-by: Ley Foon Tan +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/nios2/include/asm/thread_info.h | 2 ++ + arch/nios2/kernel/signal.c | 3 ++- + 2 files changed, 4 insertions(+), 1 deletion(-) + +--- a/arch/nios2/include/asm/thread_info.h ++++ b/arch/nios2/include/asm/thread_info.h +@@ -86,6 +86,7 @@ static inline struct thread_info *curren + #define TIF_MEMDIE 4 /* is terminating due to OOM killer */ + #define TIF_SECCOMP 5 /* secure computing */ + #define TIF_SYSCALL_AUDIT 6 /* syscall auditing active */ ++#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ + #define TIF_RESTORE_SIGMASK 9 /* restore signal mask in do_signal() */ + + #define TIF_POLLING_NRFLAG 16 /* true if poll_idle() is polling +@@ -97,6 +98,7 @@ static inline struct thread_info *curren + #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) + #define _TIF_SECCOMP (1 << TIF_SECCOMP) + #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + #define _TIF_RESTORE_SIGMASK (1 << TIF_RESTORE_SIGMASK) + #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) + +--- a/arch/nios2/kernel/signal.c ++++ b/arch/nios2/kernel/signal.c +@@ -309,7 +309,8 @@ asmlinkage int do_notify_resume(struct p + if (!user_mode(regs)) + return 0; + +- if (test_thread_flag(TIF_SIGPENDING)) { ++ if (test_thread_flag(TIF_SIGPENDING) || ++ test_thread_flag(TIF_NOTIFY_SIGNAL)) { + int restart = do_signal(regs); + + if (unlikely(restart)) { diff --git a/queue-5.10/ntb_netdev-use-dev_kfree_skb_any-in-interrupt-contex.patch b/next-5.10/ntb_netdev-use-dev_kfree_skb_any-in-interrupt-contex.patch similarity index 100% rename from queue-5.10/ntb_netdev-use-dev_kfree_skb_any-in-interrupt-contex.patch rename to next-5.10/ntb_netdev-use-dev_kfree_skb_any-in-interrupt-contex.patch diff --git a/queue-5.10/nvme-fix-the-nvme_cmd_effects_cse_mask-definition.patch b/next-5.10/nvme-fix-the-nvme_cmd_effects_cse_mask-definition.patch similarity index 100% rename from queue-5.10/nvme-fix-the-nvme_cmd_effects_cse_mask-definition.patch rename to next-5.10/nvme-fix-the-nvme_cmd_effects_cse_mask-definition.patch diff --git a/queue-5.10/nvme-pci-fix-doorbell-buffer-value-endianness.patch b/next-5.10/nvme-pci-fix-doorbell-buffer-value-endianness.patch similarity index 100% rename from queue-5.10/nvme-pci-fix-doorbell-buffer-value-endianness.patch rename to next-5.10/nvme-pci-fix-doorbell-buffer-value-endianness.patch diff --git a/queue-5.10/nvme-pci-fix-mempool-alloc-size.patch b/next-5.10/nvme-pci-fix-mempool-alloc-size.patch similarity index 100% rename from queue-5.10/nvme-pci-fix-mempool-alloc-size.patch rename to next-5.10/nvme-pci-fix-mempool-alloc-size.patch diff --git a/queue-5.10/nvme-pci-fix-page-size-checks.patch b/next-5.10/nvme-pci-fix-page-size-checks.patch similarity index 100% rename from queue-5.10/nvme-pci-fix-page-size-checks.patch rename to next-5.10/nvme-pci-fix-page-size-checks.patch diff --git a/queue-5.10/nvmet-don-t-defer-passthrough-commands-with-trivial-.patch b/next-5.10/nvmet-don-t-defer-passthrough-commands-with-trivial-.patch similarity index 100% rename from queue-5.10/nvmet-don-t-defer-passthrough-commands-with-trivial-.patch rename to next-5.10/nvmet-don-t-defer-passthrough-commands-with-trivial-.patch diff --git a/queue-5.10/objtool-fix-segfault.patch b/next-5.10/objtool-fix-segfault.patch similarity index 100% rename from queue-5.10/objtool-fix-segfault.patch rename to next-5.10/objtool-fix-segfault.patch diff --git a/queue-5.10/objtool-kcsan-add-volatile-read-write-instrumentatio.patch b/next-5.10/objtool-kcsan-add-volatile-read-write-instrumentatio.patch similarity index 100% rename from queue-5.10/objtool-kcsan-add-volatile-read-write-instrumentatio.patch rename to next-5.10/objtool-kcsan-add-volatile-read-write-instrumentatio.patch diff --git a/queue-5.10/ocfs2-fix-memory-leak-in-ocfs2_mount_volume.patch b/next-5.10/ocfs2-fix-memory-leak-in-ocfs2_mount_volume.patch similarity index 100% rename from queue-5.10/ocfs2-fix-memory-leak-in-ocfs2_mount_volume.patch rename to next-5.10/ocfs2-fix-memory-leak-in-ocfs2_mount_volume.patch diff --git a/queue-5.10/ocfs2-fix-memory-leak-in-ocfs2_stack_glue_init.patch b/next-5.10/ocfs2-fix-memory-leak-in-ocfs2_stack_glue_init.patch similarity index 100% rename from queue-5.10/ocfs2-fix-memory-leak-in-ocfs2_stack_glue_init.patch rename to next-5.10/ocfs2-fix-memory-leak-in-ocfs2_stack_glue_init.patch diff --git a/queue-5.10/ocfs2-ocfs2_mount_volume-does-cleanup-job-before-ret.patch b/next-5.10/ocfs2-ocfs2_mount_volume-does-cleanup-job-before-ret.patch similarity index 100% rename from queue-5.10/ocfs2-ocfs2_mount_volume-does-cleanup-job-before-ret.patch rename to next-5.10/ocfs2-ocfs2_mount_volume-does-cleanup-job-before-ret.patch diff --git a/queue-5.10/ocfs2-rewrite-error-handling-of-ocfs2_fill_super.patch b/next-5.10/ocfs2-rewrite-error-handling-of-ocfs2_fill_super.patch similarity index 100% rename from queue-5.10/ocfs2-rewrite-error-handling-of-ocfs2_fill_super.patch rename to next-5.10/ocfs2-rewrite-error-handling-of-ocfs2_fill_super.patch diff --git a/queue-5.10/ocxl-fix-pci-device-refcount-leak-when-calling-get_f.patch b/next-5.10/ocxl-fix-pci-device-refcount-leak-when-calling-get_f.patch similarity index 100% rename from queue-5.10/ocxl-fix-pci-device-refcount-leak-when-calling-get_f.patch rename to next-5.10/ocxl-fix-pci-device-refcount-leak-when-calling-get_f.patch diff --git a/queue-5.10/of-overlay-fix-null-pointer-dereferencing-in-find_du.patch b/next-5.10/of-overlay-fix-null-pointer-dereferencing-in-find_du.patch similarity index 100% rename from queue-5.10/of-overlay-fix-null-pointer-dereferencing-in-find_du.patch rename to next-5.10/of-overlay-fix-null-pointer-dereferencing-in-find_du.patch diff --git a/next-5.10/openrisc-add-support-for-tif_notify_signal.patch b/next-5.10/openrisc-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..6c7e4cb7ee7 --- /dev/null +++ b/next-5.10/openrisc-add-support-for-tif_notify_signal.patch @@ -0,0 +1,49 @@ +From 64cef01754a4419e13f0f5f14ec4a9e181bffbe8 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 15:24:46 -0600 +Subject: openrisc: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit e181c0aa2e532af2b17128fbde699f8578cc0562 ] + +Wire up TIF_NOTIFY_SIGNAL handling for openrisc. + +Cc: openrisc@lists.librecores.org +Acked-by: Stafford Horne +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/openrisc/include/asm/thread_info.h | 2 ++ + arch/openrisc/kernel/signal.c | 2 +- + 2 files changed, 3 insertions(+), 1 deletion(-) + +--- a/arch/openrisc/include/asm/thread_info.h ++++ b/arch/openrisc/include/asm/thread_info.h +@@ -98,6 +98,7 @@ register struct thread_info *current_thr + #define TIF_SINGLESTEP 4 /* restore singlestep on return to user + * mode + */ ++#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ + #define TIF_SYSCALL_TRACEPOINT 8 /* for ftrace syscall instrumentation */ + #define TIF_RESTORE_SIGMASK 9 + #define TIF_POLLING_NRFLAG 16 /* true if poll_idle() is polling * TIF_NEED_RESCHED +@@ -109,6 +110,7 @@ register struct thread_info *current_thr + #define _TIF_SIGPENDING (1< +Date: Fri, 9 Oct 2020 15:27:02 -0600 +Subject: parisc: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 18cb3281285d2190c0605d2e53543802319bd1a1 ] + +Wire up TIF_NOTIFY_SIGNAL handling for parisc. + +Cc: linux-parisc@vger.kernel.org +Acked-by: Helge Deller +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/parisc/include/asm/thread_info.h | 4 +++- + arch/parisc/kernel/signal.c | 3 ++- + 2 files changed, 5 insertions(+), 2 deletions(-) + +--- a/arch/parisc/include/asm/thread_info.h ++++ b/arch/parisc/include/asm/thread_info.h +@@ -52,6 +52,7 @@ struct thread_info { + #define TIF_POLLING_NRFLAG 3 /* true if poll_idle() is polling TIF_NEED_RESCHED */ + #define TIF_32BIT 4 /* 32 bit binary */ + #define TIF_MEMDIE 5 /* is terminating due to OOM killer */ ++#define TIF_NOTIFY_SIGNAL 6 /* signal notifications exist */ + #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ + #define TIF_NOTIFY_RESUME 8 /* callback before returning to user */ + #define TIF_SINGLESTEP 9 /* single stepping? */ +@@ -61,6 +62,7 @@ struct thread_info { + + #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) + #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) + #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) + #define _TIF_32BIT (1 << TIF_32BIT) +@@ -72,7 +74,7 @@ struct thread_info { + #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) + + #define _TIF_USER_WORK_MASK (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | \ +- _TIF_NEED_RESCHED) ++ _TIF_NEED_RESCHED | _TIF_NOTIFY_SIGNAL) + #define _TIF_SYSCALL_TRACE_MASK (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP | \ + _TIF_BLOCKSTEP | _TIF_SYSCALL_AUDIT | \ + _TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT) +--- a/arch/parisc/kernel/signal.c ++++ b/arch/parisc/kernel/signal.c +@@ -609,7 +609,8 @@ do_signal(struct pt_regs *regs, long in_ + + void do_notify_resume(struct pt_regs *regs, long in_syscall) + { +- if (test_thread_flag(TIF_SIGPENDING)) ++ if (test_thread_flag(TIF_SIGPENDING) || ++ test_thread_flag(TIF_NOTIFY_SIGNAL)) + do_signal(regs, in_syscall); + + if (test_thread_flag(TIF_NOTIFY_RESUME)) diff --git a/queue-5.10/pata_ipx4xx_cf-fix-unsigned-comparison-with-less-tha.patch b/next-5.10/pata_ipx4xx_cf-fix-unsigned-comparison-with-less-tha.patch similarity index 100% rename from queue-5.10/pata_ipx4xx_cf-fix-unsigned-comparison-with-less-tha.patch rename to next-5.10/pata_ipx4xx_cf-fix-unsigned-comparison-with-less-tha.patch diff --git a/queue-5.10/pci-check-for-alloc-failure-in-pci_request_irq.patch b/next-5.10/pci-check-for-alloc-failure-in-pci_request_irq.patch similarity index 100% rename from queue-5.10/pci-check-for-alloc-failure-in-pci_request_irq.patch rename to next-5.10/pci-check-for-alloc-failure-in-pci_request_irq.patch diff --git a/queue-5.10/pci-dwc-fix-n_fts-array-overrun.patch b/next-5.10/pci-dwc-fix-n_fts-array-overrun.patch similarity index 100% rename from queue-5.10/pci-dwc-fix-n_fts-array-overrun.patch rename to next-5.10/pci-dwc-fix-n_fts-array-overrun.patch diff --git a/queue-5.10/pci-pci-epf-test-register-notifier-if-only-core_init.patch b/next-5.10/pci-pci-epf-test-register-notifier-if-only-core_init.patch similarity index 100% rename from queue-5.10/pci-pci-epf-test-register-notifier-if-only-core_init.patch rename to next-5.10/pci-pci-epf-test-register-notifier-if-only-core_init.patch diff --git a/queue-5.10/perf-arm_dsu-fix-hotplug-callback-leak-in-dsu_pmu_in.patch b/next-5.10/perf-arm_dsu-fix-hotplug-callback-leak-in-dsu_pmu_in.patch similarity index 100% rename from queue-5.10/perf-arm_dsu-fix-hotplug-callback-leak-in-dsu_pmu_in.patch rename to next-5.10/perf-arm_dsu-fix-hotplug-callback-leak-in-dsu_pmu_in.patch diff --git a/queue-5.10/perf-debug-set-debug_peo_args-and-redirect_to_stderr.patch b/next-5.10/perf-debug-set-debug_peo_args-and-redirect_to_stderr.patch similarity index 100% rename from queue-5.10/perf-debug-set-debug_peo_args-and-redirect_to_stderr.patch rename to next-5.10/perf-debug-set-debug_peo_args-and-redirect_to_stderr.patch diff --git a/queue-5.10/perf-fix-possible-memleak-in-pmu_dev_alloc.patch b/next-5.10/perf-fix-possible-memleak-in-pmu_dev_alloc.patch similarity index 100% rename from queue-5.10/perf-fix-possible-memleak-in-pmu_dev_alloc.patch rename to next-5.10/perf-fix-possible-memleak-in-pmu_dev_alloc.patch diff --git a/queue-5.10/perf-smmuv3-fix-hotplug-callback-leak-in-arm_smmu_pm.patch b/next-5.10/perf-smmuv3-fix-hotplug-callback-leak-in-arm_smmu_pm.patch similarity index 100% rename from queue-5.10/perf-smmuv3-fix-hotplug-callback-leak-in-arm_smmu_pm.patch rename to next-5.10/perf-smmuv3-fix-hotplug-callback-leak-in-arm_smmu_pm.patch diff --git a/queue-5.10/perf-symbol-correction-while-adjusting-symbol.patch b/next-5.10/perf-symbol-correction-while-adjusting-symbol.patch similarity index 100% rename from queue-5.10/perf-symbol-correction-while-adjusting-symbol.patch rename to next-5.10/perf-symbol-correction-while-adjusting-symbol.patch diff --git a/queue-5.10/perf-trace-handle-failure-when-trace-point-folder-is.patch b/next-5.10/perf-trace-handle-failure-when-trace-point-folder-is.patch similarity index 100% rename from queue-5.10/perf-trace-handle-failure-when-trace-point-folder-is.patch rename to next-5.10/perf-trace-handle-failure-when-trace-point-folder-is.patch diff --git a/queue-5.10/perf-trace-return-error-if-a-system-call-doesn-t-exi.patch b/next-5.10/perf-trace-return-error-if-a-system-call-doesn-t-exi.patch similarity index 100% rename from queue-5.10/perf-trace-return-error-if-a-system-call-doesn-t-exi.patch rename to next-5.10/perf-trace-return-error-if-a-system-call-doesn-t-exi.patch diff --git a/queue-5.10/perf-trace-use-macro-raw_syscall_args_num-to-replace.patch b/next-5.10/perf-trace-use-macro-raw_syscall_args_num-to-replace.patch similarity index 100% rename from queue-5.10/perf-trace-use-macro-raw_syscall_args_num-to-replace.patch rename to next-5.10/perf-trace-use-macro-raw_syscall_args_num-to-replace.patch diff --git a/queue-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-__.patch b/next-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-__.patch similarity index 100% rename from queue-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-__.patch rename to next-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-__.patch diff --git a/queue-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-hs.patch b/next-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-hs.patch similarity index 100% rename from queue-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-hs.patch rename to next-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-hs.patch diff --git a/queue-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-sn.patch b/next-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-sn.patch similarity index 100% rename from queue-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-sn.patch rename to next-5.10/perf-x86-intel-uncore-fix-reference-count-leak-in-sn.patch diff --git a/queue-5.10/phy-usb-s2-wol-wakeup_count-not-incremented-for-usb-.patch b/next-5.10/phy-usb-s2-wol-wakeup_count-not-incremented-for-usb-.patch similarity index 100% rename from queue-5.10/phy-usb-s2-wol-wakeup_count-not-incremented-for-usb-.patch rename to next-5.10/phy-usb-s2-wol-wakeup_count-not-incremented-for-usb-.patch diff --git a/queue-5.10/pinctrl-pinconf-generic-add-missing-of_node_put.patch b/next-5.10/pinctrl-pinconf-generic-add-missing-of_node_put.patch similarity index 100% rename from queue-5.10/pinctrl-pinconf-generic-add-missing-of_node_put.patch rename to next-5.10/pinctrl-pinconf-generic-add-missing-of_node_put.patch diff --git a/queue-5.10/platform-chrome-cros_usbpd_notify-fix-error-handling.patch b/next-5.10/platform-chrome-cros_usbpd_notify-fix-error-handling.patch similarity index 100% rename from queue-5.10/platform-chrome-cros_usbpd_notify-fix-error-handling.patch rename to next-5.10/platform-chrome-cros_usbpd_notify-fix-error-handling.patch diff --git a/queue-5.10/platform-x86-huawei-wmi-fix-return-value-calculation.patch b/next-5.10/platform-x86-huawei-wmi-fix-return-value-calculation.patch similarity index 100% rename from queue-5.10/platform-x86-huawei-wmi-fix-return-value-calculation.patch rename to next-5.10/platform-x86-huawei-wmi-fix-return-value-calculation.patch diff --git a/queue-5.10/platform-x86-intel_scu_ipc-fix-possible-name-leak-in.patch b/next-5.10/platform-x86-intel_scu_ipc-fix-possible-name-leak-in.patch similarity index 100% rename from queue-5.10/platform-x86-intel_scu_ipc-fix-possible-name-leak-in.patch rename to next-5.10/platform-x86-intel_scu_ipc-fix-possible-name-leak-in.patch diff --git a/queue-5.10/platform-x86-mxm-wmi-fix-memleak-in-mxm_wmi_call_mx-.patch b/next-5.10/platform-x86-mxm-wmi-fix-memleak-in-mxm_wmi_call_mx-.patch similarity index 100% rename from queue-5.10/platform-x86-mxm-wmi-fix-memleak-in-mxm_wmi_call_mx-.patch rename to next-5.10/platform-x86-mxm-wmi-fix-memleak-in-mxm_wmi_call_mx-.patch diff --git a/queue-5.10/pm-hibernate-fix-mistake-in-kerneldoc-comment.patch b/next-5.10/pm-hibernate-fix-mistake-in-kerneldoc-comment.patch similarity index 100% rename from queue-5.10/pm-hibernate-fix-mistake-in-kerneldoc-comment.patch rename to next-5.10/pm-hibernate-fix-mistake-in-kerneldoc-comment.patch diff --git a/queue-5.10/pm-runtime-do-not-call-__rpm_callback-from-rpm_idle.patch b/next-5.10/pm-runtime-do-not-call-__rpm_callback-from-rpm_idle.patch similarity index 100% rename from queue-5.10/pm-runtime-do-not-call-__rpm_callback-from-rpm_idle.patch rename to next-5.10/pm-runtime-do-not-call-__rpm_callback-from-rpm_idle.patch diff --git a/queue-5.10/pm-runtime-improve-path-in-rpm_idle-when-no-callback.patch b/next-5.10/pm-runtime-improve-path-in-rpm_idle-when-no-callback.patch similarity index 100% rename from queue-5.10/pm-runtime-improve-path-in-rpm_idle-when-no-callback.patch rename to next-5.10/pm-runtime-improve-path-in-rpm_idle-when-no-callback.patch diff --git a/queue-5.10/pnode-terminate-at-peers-of-source.patch b/next-5.10/pnode-terminate-at-peers-of-source.patch similarity index 100% rename from queue-5.10/pnode-terminate-at-peers-of-source.patch rename to next-5.10/pnode-terminate-at-peers-of-source.patch diff --git a/queue-5.10/pnp-fix-name-memory-leak-in-pnp_alloc_dev.patch b/next-5.10/pnp-fix-name-memory-leak-in-pnp_alloc_dev.patch similarity index 100% rename from queue-5.10/pnp-fix-name-memory-leak-in-pnp_alloc_dev.patch rename to next-5.10/pnp-fix-name-memory-leak-in-pnp_alloc_dev.patch diff --git a/queue-5.10/power-supply-fix-null-pointer-dereferencing-in-power.patch b/next-5.10/power-supply-fix-null-pointer-dereferencing-in-power.patch similarity index 100% rename from queue-5.10/power-supply-fix-null-pointer-dereferencing-in-power.patch rename to next-5.10/power-supply-fix-null-pointer-dereferencing-in-power.patch diff --git a/queue-5.10/power-supply-fix-residue-sysfs-file-in-error-handle-.patch b/next-5.10/power-supply-fix-residue-sysfs-file-in-error-handle-.patch similarity index 100% rename from queue-5.10/power-supply-fix-residue-sysfs-file-in-error-handle-.patch rename to next-5.10/power-supply-fix-residue-sysfs-file-in-error-handle-.patch diff --git a/queue-5.10/powerpc-52xx-fix-a-resource-leak-in-an-error-handlin.patch b/next-5.10/powerpc-52xx-fix-a-resource-leak-in-an-error-handlin.patch similarity index 100% rename from queue-5.10/powerpc-52xx-fix-a-resource-leak-in-an-error-handlin.patch rename to next-5.10/powerpc-52xx-fix-a-resource-leak-in-an-error-handlin.patch diff --git a/queue-5.10/powerpc-83xx-mpc832x_rdb-call-platform_device_put-in.patch b/next-5.10/powerpc-83xx-mpc832x_rdb-call-platform_device_put-in.patch similarity index 100% rename from queue-5.10/powerpc-83xx-mpc832x_rdb-call-platform_device_put-in.patch rename to next-5.10/powerpc-83xx-mpc832x_rdb-call-platform_device_put-in.patch diff --git a/next-5.10/powerpc-add-support-for-tif_notify_signal.patch b/next-5.10/powerpc-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..2b37e4d2e58 --- /dev/null +++ b/next-5.10/powerpc-add-support-for-tif_notify_signal.patch @@ -0,0 +1,59 @@ +From 616d38f834d91a5af2e3afb08f12ecb8b32e4a96 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Thu, 22 Oct 2020 20:11:56 -0600 +Subject: powerpc: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 900f0713fdd730fab0f0bfa4a8ca4db2a8985bbe ] + +Wire up TIF_NOTIFY_SIGNAL handling for powerpc. + +Cc: linuxppc-dev@lists.ozlabs.org +Acked-by: Michael Ellerman +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/powerpc/include/asm/thread_info.h | 5 ++++- + arch/powerpc/kernel/signal.c | 2 +- + 2 files changed, 5 insertions(+), 2 deletions(-) + +--- a/arch/powerpc/include/asm/thread_info.h ++++ b/arch/powerpc/include/asm/thread_info.h +@@ -96,6 +96,7 @@ void arch_setup_new_exec(void); + #define TIF_SYSCALL_TRACE 0 /* syscall trace active */ + #define TIF_SIGPENDING 1 /* signal pending */ + #define TIF_NEED_RESCHED 2 /* rescheduling necessary */ ++#define TIF_NOTIFY_SIGNAL 3 /* signal notifications exist */ + #define TIF_SYSCALL_EMU 4 /* syscall emulation active */ + #define TIF_RESTORE_TM 5 /* need to restore TM FP/VEC/VSX */ + #define TIF_PATCH_PENDING 6 /* pending live patching update */ +@@ -121,6 +122,7 @@ void arch_setup_new_exec(void); + #define _TIF_SYSCALL_TRACE (1<thread.regs); + do_signal(current); + } diff --git a/queue-5.10/powerpc-eeh-drop-redundant-spinlock-initialization.patch b/next-5.10/powerpc-eeh-drop-redundant-spinlock-initialization.patch similarity index 100% rename from queue-5.10/powerpc-eeh-drop-redundant-spinlock-initialization.patch rename to next-5.10/powerpc-eeh-drop-redundant-spinlock-initialization.patch diff --git a/queue-5.10/powerpc-hv-gpci-fix-hv_gpci-event-list.patch b/next-5.10/powerpc-hv-gpci-fix-hv_gpci-event-list.patch similarity index 100% rename from queue-5.10/powerpc-hv-gpci-fix-hv_gpci-event-list.patch rename to next-5.10/powerpc-hv-gpci-fix-hv_gpci-event-list.patch diff --git a/queue-5.10/powerpc-perf-callchain-validate-kernel-stack-pointer.patch b/next-5.10/powerpc-perf-callchain-validate-kernel-stack-pointer.patch similarity index 100% rename from queue-5.10/powerpc-perf-callchain-validate-kernel-stack-pointer.patch rename to next-5.10/powerpc-perf-callchain-validate-kernel-stack-pointer.patch diff --git a/queue-5.10/powerpc-pseries-eeh-use-correct-api-for-error-log-si.patch b/next-5.10/powerpc-pseries-eeh-use-correct-api-for-error-log-si.patch similarity index 100% rename from queue-5.10/powerpc-pseries-eeh-use-correct-api-for-error-log-si.patch rename to next-5.10/powerpc-pseries-eeh-use-correct-api-for-error-log-si.patch diff --git a/queue-5.10/powerpc-rtas-avoid-device-tree-lookups-in-rtas_os_te.patch b/next-5.10/powerpc-rtas-avoid-device-tree-lookups-in-rtas_os_te.patch similarity index 100% rename from queue-5.10/powerpc-rtas-avoid-device-tree-lookups-in-rtas_os_te.patch rename to next-5.10/powerpc-rtas-avoid-device-tree-lookups-in-rtas_os_te.patch diff --git a/queue-5.10/powerpc-rtas-avoid-scheduling-in-rtas_os_term.patch b/next-5.10/powerpc-rtas-avoid-scheduling-in-rtas_os_term.patch similarity index 100% rename from queue-5.10/powerpc-rtas-avoid-scheduling-in-rtas_os_term.patch rename to next-5.10/powerpc-rtas-avoid-scheduling-in-rtas_os_term.patch diff --git a/queue-5.10/powerpc-xive-add-missing-iounmap-in-error-path-in-xi.patch b/next-5.10/powerpc-xive-add-missing-iounmap-in-error-path-in-xi.patch similarity index 100% rename from queue-5.10/powerpc-xive-add-missing-iounmap-in-error-path-in-xi.patch rename to next-5.10/powerpc-xive-add-missing-iounmap-in-error-path-in-xi.patch diff --git a/queue-5.10/powerpc-xmon-enable-breakpoints-on-8xx.patch b/next-5.10/powerpc-xmon-enable-breakpoints-on-8xx.patch similarity index 100% rename from queue-5.10/powerpc-xmon-enable-breakpoints-on-8xx.patch rename to next-5.10/powerpc-xmon-enable-breakpoints-on-8xx.patch diff --git a/queue-5.10/powerpc-xmon-fix-wswitch-unreachable-warning-in-bpt_.patch b/next-5.10/powerpc-xmon-fix-wswitch-unreachable-warning-in-bpt_.patch similarity index 100% rename from queue-5.10/powerpc-xmon-fix-wswitch-unreachable-warning-in-bpt_.patch rename to next-5.10/powerpc-xmon-fix-wswitch-unreachable-warning-in-bpt_.patch diff --git a/queue-5.10/ppp-associate-skb-with-a-device-at-tx.patch b/next-5.10/ppp-associate-skb-with-a-device-at-tx.patch similarity index 100% rename from queue-5.10/ppp-associate-skb-with-a-device-at-tx.patch rename to next-5.10/ppp-associate-skb-with-a-device-at-tx.patch diff --git a/queue-5.10/proc-fixup-uptime-selftest.patch b/next-5.10/proc-fixup-uptime-selftest.patch similarity index 100% rename from queue-5.10/proc-fixup-uptime-selftest.patch rename to next-5.10/proc-fixup-uptime-selftest.patch diff --git a/queue-5.10/pstore-avoid-kcore-oops-by-vmap-ing-with-vm_ioremap.patch b/next-5.10/pstore-avoid-kcore-oops-by-vmap-ing-with-vm_ioremap.patch similarity index 100% rename from queue-5.10/pstore-avoid-kcore-oops-by-vmap-ing-with-vm_ioremap.patch rename to next-5.10/pstore-avoid-kcore-oops-by-vmap-ing-with-vm_ioremap.patch diff --git a/queue-5.10/pstore-make-sure-config_pstore_pmsg-selects-config_r.patch b/next-5.10/pstore-make-sure-config_pstore_pmsg-selects-config_r.patch similarity index 100% rename from queue-5.10/pstore-make-sure-config_pstore_pmsg-selects-config_r.patch rename to next-5.10/pstore-make-sure-config_pstore_pmsg-selects-config_r.patch diff --git a/queue-5.10/pstore-ram-fix-error-return-code-in-ramoops_probe.patch b/next-5.10/pstore-ram-fix-error-return-code-in-ramoops_probe.patch similarity index 100% rename from queue-5.10/pstore-ram-fix-error-return-code-in-ramoops_probe.patch rename to next-5.10/pstore-ram-fix-error-return-code-in-ramoops_probe.patch diff --git a/queue-5.10/pstore-switch-pmsg_lock-to-an-rt_mutex-to-avoid-prio.patch b/next-5.10/pstore-switch-pmsg_lock-to-an-rt_mutex-to-avoid-prio.patch similarity index 100% rename from queue-5.10/pstore-switch-pmsg_lock-to-an-rt_mutex-to-avoid-prio.patch rename to next-5.10/pstore-switch-pmsg_lock-to-an-rt_mutex-to-avoid-prio.patch diff --git a/queue-5.10/pstore-zone-use-gfp_atomic-to-allocate-zone-buffer.patch b/next-5.10/pstore-zone-use-gfp_atomic-to-allocate-zone-buffer.patch similarity index 100% rename from queue-5.10/pstore-zone-use-gfp_atomic-to-allocate-zone-buffer.patch rename to next-5.10/pstore-zone-use-gfp_atomic-to-allocate-zone-buffer.patch diff --git a/queue-5.10/pwm-sifive-call-pwm_sifive_update_clock-while-mutex-.patch b/next-5.10/pwm-sifive-call-pwm_sifive_update_clock-while-mutex-.patch similarity index 100% rename from queue-5.10/pwm-sifive-call-pwm_sifive_update_clock-while-mutex-.patch rename to next-5.10/pwm-sifive-call-pwm_sifive_update_clock-while-mutex-.patch diff --git a/queue-5.10/pwm-tegra-fix-32-bit-build.patch b/next-5.10/pwm-tegra-fix-32-bit-build.patch similarity index 100% rename from queue-5.10/pwm-tegra-fix-32-bit-build.patch rename to next-5.10/pwm-tegra-fix-32-bit-build.patch diff --git a/queue-5.10/pwm-tegra-improve-required-rate-calculation.patch b/next-5.10/pwm-tegra-improve-required-rate-calculation.patch similarity index 100% rename from queue-5.10/pwm-tegra-improve-required-rate-calculation.patch rename to next-5.10/pwm-tegra-improve-required-rate-calculation.patch diff --git a/queue-5.10/qed-gcc13-use-u16-for-fid-to-be-big-enough.patch b/next-5.10/qed-gcc13-use-u16-for-fid-to-be-big-enough.patch similarity index 100% rename from queue-5.10/qed-gcc13-use-u16-for-fid-to-be-big-enough.patch rename to next-5.10/qed-gcc13-use-u16-for-fid-to-be-big-enough.patch diff --git a/queue-5.10/r6040-fix-kmemleak-in-probe-and-remove.patch b/next-5.10/r6040-fix-kmemleak-in-probe-and-remove.patch similarity index 100% rename from queue-5.10/r6040-fix-kmemleak-in-probe-and-remove.patch rename to next-5.10/r6040-fix-kmemleak-in-probe-and-remove.patch diff --git a/queue-5.10/rapidio-devices-fix-missing-put_device-in-mport_cdev.patch b/next-5.10/rapidio-devices-fix-missing-put_device-in-mport_cdev.patch similarity index 100% rename from queue-5.10/rapidio-devices-fix-missing-put_device-in-mport_cdev.patch rename to next-5.10/rapidio-devices-fix-missing-put_device-in-mport_cdev.patch diff --git a/queue-5.10/rapidio-fix-possible-name-leaks-when-rio_add_device-.patch b/next-5.10/rapidio-fix-possible-name-leaks-when-rio_add_device-.patch similarity index 100% rename from queue-5.10/rapidio-fix-possible-name-leaks-when-rio_add_device-.patch rename to next-5.10/rapidio-fix-possible-name-leaks-when-rio_add_device-.patch diff --git a/queue-5.10/rapidio-fix-possible-uaf-when-kfifo_alloc-fails.patch b/next-5.10/rapidio-fix-possible-uaf-when-kfifo_alloc-fails.patch similarity index 100% rename from queue-5.10/rapidio-fix-possible-uaf-when-kfifo_alloc-fails.patch rename to next-5.10/rapidio-fix-possible-uaf-when-kfifo_alloc-fails.patch diff --git a/queue-5.10/rapidio-rio-fix-possible-name-leak-in-rio_register_m.patch b/next-5.10/rapidio-rio-fix-possible-name-leak-in-rio_register_m.patch similarity index 100% rename from queue-5.10/rapidio-rio-fix-possible-name-leak-in-rio_register_m.patch rename to next-5.10/rapidio-rio-fix-possible-name-leak-in-rio_register_m.patch diff --git a/queue-5.10/rcu-fix-__this_cpu_read-lockdep-warning-in-rcu_force.patch b/next-5.10/rcu-fix-__this_cpu_read-lockdep-warning-in-rcu_force.patch similarity index 100% rename from queue-5.10/rcu-fix-__this_cpu_read-lockdep-warning-in-rcu_force.patch rename to next-5.10/rcu-fix-__this_cpu_read-lockdep-warning-in-rcu_force.patch diff --git a/queue-5.10/rdma-core-fix-order-of-nldev_exit-call.patch b/next-5.10/rdma-core-fix-order-of-nldev_exit-call.patch similarity index 100% rename from queue-5.10/rdma-core-fix-order-of-nldev_exit-call.patch rename to next-5.10/rdma-core-fix-order-of-nldev_exit-call.patch diff --git a/queue-5.10/rdma-hfi-decrease-pci-device-reference-count-in-erro.patch b/next-5.10/rdma-hfi-decrease-pci-device-reference-count-in-erro.patch similarity index 100% rename from queue-5.10/rdma-hfi-decrease-pci-device-reference-count-in-erro.patch rename to next-5.10/rdma-hfi-decrease-pci-device-reference-count-in-erro.patch diff --git a/queue-5.10/rdma-hfi1-fix-error-return-code-in-parse_platform_co.patch b/next-5.10/rdma-hfi1-fix-error-return-code-in-parse_platform_co.patch similarity index 100% rename from queue-5.10/rdma-hfi1-fix-error-return-code-in-parse_platform_co.patch rename to next-5.10/rdma-hfi1-fix-error-return-code-in-parse_platform_co.patch diff --git a/queue-5.10/rdma-hns-fix-ext_sge-num-error-when-post-send.patch b/next-5.10/rdma-hns-fix-ext_sge-num-error-when-post-send.patch similarity index 100% rename from queue-5.10/rdma-hns-fix-ext_sge-num-error-when-post-send.patch rename to next-5.10/rdma-hns-fix-ext_sge-num-error-when-post-send.patch diff --git a/queue-5.10/rdma-hns-fix-memory-leak-in-hns_roce_alloc_mr.patch b/next-5.10/rdma-hns-fix-memory-leak-in-hns_roce_alloc_mr.patch similarity index 100% rename from queue-5.10/rdma-hns-fix-memory-leak-in-hns_roce_alloc_mr.patch rename to next-5.10/rdma-hns-fix-memory-leak-in-hns_roce_alloc_mr.patch diff --git a/queue-5.10/rdma-hns-fix-page-size-cap-from-firmware.patch b/next-5.10/rdma-hns-fix-page-size-cap-from-firmware.patch similarity index 100% rename from queue-5.10/rdma-hns-fix-page-size-cap-from-firmware.patch rename to next-5.10/rdma-hns-fix-page-size-cap-from-firmware.patch diff --git a/queue-5.10/rdma-hns-fix-pbl-page-mtr-find.patch b/next-5.10/rdma-hns-fix-pbl-page-mtr-find.patch similarity index 100% rename from queue-5.10/rdma-hns-fix-pbl-page-mtr-find.patch rename to next-5.10/rdma-hns-fix-pbl-page-mtr-find.patch diff --git a/queue-5.10/rdma-hns-repacing-dseg_len-by-macros-in-fill_ext_sge.patch b/next-5.10/rdma-hns-repacing-dseg_len-by-macros-in-fill_ext_sge.patch similarity index 100% rename from queue-5.10/rdma-hns-repacing-dseg_len-by-macros-in-fill_ext_sge.patch rename to next-5.10/rdma-hns-repacing-dseg_len-by-macros-in-fill_ext_sge.patch diff --git a/queue-5.10/rdma-nldev-add-checks-for-nla_nest_start-in-fill_sta.patch b/next-5.10/rdma-nldev-add-checks-for-nla_nest_start-in-fill_sta.patch similarity index 100% rename from queue-5.10/rdma-nldev-add-checks-for-nla_nest_start-in-fill_sta.patch rename to next-5.10/rdma-nldev-add-checks-for-nla_nest_start-in-fill_sta.patch diff --git a/queue-5.10/rdma-nldev-fix-failure-to-send-large-messages.patch b/next-5.10/rdma-nldev-fix-failure-to-send-large-messages.patch similarity index 100% rename from queue-5.10/rdma-nldev-fix-failure-to-send-large-messages.patch rename to next-5.10/rdma-nldev-fix-failure-to-send-large-messages.patch diff --git a/queue-5.10/rdma-nldev-return-eagain-if-the-cm_id-isn-t-from-exp.patch b/next-5.10/rdma-nldev-return-eagain-if-the-cm_id-isn-t-from-exp.patch similarity index 100% rename from queue-5.10/rdma-nldev-return-eagain-if-the-cm_id-isn-t-from-exp.patch rename to next-5.10/rdma-nldev-return-eagain-if-the-cm_id-isn-t-from-exp.patch diff --git a/queue-5.10/rdma-rxe-fix-null-ptr-deref-in-rxe_qp_do_cleanup-whe.patch b/next-5.10/rdma-rxe-fix-null-ptr-deref-in-rxe_qp_do_cleanup-whe.patch similarity index 100% rename from queue-5.10/rdma-rxe-fix-null-ptr-deref-in-rxe_qp_do_cleanup-whe.patch rename to next-5.10/rdma-rxe-fix-null-ptr-deref-in-rxe_qp_do_cleanup-whe.patch diff --git a/queue-5.10/rdma-siw-fix-immediate-work-request-flush-to-complet.patch b/next-5.10/rdma-siw-fix-immediate-work-request-flush-to-complet.patch similarity index 100% rename from queue-5.10/rdma-siw-fix-immediate-work-request-flush-to-complet.patch rename to next-5.10/rdma-siw-fix-immediate-work-request-flush-to-complet.patch diff --git a/queue-5.10/rdma-siw-fix-pointer-cast-warning.patch b/next-5.10/rdma-siw-fix-pointer-cast-warning.patch similarity index 100% rename from queue-5.10/rdma-siw-fix-pointer-cast-warning.patch rename to next-5.10/rdma-siw-fix-pointer-cast-warning.patch diff --git a/queue-5.10/rdma-siw-set-defined-status-for-work-completion-with.patch b/next-5.10/rdma-siw-set-defined-status-for-work-completion-with.patch similarity index 100% rename from queue-5.10/rdma-siw-set-defined-status-for-work-completion-with.patch rename to next-5.10/rdma-siw-set-defined-status-for-work-completion-with.patch diff --git a/queue-5.10/rdma-srp-fix-error-return-code-in-srp_parse_options.patch b/next-5.10/rdma-srp-fix-error-return-code-in-srp_parse_options.patch similarity index 100% rename from queue-5.10/rdma-srp-fix-error-return-code-in-srp_parse_options.patch rename to next-5.10/rdma-srp-fix-error-return-code-in-srp_parse_options.patch diff --git a/queue-5.10/regulator-core-fix-deadlock-on-regulator-enable.patch b/next-5.10/regulator-core-fix-deadlock-on-regulator-enable.patch similarity index 100% rename from queue-5.10/regulator-core-fix-deadlock-on-regulator-enable.patch rename to next-5.10/regulator-core-fix-deadlock-on-regulator-enable.patch diff --git a/queue-5.10/regulator-core-fix-module-refcount-leak-in-set_suppl.patch b/next-5.10/regulator-core-fix-module-refcount-leak-in-set_suppl.patch similarity index 100% rename from queue-5.10/regulator-core-fix-module-refcount-leak-in-set_suppl.patch rename to next-5.10/regulator-core-fix-module-refcount-leak-in-set_suppl.patch diff --git a/queue-5.10/regulator-core-fix-resource-leak-in-regulator_regist.patch b/next-5.10/regulator-core-fix-resource-leak-in-regulator_regist.patch similarity index 100% rename from queue-5.10/regulator-core-fix-resource-leak-in-regulator_regist.patch rename to next-5.10/regulator-core-fix-resource-leak-in-regulator_regist.patch diff --git a/queue-5.10/regulator-core-fix-unbalanced-of-node-refcount-in-re.patch b/next-5.10/regulator-core-fix-unbalanced-of-node-refcount-in-re.patch similarity index 100% rename from queue-5.10/regulator-core-fix-unbalanced-of-node-refcount-in-re.patch rename to next-5.10/regulator-core-fix-unbalanced-of-node-refcount-in-re.patch diff --git a/queue-5.10/regulator-core-fix-use_count-leakage-when-handling-b.patch b/next-5.10/regulator-core-fix-use_count-leakage-when-handling-b.patch similarity index 100% rename from queue-5.10/regulator-core-fix-use_count-leakage-when-handling-b.patch rename to next-5.10/regulator-core-fix-use_count-leakage-when-handling-b.patch diff --git a/queue-5.10/regulator-core-use-kfree_const-to-free-space-conditi.patch b/next-5.10/regulator-core-use-kfree_const-to-free-space-conditi.patch similarity index 100% rename from queue-5.10/regulator-core-use-kfree_const-to-free-space-conditi.patch rename to next-5.10/regulator-core-use-kfree_const-to-free-space-conditi.patch diff --git a/queue-5.10/reiserfs-add-missing-calls-to-reiserfs_security_free.patch b/next-5.10/reiserfs-add-missing-calls-to-reiserfs_security_free.patch similarity index 100% rename from queue-5.10/reiserfs-add-missing-calls-to-reiserfs_security_free.patch rename to next-5.10/reiserfs-add-missing-calls-to-reiserfs_security_free.patch diff --git a/queue-5.10/relay-fix-type-mismatch-when-allocating-memory-in-re.patch b/next-5.10/relay-fix-type-mismatch-when-allocating-memory-in-re.patch similarity index 100% rename from queue-5.10/relay-fix-type-mismatch-when-allocating-memory-in-re.patch rename to next-5.10/relay-fix-type-mismatch-when-allocating-memory-in-re.patch diff --git a/queue-5.10/remoteproc-qcom_q6v5_pas-detach-power-domains-on-rem.patch b/next-5.10/remoteproc-qcom_q6v5_pas-detach-power-domains-on-rem.patch similarity index 100% rename from queue-5.10/remoteproc-qcom_q6v5_pas-detach-power-domains-on-rem.patch rename to next-5.10/remoteproc-qcom_q6v5_pas-detach-power-domains-on-rem.patch diff --git a/queue-5.10/remoteproc-qcom_q6v5_pas-disable-wakeup-on-probe-fai.patch b/next-5.10/remoteproc-qcom_q6v5_pas-disable-wakeup-on-probe-fai.patch similarity index 100% rename from queue-5.10/remoteproc-qcom_q6v5_pas-disable-wakeup-on-probe-fai.patch rename to next-5.10/remoteproc-qcom_q6v5_pas-disable-wakeup-on-probe-fai.patch diff --git a/queue-5.10/remoteproc-qcom_q6v5_pas-fix-missing-of_node_put-in-.patch b/next-5.10/remoteproc-qcom_q6v5_pas-fix-missing-of_node_put-in-.patch similarity index 100% rename from queue-5.10/remoteproc-qcom_q6v5_pas-fix-missing-of_node_put-in-.patch rename to next-5.10/remoteproc-qcom_q6v5_pas-fix-missing-of_node_put-in-.patch diff --git a/queue-5.10/remoteproc-sysmon-fix-memory-leak-in-qcom_add_sysmon.patch b/next-5.10/remoteproc-sysmon-fix-memory-leak-in-qcom_add_sysmon.patch similarity index 100% rename from queue-5.10/remoteproc-sysmon-fix-memory-leak-in-qcom_add_sysmon.patch rename to next-5.10/remoteproc-sysmon-fix-memory-leak-in-qcom_add_sysmon.patch diff --git a/next-5.10/revert-proc-don-t-allow-async-path-resolution-of-proc-self-components.patch b/next-5.10/revert-proc-don-t-allow-async-path-resolution-of-proc-self-components.patch new file mode 100644 index 00000000000..d5b8f40d4df --- /dev/null +++ b/next-5.10/revert-proc-don-t-allow-async-path-resolution-of-proc-self-components.patch @@ -0,0 +1,35 @@ +From 2d7b7e632050476301e4fdfaf7a55751c97e6a83 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Mon, 15 Feb 2021 13:42:42 -0700 +Subject: Revert "proc: don't allow async path resolution of /proc/self components" + +From: Jens Axboe + +[ Upstream commit 9e8d9e829c2142cf1d7756e9ed2e0b4c7569d84c ] + +This reverts commit 8d4c3e76e3be11a64df95ddee52e99092d42fc19. + +No longer needed, as the io-wq worker threads have the right identity. + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + fs/proc/self.c | 7 ------- + 1 file changed, 7 deletions(-) + +--- a/fs/proc/self.c ++++ b/fs/proc/self.c +@@ -16,13 +16,6 @@ static const char *proc_self_get_link(st + pid_t tgid = task_tgid_nr_ns(current, ns); + char *name; + +- /* +- * Not currently supported. Once we can inherit all of struct pid, +- * we can allow this. +- */ +- if (current->flags & PF_KTHREAD) +- return ERR_PTR(-EOPNOTSUPP); +- + if (!tgid) + return ERR_PTR(-ENOENT); + /* max length of unsigned int in decimal + NULL term */ diff --git a/next-5.10/revert-proc-don-t-allow-async-path-resolution-of-proc-thread-self-components.patch b/next-5.10/revert-proc-don-t-allow-async-path-resolution-of-proc-thread-self-components.patch new file mode 100644 index 00000000000..c8fa41d9dea --- /dev/null +++ b/next-5.10/revert-proc-don-t-allow-async-path-resolution-of-proc-thread-self-components.patch @@ -0,0 +1,47 @@ +From 361b2a5293b998f8903a5d343ad5cf683a2970fa Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Mon, 15 Feb 2021 13:42:18 -0700 +Subject: Revert "proc: don't allow async path resolution of /proc/thread-self components" + +From: Jens Axboe + +[ Upstream commit 2587890b5e2892dfecaa5e5126bdac8076a4e6f7 ] + +This reverts commit 0d4370cfe36b7f1719123b621a4ec4d9c7a25f89. + +No longer needed, as the io-wq worker threads have the right identity. + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + fs/proc/self.c | 2 +- + fs/proc/thread_self.c | 7 ------- + 2 files changed, 1 insertion(+), 8 deletions(-) + +--- a/fs/proc/self.c ++++ b/fs/proc/self.c +@@ -20,7 +20,7 @@ static const char *proc_self_get_link(st + * Not currently supported. Once we can inherit all of struct pid, + * we can allow this. + */ +- if (current->flags & PF_IO_WORKER) ++ if (current->flags & PF_KTHREAD) + return ERR_PTR(-EOPNOTSUPP); + + if (!tgid) +--- a/fs/proc/thread_self.c ++++ b/fs/proc/thread_self.c +@@ -17,13 +17,6 @@ static const char *proc_thread_self_get_ + pid_t pid = task_pid_nr_ns(current, ns); + char *name; + +- /* +- * Not currently supported. Once we can inherit all of struct pid, +- * we can allow this. +- */ +- if (current->flags & PF_IO_WORKER) +- return ERR_PTR(-EOPNOTSUPP); +- + if (!pid) + return ERR_PTR(-ENOENT); + name = kmalloc(10 + 6 + 10 + 1, dentry ? GFP_KERNEL : GFP_ATOMIC); diff --git a/next-5.10/riscv-add-support-for-tif_notify_signal.patch b/next-5.10/riscv-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..9380226552b --- /dev/null +++ b/next-5.10/riscv-add-support-for-tif_notify_signal.patch @@ -0,0 +1,53 @@ +From 34a226ec555bf65f3b49bd173889554a04b5882a Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 14:29:17 -0600 +Subject: riscv: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 24a31b81e38309b1604f24520110aae1f83f3cbf ] + +Wire up TIF_NOTIFY_SIGNAL handling for riscv. + +Cc: linux-riscv@lists.infradead.org +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/riscv/include/asm/thread_info.h | 5 ++++- + arch/riscv/kernel/signal.c | 2 +- + 2 files changed, 5 insertions(+), 2 deletions(-) + +--- a/arch/riscv/include/asm/thread_info.h ++++ b/arch/riscv/include/asm/thread_info.h +@@ -80,6 +80,7 @@ struct thread_info { + #define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */ + #define TIF_SYSCALL_AUDIT 7 /* syscall auditing */ + #define TIF_SECCOMP 8 /* syscall secure computing */ ++#define TIF_NOTIFY_SIGNAL 9 /* signal notifications exist */ + + #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) + #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) +@@ -88,9 +89,11 @@ struct thread_info { + #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) + #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) + #define _TIF_SECCOMP (1 << TIF_SECCOMP) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + + #define _TIF_WORK_MASK \ +- (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED) ++ (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \ ++ _TIF_NOTIFY_SIGNAL) + + #define _TIF_SYSCALL_WORK \ + (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT | \ +--- a/arch/riscv/kernel/signal.c ++++ b/arch/riscv/kernel/signal.c +@@ -312,7 +312,7 @@ asmlinkage __visible void do_notify_resu + unsigned long thread_info_flags) + { + /* Handle pending signal delivery */ +- if (thread_info_flags & _TIF_SIGPENDING) ++ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs); + + if (thread_info_flags & _TIF_NOTIFY_RESUME) diff --git a/queue-5.10/riscv-mm-add-arch-hook-arch_clear_hugepage_flags.patch b/next-5.10/riscv-mm-add-arch-hook-arch_clear_hugepage_flags.patch similarity index 100% rename from queue-5.10/riscv-mm-add-arch-hook-arch_clear_hugepage_flags.patch rename to next-5.10/riscv-mm-add-arch-hook-arch_clear_hugepage_flags.patch diff --git a/queue-5.10/rtc-cmos-call-cmos_wake_setup-from-cmos_do_probe.patch b/next-5.10/rtc-cmos-call-cmos_wake_setup-from-cmos_do_probe.patch similarity index 100% rename from queue-5.10/rtc-cmos-call-cmos_wake_setup-from-cmos_do_probe.patch rename to next-5.10/rtc-cmos-call-cmos_wake_setup-from-cmos_do_probe.patch diff --git a/queue-5.10/rtc-cmos-call-rtc_wake_setup-from-cmos_do_probe.patch b/next-5.10/rtc-cmos-call-rtc_wake_setup-from-cmos_do_probe.patch similarity index 100% rename from queue-5.10/rtc-cmos-call-rtc_wake_setup-from-cmos_do_probe.patch rename to next-5.10/rtc-cmos-call-rtc_wake_setup-from-cmos_do_probe.patch diff --git a/queue-5.10/rtc-cmos-disable-acpi-rtc-event-on-removal.patch b/next-5.10/rtc-cmos-disable-acpi-rtc-event-on-removal.patch similarity index 100% rename from queue-5.10/rtc-cmos-disable-acpi-rtc-event-on-removal.patch rename to next-5.10/rtc-cmos-disable-acpi-rtc-event-on-removal.patch diff --git a/queue-5.10/rtc-cmos-eliminate-forward-declarations-of-some-func.patch b/next-5.10/rtc-cmos-eliminate-forward-declarations-of-some-func.patch similarity index 100% rename from queue-5.10/rtc-cmos-eliminate-forward-declarations-of-some-func.patch rename to next-5.10/rtc-cmos-eliminate-forward-declarations-of-some-func.patch diff --git a/queue-5.10/rtc-cmos-fix-build-on-non-acpi-platforms.patch b/next-5.10/rtc-cmos-fix-build-on-non-acpi-platforms.patch similarity index 100% rename from queue-5.10/rtc-cmos-fix-build-on-non-acpi-platforms.patch rename to next-5.10/rtc-cmos-fix-build-on-non-acpi-platforms.patch diff --git a/queue-5.10/rtc-cmos-fix-event-handler-registration-ordering-iss.patch b/next-5.10/rtc-cmos-fix-event-handler-registration-ordering-iss.patch similarity index 100% rename from queue-5.10/rtc-cmos-fix-event-handler-registration-ordering-iss.patch rename to next-5.10/rtc-cmos-fix-event-handler-registration-ordering-iss.patch diff --git a/queue-5.10/rtc-cmos-fix-wake-alarm-breakage.patch b/next-5.10/rtc-cmos-fix-wake-alarm-breakage.patch similarity index 100% rename from queue-5.10/rtc-cmos-fix-wake-alarm-breakage.patch rename to next-5.10/rtc-cmos-fix-wake-alarm-breakage.patch diff --git a/queue-5.10/rtc-cmos-rename-acpi-related-functions.patch b/next-5.10/rtc-cmos-rename-acpi-related-functions.patch similarity index 100% rename from queue-5.10/rtc-cmos-rename-acpi-related-functions.patch rename to next-5.10/rtc-cmos-rename-acpi-related-functions.patch diff --git a/queue-5.10/rtc-mxc_v2-add-missing-clk_disable_unprepare.patch b/next-5.10/rtc-mxc_v2-add-missing-clk_disable_unprepare.patch similarity index 100% rename from queue-5.10/rtc-mxc_v2-add-missing-clk_disable_unprepare.patch rename to next-5.10/rtc-mxc_v2-add-missing-clk_disable_unprepare.patch diff --git a/queue-5.10/rtc-pcf85063-fix-pcf85063_clkout_control.patch b/next-5.10/rtc-pcf85063-fix-pcf85063_clkout_control.patch similarity index 100% rename from queue-5.10/rtc-pcf85063-fix-pcf85063_clkout_control.patch rename to next-5.10/rtc-pcf85063-fix-pcf85063_clkout_control.patch diff --git a/queue-5.10/rtc-pcf85063-fix-reading-alarm.patch b/next-5.10/rtc-pcf85063-fix-reading-alarm.patch similarity index 100% rename from queue-5.10/rtc-pcf85063-fix-reading-alarm.patch rename to next-5.10/rtc-pcf85063-fix-reading-alarm.patch diff --git a/queue-5.10/rtc-pic32-move-devm_rtc_allocate_device-earlier-in-p.patch b/next-5.10/rtc-pic32-move-devm_rtc_allocate_device-earlier-in-p.patch similarity index 100% rename from queue-5.10/rtc-pic32-move-devm_rtc_allocate_device-earlier-in-p.patch rename to next-5.10/rtc-pic32-move-devm_rtc_allocate_device-earlier-in-p.patch diff --git a/queue-5.10/rtc-rtc-cmos-do-not-check-acpi_fadt_low_power_s0.patch b/next-5.10/rtc-rtc-cmos-do-not-check-acpi_fadt_low_power_s0.patch similarity index 100% rename from queue-5.10/rtc-rtc-cmos-do-not-check-acpi_fadt_low_power_s0.patch rename to next-5.10/rtc-rtc-cmos-do-not-check-acpi_fadt_low_power_s0.patch diff --git a/queue-5.10/rtc-snvs-allow-a-time-difference-on-clock-register-r.patch b/next-5.10/rtc-snvs-allow-a-time-difference-on-clock-register-r.patch similarity index 100% rename from queue-5.10/rtc-snvs-allow-a-time-difference-on-clock-register-r.patch rename to next-5.10/rtc-snvs-allow-a-time-difference-on-clock-register-r.patch diff --git a/queue-5.10/rtc-st-lpc-add-missing-clk_disable_unprepare-in-st_r.patch b/next-5.10/rtc-st-lpc-add-missing-clk_disable_unprepare-in-st_r.patch similarity index 100% rename from queue-5.10/rtc-st-lpc-add-missing-clk_disable_unprepare-in-st_r.patch rename to next-5.10/rtc-st-lpc-add-missing-clk_disable_unprepare-in-st_r.patch diff --git a/queue-5.10/rxrpc-fix-ack.buffersize-to-be-0-when-generating-an-.patch b/next-5.10/rxrpc-fix-ack.buffersize-to-be-0-when-generating-an-.patch similarity index 100% rename from queue-5.10/rxrpc-fix-ack.buffersize-to-be-0-when-generating-an-.patch rename to next-5.10/rxrpc-fix-ack.buffersize-to-be-0-when-generating-an-.patch diff --git a/queue-5.10/rxrpc-fix-missing-unlock-in-rxrpc_do_sendmsg.patch b/next-5.10/rxrpc-fix-missing-unlock-in-rxrpc_do_sendmsg.patch similarity index 100% rename from queue-5.10/rxrpc-fix-missing-unlock-in-rxrpc_do_sendmsg.patch rename to next-5.10/rxrpc-fix-missing-unlock-in-rxrpc_do_sendmsg.patch diff --git a/next-5.10/s390-add-support-for-tif_notify_signal.patch b/next-5.10/s390-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..a5730bc611e --- /dev/null +++ b/next-5.10/s390-add-support-for-tif_notify_signal.patch @@ -0,0 +1,85 @@ +From 832028e42ad081b88ee7a72c8c9bcdea6eed3f97 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 15:34:12 -0600 +Subject: s390: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 75309018a24ddfb930c51bad8f4070b9bc2c923b ] + +Wire up TIF_NOTIFY_SIGNAL handling for s390. + +Cc: linux-s390@vger.kernel.org +Acked-by: Heiko Carstens +Acked-by: Sven Schnelle +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/s390/include/asm/thread_info.h | 2 ++ + arch/s390/kernel/entry.S | 11 ++++++----- + arch/s390/kernel/signal.c | 2 +- + 3 files changed, 9 insertions(+), 6 deletions(-) + +--- a/arch/s390/include/asm/thread_info.h ++++ b/arch/s390/include/asm/thread_info.h +@@ -65,6 +65,7 @@ void arch_setup_new_exec(void); + #define TIF_GUARDED_STORAGE 4 /* load guarded storage control block */ + #define TIF_PATCH_PENDING 5 /* pending live patching update */ + #define TIF_PGSTE 6 /* New mm's will use 4K page tables */ ++#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ + #define TIF_ISOLATE_BP 8 /* Run process with isolated BP */ + #define TIF_ISOLATE_BP_GUEST 9 /* Run KVM guests with isolated BP */ + +@@ -82,6 +83,7 @@ void arch_setup_new_exec(void); + #define TIF_SYSCALL_TRACEPOINT 27 /* syscall tracepoint instrumentation */ + + #define _TIF_NOTIFY_RESUME BIT(TIF_NOTIFY_RESUME) ++#define _TIF_NOTIFY_SIGNAL BIT(TIF_NOTIFY_SIGNAL) + #define _TIF_SIGPENDING BIT(TIF_SIGPENDING) + #define _TIF_NEED_RESCHED BIT(TIF_NEED_RESCHED) + #define _TIF_UPROBE BIT(TIF_UPROBE) +--- a/arch/s390/kernel/entry.S ++++ b/arch/s390/kernel/entry.S +@@ -52,7 +52,8 @@ STACK_SIZE = 1 << STACK_SHIFT + STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE + + _TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ +- _TIF_UPROBE | _TIF_GUARDED_STORAGE | _TIF_PATCH_PENDING) ++ _TIF_UPROBE | _TIF_GUARDED_STORAGE | _TIF_PATCH_PENDING | \ ++ _TIF_NOTIFY_SIGNAL) + _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ + _TIF_SYSCALL_TRACEPOINT) + _CIF_WORK = (_CIF_ASCE_PRIMARY | _CIF_ASCE_SECONDARY | _CIF_FPU) +@@ -481,8 +482,8 @@ ENTRY(system_call) + #endif + TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL_RESTART + jo .Lsysc_syscall_restart +- TSTMSK __TI_flags(%r12),_TIF_SIGPENDING +- jo .Lsysc_sigpending ++ TSTMSK __TI_flags(%r12),(_TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL) ++ jnz .Lsysc_sigpending + TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME + jo .Lsysc_notify_resume + TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY) +@@ -863,8 +864,8 @@ ENTRY(io_int_handler) + TSTMSK __TI_flags(%r12),_TIF_PATCH_PENDING + jo .Lio_patch_pending + #endif +- TSTMSK __TI_flags(%r12),_TIF_SIGPENDING +- jo .Lio_sigpending ++ TSTMSK __TI_flags(%r12),(_TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL) ++ jnz .Lio_sigpending + TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME + jo .Lio_notify_resume + TSTMSK __TI_flags(%r12),_TIF_GUARDED_STORAGE +--- a/arch/s390/kernel/signal.c ++++ b/arch/s390/kernel/signal.c +@@ -472,7 +472,7 @@ void do_signal(struct pt_regs *regs) + current->thread.system_call = + test_pt_regs_flag(regs, PIF_SYSCALL) ? regs->int_code : 0; + +- if (get_signal(&ksig)) { ++ if (test_thread_flag(TIF_SIGPENDING) && get_signal(&ksig)) { + /* Whee! Actually deliver the signal. */ + if (current->thread.system_call) { + regs->int_code = current->thread.system_call; diff --git a/queue-5.10/s390-ctcm-fix-return-type-of-ctc-mp-m_tx.patch b/next-5.10/s390-ctcm-fix-return-type-of-ctc-mp-m_tx.patch similarity index 100% rename from queue-5.10/s390-ctcm-fix-return-type-of-ctc-mp-m_tx.patch rename to next-5.10/s390-ctcm-fix-return-type-of-ctc-mp-m_tx.patch diff --git a/queue-5.10/s390-lcs-fix-return-type-of-lcs_start_xmit.patch b/next-5.10/s390-lcs-fix-return-type-of-lcs_start_xmit.patch similarity index 100% rename from queue-5.10/s390-lcs-fix-return-type-of-lcs_start_xmit.patch rename to next-5.10/s390-lcs-fix-return-type-of-lcs_start_xmit.patch diff --git a/queue-5.10/s390-netiucv-fix-return-type-of-netiucv_tx.patch b/next-5.10/s390-netiucv-fix-return-type-of-netiucv_tx.patch similarity index 100% rename from queue-5.10/s390-netiucv-fix-return-type-of-netiucv_tx.patch rename to next-5.10/s390-netiucv-fix-return-type-of-netiucv_tx.patch diff --git a/queue-5.10/samples-vfio-mdev-fix-missing-pci_disable_device-in-.patch b/next-5.10/samples-vfio-mdev-fix-missing-pci_disable_device-in-.patch similarity index 100% rename from queue-5.10/samples-vfio-mdev-fix-missing-pci_disable_device-in-.patch rename to next-5.10/samples-vfio-mdev-fix-missing-pci_disable_device-in-.patch diff --git a/next-5.10/saner-calling-conventions-for-unlazy_child.patch b/next-5.10/saner-calling-conventions-for-unlazy_child.patch new file mode 100644 index 00000000000..8078551db8b --- /dev/null +++ b/next-5.10/saner-calling-conventions-for-unlazy_child.patch @@ -0,0 +1,91 @@ +From 2f065a20a2d0b75c25a9125f452991049dcd4205 Mon Sep 17 00:00:00 2001 +From: Al Viro +Date: Mon, 4 Jan 2021 00:08:41 -0500 +Subject: saner calling conventions for unlazy_child() + +From: Al Viro + +[ Upstream commit ae66db45fd309fd1c6d4e846dfc8414dfec7d6ad ] + +same as for the previous commit - instead of 0/-ECHILD make +it return true/false, rename to try_to_unlazy_child(). + +Signed-off-by: Al Viro +Signed-off-by: Greg Kroah-Hartman +--- + fs/namei.c | 30 +++++++++++++++--------------- + 1 file changed, 15 insertions(+), 15 deletions(-) + +--- a/fs/namei.c ++++ b/fs/namei.c +@@ -705,19 +705,19 @@ out: + } + + /** +- * unlazy_child - try to switch to ref-walk mode. ++ * try_to_unlazy_next - try to switch to ref-walk mode. + * @nd: nameidata pathwalk data +- * @dentry: child of nd->path.dentry +- * @seq: seq number to check dentry against +- * Returns: 0 on success, -ECHILD on failure +- * +- * unlazy_child attempts to legitimize the current nd->path, nd->root and dentry +- * for ref-walk mode. @dentry must be a path found by a do_lookup call on +- * @nd. Must be called from rcu-walk context. +- * Nothing should touch nameidata between unlazy_child() failure and ++ * @dentry: next dentry to step into ++ * @seq: seq number to check @dentry against ++ * Returns: true on success, false on failure ++ * ++ * Similar to to try_to_unlazy(), but here we have the next dentry already ++ * picked by rcu-walk and want to legitimize that in addition to the current ++ * nd->path and nd->root for ref-walk mode. Must be called from rcu-walk context. ++ * Nothing should touch nameidata between try_to_unlazy_next() failure and + * terminate_walk(). + */ +-static int unlazy_child(struct nameidata *nd, struct dentry *dentry, unsigned seq) ++static bool try_to_unlazy_next(struct nameidata *nd, struct dentry *dentry, unsigned seq) + { + BUG_ON(!(nd->flags & LOOKUP_RCU)); + +@@ -747,7 +747,7 @@ static int unlazy_child(struct nameidata + if (unlikely(!legitimize_root(nd))) + goto out_dput; + rcu_read_unlock(); +- return 0; ++ return true; + + out2: + nd->path.mnt = NULL; +@@ -755,11 +755,11 @@ out1: + nd->path.dentry = NULL; + out: + rcu_read_unlock(); +- return -ECHILD; ++ return false; + out_dput: + rcu_read_unlock(); + dput(dentry); +- return -ECHILD; ++ return false; + } + + static inline int d_revalidate(struct dentry *dentry, unsigned int flags) +@@ -1374,7 +1374,7 @@ static inline int handle_mounts(struct n + return -ENOENT; + if (likely(__follow_mount_rcu(nd, path, inode, seqp))) + return 0; +- if (unlazy_child(nd, dentry, seq)) ++ if (!try_to_unlazy_next(nd, dentry, seq)) + return -ECHILD; + // *path might've been clobbered by __follow_mount_rcu() + path->mnt = nd->path.mnt; +@@ -1495,7 +1495,7 @@ static struct dentry *lookup_fast(struct + status = d_revalidate(dentry, nd->flags); + if (likely(status > 0)) + return dentry; +- if (unlazy_child(nd, dentry, seq)) ++ if (!try_to_unlazy_next(nd, dentry, seq)) + return ERR_PTR(-ECHILD); + if (unlikely(status == -ECHILD)) + /* we'd been told to redo it in non-rcu mode */ diff --git a/queue-5.10/sched-fair-cleanup-task_util-and-capacity-type.patch b/next-5.10/sched-fair-cleanup-task_util-and-capacity-type.patch similarity index 100% rename from queue-5.10/sched-fair-cleanup-task_util-and-capacity-type.patch rename to next-5.10/sched-fair-cleanup-task_util-and-capacity-type.patch diff --git a/queue-5.10/sched-uclamp-fix-relationship-between-uclamp-and-mig.patch b/next-5.10/sched-uclamp-fix-relationship-between-uclamp-and-mig.patch similarity index 100% rename from queue-5.10/sched-uclamp-fix-relationship-between-uclamp-and-mig.patch rename to next-5.10/sched-uclamp-fix-relationship-between-uclamp-and-mig.patch diff --git a/queue-5.10/scsi-core-fix-a-race-between-scsi_done-and-scsi_time.patch b/next-5.10/scsi-core-fix-a-race-between-scsi_done-and-scsi_time.patch similarity index 100% rename from queue-5.10/scsi-core-fix-a-race-between-scsi_done-and-scsi_time.patch rename to next-5.10/scsi-core-fix-a-race-between-scsi_done-and-scsi_time.patch diff --git a/queue-5.10/scsi-fcoe-fix-possible-name-leak-when-device_registe.patch b/next-5.10/scsi-fcoe-fix-possible-name-leak-when-device_registe.patch similarity index 100% rename from queue-5.10/scsi-fcoe-fix-possible-name-leak-when-device_registe.patch rename to next-5.10/scsi-fcoe-fix-possible-name-leak-when-device_registe.patch diff --git a/queue-5.10/scsi-fcoe-fix-transport-not-deattached-when-fcoe_if_.patch b/next-5.10/scsi-fcoe-fix-transport-not-deattached-when-fcoe_if_.patch similarity index 100% rename from queue-5.10/scsi-fcoe-fix-transport-not-deattached-when-fcoe_if_.patch rename to next-5.10/scsi-fcoe-fix-transport-not-deattached-when-fcoe_if_.patch diff --git a/queue-5.10/scsi-hpsa-fix-error-handling-in-hpsa_add_sas_host.patch b/next-5.10/scsi-hpsa-fix-error-handling-in-hpsa_add_sas_host.patch similarity index 100% rename from queue-5.10/scsi-hpsa-fix-error-handling-in-hpsa_add_sas_host.patch rename to next-5.10/scsi-hpsa-fix-error-handling-in-hpsa_add_sas_host.patch diff --git a/queue-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_add_sas_d.patch b/next-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_add_sas_d.patch similarity index 100% rename from queue-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_add_sas_d.patch rename to next-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_add_sas_d.patch diff --git a/queue-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_init_one.patch b/next-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_init_one.patch similarity index 100% rename from queue-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_init_one.patch rename to next-5.10/scsi-hpsa-fix-possible-memory-leak-in-hpsa_init_one.patch diff --git a/queue-5.10/scsi-ipr-fix-warning-in-ipr_init.patch b/next-5.10/scsi-ipr-fix-warning-in-ipr_init.patch similarity index 100% rename from queue-5.10/scsi-ipr-fix-warning-in-ipr_init.patch rename to next-5.10/scsi-ipr-fix-warning-in-ipr_init.patch diff --git a/queue-5.10/scsi-mpt3sas-fix-possible-resource-leaks-in-mpt3sas_.patch b/next-5.10/scsi-mpt3sas-fix-possible-resource-leaks-in-mpt3sas_.patch similarity index 100% rename from queue-5.10/scsi-mpt3sas-fix-possible-resource-leaks-in-mpt3sas_.patch rename to next-5.10/scsi-mpt3sas-fix-possible-resource-leaks-in-mpt3sas_.patch diff --git a/queue-5.10/scsi-scsi_debug-fix-a-warning-in-resp_report_zones.patch b/next-5.10/scsi-scsi_debug-fix-a-warning-in-resp_report_zones.patch similarity index 100% rename from queue-5.10/scsi-scsi_debug-fix-a-warning-in-resp_report_zones.patch rename to next-5.10/scsi-scsi_debug-fix-a-warning-in-resp_report_zones.patch diff --git a/queue-5.10/scsi-scsi_debug-fix-a-warning-in-resp_verify.patch b/next-5.10/scsi-scsi_debug-fix-a-warning-in-resp_verify.patch similarity index 100% rename from queue-5.10/scsi-scsi_debug-fix-a-warning-in-resp_verify.patch rename to next-5.10/scsi-scsi_debug-fix-a-warning-in-resp_verify.patch diff --git a/queue-5.10/scsi-scsi_debug-fix-a-warning-in-resp_write_scat.patch b/next-5.10/scsi-scsi_debug-fix-a-warning-in-resp_write_scat.patch similarity index 100% rename from queue-5.10/scsi-scsi_debug-fix-a-warning-in-resp_write_scat.patch rename to next-5.10/scsi-scsi_debug-fix-a-warning-in-resp_write_scat.patch diff --git a/queue-5.10/scsi-scsi_debug-fix-possible-name-leak-in-sdebug_add.patch b/next-5.10/scsi-scsi_debug-fix-possible-name-leak-in-sdebug_add.patch similarity index 100% rename from queue-5.10/scsi-scsi_debug-fix-possible-name-leak-in-sdebug_add.patch rename to next-5.10/scsi-scsi_debug-fix-possible-name-leak-in-sdebug_add.patch diff --git a/queue-5.10/scsi-snic-fix-possible-uaf-in-snic_tgt_create.patch b/next-5.10/scsi-snic-fix-possible-uaf-in-snic_tgt_create.patch similarity index 100% rename from queue-5.10/scsi-snic-fix-possible-uaf-in-snic_tgt_create.patch rename to next-5.10/scsi-snic-fix-possible-uaf-in-snic_tgt_create.patch diff --git a/queue-5.10/sctp-sysctl-make-extra-pointers-netns-aware.patch b/next-5.10/sctp-sysctl-make-extra-pointers-netns-aware.patch similarity index 100% rename from queue-5.10/sctp-sysctl-make-extra-pointers-netns-aware.patch rename to next-5.10/sctp-sysctl-make-extra-pointers-netns-aware.patch diff --git a/queue-5.10/selftests-devlink-fix-the-fd-redirect-in-dummy_repor.patch b/next-5.10/selftests-devlink-fix-the-fd-redirect-in-dummy_repor.patch similarity index 100% rename from queue-5.10/selftests-devlink-fix-the-fd-redirect-in-dummy_repor.patch rename to next-5.10/selftests-devlink-fix-the-fd-redirect-in-dummy_repor.patch diff --git a/queue-5.10/selftests-efivarfs-add-checking-of-the-test-return-v.patch b/next-5.10/selftests-efivarfs-add-checking-of-the-test-return-v.patch similarity index 100% rename from queue-5.10/selftests-efivarfs-add-checking-of-the-test-return-v.patch rename to next-5.10/selftests-efivarfs-add-checking-of-the-test-return-v.patch diff --git a/queue-5.10/selftests-ftrace-event_triggers-wait-longer-for-test.patch b/next-5.10/selftests-ftrace-event_triggers-wait-longer-for-test.patch similarity index 100% rename from queue-5.10/selftests-ftrace-event_triggers-wait-longer-for-test.patch rename to next-5.10/selftests-ftrace-event_triggers-wait-longer-for-test.patch diff --git a/queue-5.10/selftests-powerpc-fix-resource-leaks.patch b/next-5.10/selftests-powerpc-fix-resource-leaks.patch similarity index 100% rename from queue-5.10/selftests-powerpc-fix-resource-leaks.patch rename to next-5.10/selftests-powerpc-fix-resource-leaks.patch diff --git a/queue-5.10/serial-altera_uart-fix-locking-in-polling-mode.patch b/next-5.10/serial-altera_uart-fix-locking-in-polling-mode.patch similarity index 100% rename from queue-5.10/serial-altera_uart-fix-locking-in-polling-mode.patch rename to next-5.10/serial-altera_uart-fix-locking-in-polling-mode.patch diff --git a/queue-5.10/serial-amba-pl011-avoid-sbsa-uart-accessing-dmacr-re.patch b/next-5.10/serial-amba-pl011-avoid-sbsa-uart-accessing-dmacr-re.patch similarity index 100% rename from queue-5.10/serial-amba-pl011-avoid-sbsa-uart-accessing-dmacr-re.patch rename to next-5.10/serial-amba-pl011-avoid-sbsa-uart-accessing-dmacr-re.patch diff --git a/queue-5.10/serial-pch-fix-pci-device-refcount-leak-in-pch_reque.patch b/next-5.10/serial-pch-fix-pci-device-refcount-leak-in-pch_reque.patch similarity index 100% rename from queue-5.10/serial-pch-fix-pci-device-refcount-leak-in-pch_reque.patch rename to next-5.10/serial-pch-fix-pci-device-refcount-leak-in-pch_reque.patch diff --git a/queue-5.10/serial-pl011-do-not-clear-rx-fifo-rx-interrupt-in-un.patch b/next-5.10/serial-pl011-do-not-clear-rx-fifo-rx-interrupt-in-un.patch similarity index 100% rename from queue-5.10/serial-pl011-do-not-clear-rx-fifo-rx-interrupt-in-un.patch rename to next-5.10/serial-pl011-do-not-clear-rx-fifo-rx-interrupt-in-un.patch diff --git a/queue-5.10/serial-sunsab-fix-error-handling-in-sunsab_init.patch b/next-5.10/serial-sunsab-fix-error-handling-in-sunsab_init.patch similarity index 100% rename from queue-5.10/serial-sunsab-fix-error-handling-in-sunsab_init.patch rename to next-5.10/serial-sunsab-fix-error-handling-in-sunsab_init.patch diff --git a/queue-5.10/serial-tegra-read-dma-status-before-terminating.patch b/next-5.10/serial-tegra-read-dma-status-before-terminating.patch similarity index 100% rename from queue-5.10/serial-tegra-read-dma-status-before-terminating.patch rename to next-5.10/serial-tegra-read-dma-status-before-terminating.patch diff --git a/next-5.10/series b/next-5.10/series new file mode 100644 index 00000000000..ba4de0dadb8 --- /dev/null +++ b/next-5.10/series @@ -0,0 +1,663 @@ +usb-musb-remove-extra-check-in-musb_gadget_vbus_draw.patch +arm64-dts-qcom-ipq6018-cp01-c1-use-blspi1-pins.patch +arm64-dts-qcom-msm8996-fix-gpu-opp-table.patch +arm-dts-qcom-apq8064-fix-coresight-compatible.patch +arm64-dts-qcom-sdm630-fix-uart1-pin-bias.patch +arm64-dts-qcom-sdm845-cheza-fix-ap-suspend-pin-bias.patch +arm64-dts-qcom-msm8916-drop-mss-fallback-compatible.patch +objtool-kcsan-add-volatile-read-write-instrumentatio.patch +arm-dts-stm32-drop-stm32mp15xc.dtsi-from-avenger96.patch +arm-dts-stm32-fix-av96-wlan-regulator-gpio-property.patch +drivers-soc-ti-knav_qmss_queue-mark-knav_acc_firmwar.patch +soc-qcom-llcc-make-irq-truly-optional.patch +soc-qcom-apr-make-code-more-reuseable.patch +soc-qcom-apr-add-check-for-idr_alloc-and-of_property.patch +arm-dts-spear600-fix-clcd-interrupt.patch +soc-ti-knav_qmss_queue-use-pm_runtime_resume_and_get.patch +soc-ti-knav_qmss_queue-fix-pm-disable-depth-imbalanc.patch +soc-ti-smartreflex-fix-pm-disable-depth-imbalance-in.patch +perf-arm_dsu-fix-hotplug-callback-leak-in-dsu_pmu_in.patch +perf-smmuv3-fix-hotplug-callback-leak-in-arm_smmu_pm.patch +arm64-dts-ti-k3-am65-main-drop-dma-coherent-in-crypt.patch +arm64-dts-ti-k3-j721e-main-drop-dma-coherent-in-cryp.patch +arm64-dts-mt2712e-fix-unit_address_vs_reg-warning-fo.patch +arm64-dts-mt2712e-fix-unit-address-for-pinctrl-node.patch +arm64-dts-mt2712-evb-fix-vproc-fixed-regulators-unit.patch +arm64-dts-mt2712-evb-fix-usb-vbus-regulators-unit-na.patch +arm64-dts-mediatek-pumpkin-common-fix-devicetree-war.patch +arm64-dts-mediatek-mt6797-fix-26m-oscillator-unit-na.patch +arm-dts-dove-fix-assigned-addresses-for-every-pcie-r.patch +arm-dts-armada-370-fix-assigned-addresses-for-every-.patch +arm-dts-armada-xp-fix-assigned-addresses-for-every-p.patch +arm-dts-armada-375-fix-assigned-addresses-for-every-.patch +arm-dts-armada-38x-fix-assigned-addresses-for-every-.patch +arm-dts-armada-39x-fix-assigned-addresses-for-every-.patch +arm-dts-turris-omnia-add-ethernet-aliases.patch +arm-dts-turris-omnia-add-switch-port-6-node.patch +arm64-dts-armada-3720-turris-mox-add-missing-interru.patch +pstore-ram-fix-error-return-code-in-ramoops_probe.patch +arm-mmp-fix-timer_read-delay.patch +pstore-avoid-kcore-oops-by-vmap-ing-with-vm_ioremap.patch +tpm-tpm_ftpm_tee-fix-error-handling-in-ftpm_mod_init.patch +tpm-tpm_crb-fix-error-message-in-__crb_relinquish_lo.patch +sched-fair-cleanup-task_util-and-capacity-type.patch +sched-uclamp-fix-relationship-between-uclamp-and-mig.patch +cpuidle-dt-return-the-correct-numbers-of-parsed-idle.patch +alpha-fix-syscall-entry-in-audut_syscall-case.patch +pm-hibernate-fix-mistake-in-kerneldoc-comment.patch +fs-don-t-audit-the-capability-check-in-simple_xattr_.patch +cpufreq-qcom-hw-fix-memory-leak-in-qcom_cpufreq_hw_r.patch +selftests-ftrace-event_triggers-wait-longer-for-test.patch +perf-fix-possible-memleak-in-pmu_dev_alloc.patch +lib-debugobjects-fix-stat-count-and-optimize-debug_o.patch +platform-x86-huawei-wmi-fix-return-value-calculation.patch +timerqueue-use-rb_entry_safe-in-timerqueue_getnext.patch +proc-fixup-uptime-selftest.patch +lib-fonts-fix-undefined-behavior-in-bit-shift-for-ge.patch +ocfs2-fix-memory-leak-in-ocfs2_stack_glue_init.patch +mips-vpe-mt-fix-possible-memory-leak-while-module-ex.patch +mips-vpe-cmp-fix-possible-memory-leak-while-module-e.patch +selftests-efivarfs-add-checking-of-the-test-return-v.patch +pnp-fix-name-memory-leak-in-pnp_alloc_dev.patch +perf-x86-intel-uncore-fix-reference-count-leak-in-hs.patch +perf-x86-intel-uncore-fix-reference-count-leak-in-sn.patch +perf-x86-intel-uncore-fix-reference-count-leak-in-__.patch +platform-chrome-cros_usbpd_notify-fix-error-handling.patch +irqchip-gic-pm-use-pm_runtime_resume_and_get-in-gic_.patch +edac-i10nm-fix-refcount-leak-in-pci_get_dev_wrapper.patch +nfsd-don-t-call-nfsd_file_put-from-client-states-seq.patch +genirq-irqdesc-don-t-try-to-remove-non-existing-sysf.patch +cpufreq-amd_freq_sensitivity-add-missing-pci_dev_put.patch +libfs-add-define_simple_attribute_signed-for-signed-.patch +lib-notifier-error-inject-fix-error-when-writing-err.patch +docs-fault-injection-fix-non-working-usage-of-negati.patch +debugfs-fix-error-when-writing-negative-value-to-ato.patch +ocfs2-ocfs2_mount_volume-does-cleanup-job-before-ret.patch +ocfs2-rewrite-error-handling-of-ocfs2_fill_super.patch +ocfs2-fix-memory-leak-in-ocfs2_mount_volume.patch +rapidio-fix-possible-name-leaks-when-rio_add_device-.patch +rapidio-rio-fix-possible-name-leak-in-rio_register_m.patch +clocksource-drivers-sh_cmt-make-sure-channel-clock-s.patch +clocksource-drivers-sh_cmt-access-registers-accordin.patch +futex-move-to-kernel-futex.patch +futex-resend-potentially-swallowed-owner-death-notif.patch +cpu-hotplug-make-target_store-a-nop-when-target-stat.patch +clocksource-drivers-timer-ti-dm-fix-missing-clk_disa.patch +acpica-fix-use-after-free-in-acpi_ut_copy_ipackage_t.patch +uprobes-x86-allow-to-probe-a-nop-instruction-with-0x.patch +x86-xen-fix-memory-leak-in-xen_smp_intr_init-_pv.patch +x86-xen-fix-memory-leak-in-xen_init_lock_cpu.patch +xen-privcmd-fix-a-possible-warning-in-privcmd_ioctl_.patch +pm-runtime-improve-path-in-rpm_idle-when-no-callback.patch +pm-runtime-do-not-call-__rpm_callback-from-rpm_idle.patch +platform-x86-mxm-wmi-fix-memleak-in-mxm_wmi_call_mx-.patch +platform-x86-intel_scu_ipc-fix-possible-name-leak-in.patch +mips-bcm63xx-add-check-for-null-for-clk-in-clk_enabl.patch +mips-octeon-warn-only-once-if-deprecated-link-status.patch +fs-sysv-fix-sysv_nblocks-returns-wrong-value.patch +rapidio-fix-possible-uaf-when-kfifo_alloc-fails.patch +eventfd-change-int-to-__u64-in-eventfd_signal-ifndef.patch +relay-fix-type-mismatch-when-allocating-memory-in-re.patch +hfs-fix-oob-write-in-hfs_asc2mac.patch +rapidio-devices-fix-missing-put_device-in-mport_cdev.patch +wifi-ath9k-hif_usb-fix-memory-leak-of-urbs-in-ath9k_.patch +wifi-ath9k-hif_usb-fix-use-after-free-in-ath9k_hif_u.patch +wifi-rtl8xxxu-fix-reading-the-vendor-of-combo-chips.patch +drm-bridge-adv7533-remove-dynamic-lane-switching-fro.patch +libbpf-fix-use-after-free-in-btf_dump_name_dups.patch +libbpf-fix-null-pointer-dereference-in-find_prog_by_.patch +pata_ipx4xx_cf-fix-unsigned-comparison-with-less-tha.patch +media-coda-jpeg-add-check-for-kmalloc.patch +media-i2c-ad5820-fix-error-path.patch +venus-pm_helpers-fix-error-check-in-vcodec_domains_g.patch +media-exynos4-is-use-v4l2_async_notifier_add_fwnode_.patch +media-exynos4-is-don-t-rely-on-the-v4l2_async_subdev.patch +can-kvaser_usb-do-not-increase-tx-statistics-when-se.patch +can-kvaser_usb-kvaser_usb_leaf-get-capabilities-from.patch +can-kvaser_usb-kvaser_usb_leaf-rename-leaf-usbcan-_c.patch +can-kvaser_usb-kvaser_usb_leaf-handle-cmd_error_even.patch +can-kvaser_usb_leaf-set-warning-state-even-without-b.patch +can-kvaser_usb_leaf-fix-improved-state-not-being-rep.patch +can-kvaser_usb_leaf-fix-wrong-can-state-after-stoppi.patch +can-kvaser_usb_leaf-fix-bogus-restart-events.patch +can-kvaser_usb-add-struct-kvaser_usb_busparams.patch +can-kvaser_usb-compare-requested-bittiming-parameter.patch +drm-rockchip-lvds-fix-pm-usage-counter-unbalance-in-.patch +clk-renesas-r9a06g032-repair-grave-increment-error.patch +spi-update-reference-to-struct-spi_controller.patch +drm-panel-panel-sitronix-st7701-remove-panel-on-dsi-.patch +ima-fix-fall-through-warnings-for-clang.patch +ima-handle-estale-returned-by-ima_filter_rule_match.patch +drm-msm-hdmi-switch-to-drm_bridge_connector.patch +drm-msm-hdmi-drop-unused-gpio-support.patch +bpf-fix-slot-type-check-in-check_stack_write_var_off.patch +media-vivid-fix-compose-size-exceed-boundary.patch +media-platform-exynos4-is-fix-return-value-check-in-.patch +bpf-propagate-precision-in-alu-alu64-operations.patch +bpf-check-the-other-end-of-slot_type-for-stack_spill.patch +bpf-propagate-precision-across-all-frames-not-just-t.patch +clk-qcom-gcc-sm8250-use-retention-mode-for-usb-gdscs.patch +mtd-fix-device-name-leak-when-register-device-failed.patch +input-joystick-fix-kconfig-warning-for-joystick_adc.patch +wifi-rsi-fix-handling-of-802.3-eapol-frames-sent-via.patch +media-camss-clean-up-received-buffers-on-failed-star.patch +net-proc-provide-proc_fs-n-fallback-for-proc_create_.patch +rxrpc-fix-ack.buffersize-to-be-0-when-generating-an-.patch +drm-radeon-add-the-missed-acpi_put_table-to-fix-memo.patch +drm-mediatek-modify-dpi-power-on-off-sequence.patch +asoc-pxa-fix-null-pointer-dereference-in-filter.patch +regulator-core-fix-unbalanced-of-node-refcount-in-re.patch +amdgpu-pm-prevent-array-underflow-in-vega20_odn_edit.patch +drm-fourcc-add-packed-10bit-yuv-4-2-0-format.patch +drm-fourcc-fix-vsub-hsub-for-q410-and-q401.patch +integrity-fix-memory-leakage-in-keyring-allocation-e.patch +ima-fix-misuse-of-dereference-of-pointer-in-template.patch +wifi-ath10k-fix-return-value-in-ath10k_pci_init.patch +mtd-lpddr2_nvm-fix-possible-null-ptr-deref.patch +input-elants_i2c-properly-handle-the-reset-gpio-when.patch +media-vidtv-fix-use-after-free-in-vidtv_bridge_dvb_i.patch +media-solo6x10-fix-possible-memory-leak-in-solo_sysf.patch +media-platform-exynos4-is-fix-error-handling-in-fimc.patch +media-videobuf-dma-contig-use-dma_mmap_coherent.patch +inet-add-read_once-sk-sk_bound_dev_if-in-inet_csk_bi.patch +bpf-move-skb-len-0-checks-into-__bpf_redirect.patch +hid-hid-sensor-custom-set-fixed-size-for-custom-attr.patch +alsa-pcm-fix-undefined-behavior-in-bit-shift-for-snd.patch +alsa-seq-fix-undefined-behavior-in-bit-shift-for-snd.patch +regulator-core-use-kfree_const-to-free-space-conditi.patch +clk-rockchip-fix-memory-leak-in-rockchip_clk_registe.patch +drm-amdgpu-fix-pci-device-refcount-leak.patch +bonding-fix-link-recovery-in-mode-2-when-updelay-is-.patch +mtd-maps-pxa2xx-flash-fix-memory-leak-in-probe.patch +drbd-fix-an-invalid-memory-access-caused-by-incorrec.patch +asoc-qcom-add-checks-for-devm_kcalloc.patch +media-vimc-fix-wrong-function-called-when-vimc_init-.patch +media-imon-fix-a-race-condition-in-send_packet.patch +clk-imx-replace-osc_hdmi-with-dummy.patch +pinctrl-pinconf-generic-add-missing-of_node_put.patch +media-dvb-core-fix-ignored-return-value-in-dvb_regis.patch +media-dvb-usb-az6027-fix-null-ptr-deref-in-az6027_i2.patch +media-s5p-mfc-add-variant-data-for-mfc-v7-hardware-f.patch +drm-tegra-add-missing-clk_disable_unprepare-in-tegra.patch +asoc-dt-bindings-wcd9335-fix-reset-line-polarity-in-.patch +asoc-mediatek-mtk-btcvsd-add-checks-for-write-and-re.patch +nfsv4.2-clear-fattr4_word2_security_label-when-done-.patch +nfsv4.2-fix-a-memory-stomp-in-decode_attr_security_l.patch +nfsv4.2-fix-initialisation-of-struct-nfs4_label.patch +nfsv4-fix-a-deadlock-between-nfs4_open_recover_helpe.patch +nfs-fix-an-oops-in-nfs_d_automount.patch +alsa-asihpi-fix-missing-pci_disable_device.patch +wifi-iwlwifi-mvm-fix-double-free-on-tx-path.patch +asoc-mediatek-mt8173-fix-debugfs-registration-for-co.patch +asoc-mediatek-mt8173-enable-irq-when-pdata-is-ready.patch +drm-amd-pm-smu11-baco-is-supported-when-it-s-in-baco.patch +drm-radeon-fix-pci-device-refcount-leak-in-radeon_at.patch +drm-amdgpu-fix-pci-device-refcount-leak-in-amdgpu_at.patch +asoc-pcm512x-fix-pm-disable-depth-imbalance-in-pcm51.patch +netfilter-conntrack-set-icmpv6-redirects-as-related.patch +bpf-sockmap-fix-repeated-calls-to-sock_put-when-msg-.patch +bpf-sockmap-fix-data-loss-caused-by-using-apply_byte.patch +bonding-uninitialized-variable-in-bond_miimon_inspec.patch +spi-spidev-mask-spi_cs_high-in-spi_ioc_rd_mode.patch +wifi-mac80211-fix-memory-leak-in-ieee80211_if_add.patch +wifi-cfg80211-fix-not-unregister-reg_pdev-when-load_.patch +wifi-mt76-fix-coverity-overrun-call-in-mt76_get_txpo.patch +regulator-core-fix-module-refcount-leak-in-set_suppl.patch +clk-qcom-clk-krait-fix-wrong-div2-functions.patch +hsr-add-a-rcu-read-lock-to-hsr_forward_skb.patch +net-hsr-generate-supervision-frame-without-hsr-prp-t.patch +hsr-disable-netpoll.patch +hsr-synchronize-sending-frames-to-have-always-increm.patch +hsr-synchronize-sequence-number-updates.patch +configfs-fix-possible-memory-leak-in-configfs_create.patch +regulator-core-fix-resource-leak-in-regulator_regist.patch +hwmon-jc42-convert-register-access-and-caching-to-re.patch +hwmon-jc42-restore-the-min-max-critical-temperatures.patch +bpf-sockmap-fix-race-in-sock_map_free.patch +alsa-pcm-set-missing-stop_operating-flag-at-undoing-.patch +media-saa7164-fix-missing-pci_disable_device.patch +alsa-mts64-fix-possible-null-ptr-defer-in-snd_mts64_.patch +xprtrdma-fix-regbuf-data-not-freed-in-rpcrdma_req_cr.patch +sunrpc-fix-missing-release-socket-in-rpc_sockname.patch +nfsv4.x-fail-client-initialisation-if-state-manager-.patch +mmc-alcor-fix-return-value-check-of-mmc_add_host.patch +mmc-moxart-fix-return-value-check-of-mmc_add_host.patch +mmc-mxcmmc-fix-return-value-check-of-mmc_add_host.patch +mmc-pxamci-fix-return-value-check-of-mmc_add_host.patch +mmc-rtsx_usb_sdmmc-fix-return-value-check-of-mmc_add.patch +mmc-toshsd-fix-return-value-check-of-mmc_add_host.patch +mmc-vub300-fix-return-value-check-of-mmc_add_host.patch +mmc-wmt-sdmmc-fix-return-value-check-of-mmc_add_host.patch +mmc-atmel-mci-fix-return-value-check-of-mmc_add_host.patch +mmc-omap_hsmmc-fix-return-value-check-of-mmc_add_hos.patch +mmc-meson-gx-fix-return-value-check-of-mmc_add_host.patch +mmc-via-sdmmc-fix-return-value-check-of-mmc_add_host.patch +mmc-wbsd-fix-return-value-check-of-mmc_add_host.patch +mmc-mmci-fix-return-value-check-of-mmc_add_host.patch +media-c8sectpfe-add-of_node_put-when-breaking-out-of.patch +media-coda-add-check-for-dcoda_iram_alloc.patch +media-coda-add-check-for-kmalloc.patch +clk-samsung-fix-memory-leak-in-_samsung_clk_register.patch +spi-spi-gpio-don-t-set-mosi-as-an-input-if-not-3wire.patch +wifi-rtl8xxxu-add-__packed-to-struct-rtl8723bu_c2h.patch +wifi-rtl8xxxu-fix-the-channel-width-reporting.patch +wifi-brcmfmac-fix-error-return-code-in-brcmf_sdio_do.patch +blktrace-fix-output-non-blktrace-event-when-blk_clas.patch +clk-socfpga-clk-pll-remove-unused-variable-rc.patch +clk-socfpga-use-clk_hw_register-for-a5-c5.patch +clk-socfpga-fix-memory-leak-in-socfpga_gate_init.patch +net-vmw_vsock-vmci-check-memcpy_from_msg.patch +net-defxx-fix-missing-err-handling-in-dfx_init.patch +net-stmmac-selftests-fix-potential-memleak-in-stmmac.patch +drivers-net-qlcnic-fix-potential-memory-leak-in-qlcn.patch +of-overlay-fix-null-pointer-dereferencing-in-find_du.patch +ethernet-s2io-don-t-call-dev_kfree_skb-under-spin_lo.patch +net-farsync-fix-kmemleak-when-rmmods-farsync.patch +net-tunnel-wait-until-all-sk_user_data-reader-finish.patch +net-apple-mace-don-t-call-dev_kfree_skb-under-spin_l.patch +net-apple-bmac-don-t-call-dev_kfree_skb-under-spin_l.patch +net-emaclite-don-t-call-dev_kfree_skb-under-spin_loc.patch +net-ethernet-dnet-don-t-call-dev_kfree_skb-under-spi.patch +hamradio-don-t-call-dev_kfree_skb-under-spin_lock_ir.patch +net-amd-lance-don-t-call-dev_kfree_skb-under-spin_lo.patch +net-amd-xgbe-fix-logic-around-active-and-passive-cab.patch +net-amd-xgbe-check-only-the-minimum-speed-for-active.patch +can-tcan4x5x-remove-invalid-write-in-clear_interrupt.patch +net-lan9303-fix-read-error-execution-path.patch +ntb_netdev-use-dev_kfree_skb_any-in-interrupt-contex.patch +sctp-sysctl-make-extra-pointers-netns-aware.patch +bluetooth-btusb-don-t-call-kfree_skb-under-spin_lock.patch +bluetooth-hci_qca-don-t-call-kfree_skb-under-spin_lo.patch +bluetooth-hci_ll-don-t-call-kfree_skb-under-spin_loc.patch +bluetooth-hci_h5-don-t-call-kfree_skb-under-spin_loc.patch +bluetooth-hci_bcsp-don-t-call-kfree_skb-under-spin_l.patch +bluetooth-hci_core-don-t-call-kfree_skb-under-spin_l.patch +bluetooth-rfcomm-don-t-call-kfree_skb-under-spin_loc.patch +stmmac-fix-potential-division-by-0.patch +apparmor-fix-a-memleak-in-multi_transaction_new.patch +apparmor-fix-lockdep-warning-when-removing-a-namespa.patch +apparmor-fix-abi-check-to-include-v8-abi.patch +crypto-sun8i-ss-use-dma_addr-instead-u32.patch +crypto-nitrox-avoid-double-free-on-error-path-in-nit.patch +scsi-core-fix-a-race-between-scsi_done-and-scsi_time.patch +apparmor-use-pointer-to-struct-aa_label-for-lbs_cred.patch +pci-dwc-fix-n_fts-array-overrun.patch +rdma-core-fix-order-of-nldev_exit-call.patch +pci-pci-epf-test-register-notifier-if-only-core_init.patch +f2fs-fix-the-race-condition-of-resize-flag-between-r.patch +crypto-rockchip-do-not-do-custom-power-management.patch +crypto-rockchip-do-not-store-mode-globally.patch +crypto-rockchip-add-fallback-for-cipher.patch +crypto-rockchip-add-fallback-for-ahash.patch +crypto-rockchip-better-handle-cipher-key.patch +crypto-rockchip-remove-non-aligned-handling.patch +crypto-rockchip-delete-unneeded-variable-initializat.patch +crypto-rockchip-rework-by-using-crypto_engine.patch +apparmor-fix-memleak-in-alloc_ns.patch +f2fs-fix-normal-discard-process.patch +rdma-siw-fix-immediate-work-request-flush-to-complet.patch +rdma-nldev-return-eagain-if-the-cm_id-isn-t-from-exp.patch +rdma-siw-set-defined-status-for-work-completion-with.patch +scsi-scsi_debug-fix-a-warning-in-resp_write_scat.patch +crypto-ccree-remove-debugfs-when-platform_driver_reg.patch +crypto-cryptd-use-request-context-instead-of-stack-f.patch +crypto-hisilicon-qm-add-missing-pci_dev_put-in-q_num.patch +rdma-hns-repacing-dseg_len-by-macros-in-fill_ext_sge.patch +rdma-hns-fix-ext_sge-num-error-when-post-send.patch +pci-check-for-alloc-failure-in-pci_request_irq.patch +rdma-hfi-decrease-pci-device-reference-count-in-erro.patch +crypto-ccree-make-cc_debugfs_global_fini-available-f.patch +rdma-hns-fix-memory-leak-in-hns_roce_alloc_mr.patch +rdma-rxe-fix-null-ptr-deref-in-rxe_qp_do_cleanup-whe.patch +scsi-hpsa-fix-possible-memory-leak-in-hpsa_init_one.patch +crypto-tcrypt-fix-multibuffer-skcipher-speed-test-me.patch +padata-always-leave-bhs-disabled-when-running-parall.patch +padata-fix-list-iterator-in-padata_do_serial.patch +scsi-mpt3sas-fix-possible-resource-leaks-in-mpt3sas_.patch +scsi-hpsa-fix-error-handling-in-hpsa_add_sas_host.patch +scsi-hpsa-fix-possible-memory-leak-in-hpsa_add_sas_d.patch +scsi-scsi_debug-fix-a-warning-in-resp_verify.patch +scsi-scsi_debug-fix-a-warning-in-resp_report_zones.patch +scsi-fcoe-fix-possible-name-leak-when-device_registe.patch +scsi-scsi_debug-fix-possible-name-leak-in-sdebug_add.patch +scsi-ipr-fix-warning-in-ipr_init.patch +scsi-fcoe-fix-transport-not-deattached-when-fcoe_if_.patch +scsi-snic-fix-possible-uaf-in-snic_tgt_create.patch +rdma-nldev-add-checks-for-nla_nest_start-in-fill_sta.patch +f2fs-avoid-victim-selection-from-previous-victim-sec.patch +rdma-nldev-fix-failure-to-send-large-messages.patch +crypto-amlogic-remove-kcalloc-without-check.patch +crypto-omap-sham-use-pm_runtime_resume_and_get-in-om.patch +riscv-mm-add-arch-hook-arch_clear_hugepage_flags.patch +rdma-hfi1-fix-error-return-code-in-parse_platform_co.patch +rdma-srp-fix-error-return-code-in-srp_parse_options.patch +orangefs-fix-sysfs-not-cleanup-when-dev-init-failed.patch +rdma-hns-fix-pbl-page-mtr-find.patch +rdma-hns-fix-page-size-cap-from-firmware.patch +crypto-img-hash-fix-variable-dereferenced-before-che.patch +hwrng-amd-fix-pci-device-refcount-leak.patch +hwrng-geode-fix-pci-device-refcount-leak.patch +ib-ipoib-fix-queue-count-inconsistency-for-pkey-chil.patch +drivers-dio-fix-possible-memory-leak-in-dio_init.patch +serial-tegra-read-dma-status-before-terminating.patch +class-fix-possible-memory-leak-in-__class_register.patch +vfio-platform-do-not-pass-return-buffer-to-acpi-_rst.patch +uio-uio_dmem_genirq-fix-missing-unlock-in-irq-config.patch +uio-uio_dmem_genirq-fix-deadlock-between-irq-config-.patch +usb-fotg210-udc-fix-ages-old-endianness-issues.patch +staging-vme_user-fix-possible-uaf-in-tsi148_dma_list.patch +usb-typec-check-for-ops-exit-instead-of-ops-enter-in.patch +usb-typec-tcpci-fix-of-node-refcount-leak-in-tcpci_r.patch +usb-typec-tipd-fix-spurious-fwnode_handle_put-in-err.patch +serial-amba-pl011-avoid-sbsa-uart-accessing-dmacr-re.patch +serial-pl011-do-not-clear-rx-fifo-rx-interrupt-in-un.patch +serial-pch-fix-pci-device-refcount-leak-in-pch_reque.patch +tty-serial-clean-up-stop-tx-part-in-altera_uart_tx_c.patch +tty-serial-altera_uart_-r-t-x_chars-need-only-uart_p.patch +serial-altera_uart-fix-locking-in-polling-mode.patch +serial-sunsab-fix-error-handling-in-sunsab_init.patch +test_firmware-fix-memory-leak-in-test_firmware_init.patch +misc-ocxl-fix-possible-name-leak-in-ocxl_file_regist.patch +ocxl-fix-pci-device-refcount-leak-when-calling-get_f.patch +misc-tifm-fix-possible-memory-leak-in-tifm_7xx1_swit.patch +misc-sgi-gru-fix-use-after-free-error-in-gru_set_con.patch +firmware-raspberrypi-fix-possible-memory-leak-in-rpi.patch +cxl-fix-possible-null-ptr-deref-in-cxl_guest_init_af.patch +cxl-fix-possible-null-ptr-deref-in-cxl_pci_init_afu-.patch +iio-temperature-ltc2983-make-bulk-write-buffer-dma-s.patch +genirq-add-irqf_no_autoen-for-request_irq-nmi.patch +iio-imu-adis-use-irqf_no_autoen-instead-of-irq-reque.patch +iio-adis-handle-devices-that-cannot-unmask-the-drdy-.patch +iio-adis-stylistic-changes.patch +iio-imu-adis-move-exports-into-iio_adislib-namespace.patch +iio-adis-add-__adis_enable_irq-implementation.patch +counter-stm32-lptimer-cnt-fix-the-check-on-arr-and-c.patch +usb-roles-fix-of-node-refcount-leak-in-usb_role_swit.patch +usb-gadget-f_hid-optional-setup-set_report-mode.patch +usb-gadget-f_hid-fix-f_hidg-lifetime-vs-cdev.patch +usb-gadget-f_hid-fix-refcount-leak-on-error-path.patch +drivers-mcb-fix-resource-leak-in-mcb_probe.patch +mcb-mcb-parse-fix-error-handing-in-chameleon_parse_g.patch +chardev-fix-error-handling-in-cdev_device_add.patch +i2c-pxa-pci-fix-missing-pci_disable_device-on-error-.patch +staging-rtl8192u-fix-use-after-free-in-ieee80211_rx.patch +staging-rtl8192e-fix-potential-use-after-free-in-rtl.patch +vme-fix-error-not-catched-in-fake_init.patch +gpiolib-get-rid-of-redundant-else.patch +gpiolib-cdev-fix-null-pointer-dereferences.patch +i2c-mux-reg-check-return-value-after-calling-platfor.patch +i2c-ismt-fix-an-out-of-bounds-bug-in-ismt_access.patch +usb-storage-add-check-for-kcalloc.patch +tracing-hist-fix-issue-of-losting-command-info-in-er.patch +samples-vfio-mdev-fix-missing-pci_disable_device-in-.patch +thermal-drivers-imx8mm_thermal-validate-temperature-.patch +fbdev-ssd1307fb-drop-optional-dependency.patch +fbdev-pm2fb-fix-missing-pci_disable_device.patch +fbdev-via-fix-error-in-via_core_init.patch +fbdev-vermilion-decrease-reference-count-in-error-pa.patch +fbdev-uvesafb-fixes-an-error-handling-path-in-uvesaf.patch +hsi-omap_ssi_core-fix-unbalanced-pm_runtime_disable.patch +hsi-omap_ssi_core-fix-possible-memory-leak-in-ssi_pr.patch +power-supply-fix-residue-sysfs-file-in-error-handle-.patch +perf-trace-return-error-if-a-system-call-doesn-t-exi.patch +perf-trace-use-macro-raw_syscall_args_num-to-replace.patch +perf-trace-handle-failure-when-trace-point-folder-is.patch +perf-symbol-correction-while-adjusting-symbol.patch +hsi-omap_ssi_core-fix-error-handling-in-ssi_init.patch +power-supply-fix-null-pointer-dereferencing-in-power.patch +rdma-siw-fix-pointer-cast-warning.patch +iommu-sun50i-fix-reset-release.patch +iommu-sun50i-consider-all-fault-sources-for-reset.patch +iommu-sun50i-fix-r-w-permission-check.patch +iommu-sun50i-fix-flush-size.patch +phy-usb-s2-wol-wakeup_count-not-incremented-for-usb-.patch +include-uapi-linux-swab-fix-potentially-missing-__al.patch +pwm-tegra-improve-required-rate-calculation.patch +dmaengine-idxd-fix-crc_val-field-for-completion-reco.patch +rtc-rtc-cmos-do-not-check-acpi_fadt_low_power_s0.patch +rtc-cmos-fix-event-handler-registration-ordering-iss.patch +rtc-cmos-fix-wake-alarm-breakage.patch +rtc-cmos-fix-build-on-non-acpi-platforms.patch +rtc-cmos-call-cmos_wake_setup-from-cmos_do_probe.patch +rtc-cmos-call-rtc_wake_setup-from-cmos_do_probe.patch +rtc-cmos-eliminate-forward-declarations-of-some-func.patch +rtc-cmos-rename-acpi-related-functions.patch +rtc-cmos-disable-acpi-rtc-event-on-removal.patch +rtc-snvs-allow-a-time-difference-on-clock-register-r.patch +rtc-pcf85063-fix-reading-alarm.patch +iommu-amd-fix-pci-device-refcount-leak-in-ppr_notifi.patch +iommu-fsl_pamu-fix-resource-leak-in-fsl_pamu_probe.patch +macintosh-fix-possible-memory-leak-in-macio_add_one_.patch +macintosh-macio-adb-check-the-return-value-of-iorema.patch +powerpc-52xx-fix-a-resource-leak-in-an-error-handlin.patch +cxl-fix-refcount-leak-in-cxl_calc_capp_routing.patch +powerpc-xmon-enable-breakpoints-on-8xx.patch +powerpc-xmon-fix-wswitch-unreachable-warning-in-bpt_.patch +powerpc-xive-add-missing-iounmap-in-error-path-in-xi.patch +kbuild-remove-unneeded-mkdir-for-external-modules_in.patch +kbuild-unify-modules-_install-for-in-tree-and-extern.patch +kbuild-refactor-single-builds-of-.ko.patch +powerpc-perf-callchain-validate-kernel-stack-pointer.patch +powerpc-83xx-mpc832x_rdb-call-platform_device_put-in.patch +powerpc-hv-gpci-fix-hv_gpci-event-list.patch +selftests-powerpc-fix-resource-leaks.patch +iommu-sun50i-remove-iommu_domain_identity.patch +pwm-sifive-call-pwm_sifive_update_clock-while-mutex-.patch +remoteproc-sysmon-fix-memory-leak-in-qcom_add_sysmon.patch +remoteproc-qcom_q6v5_pas-disable-wakeup-on-probe-fai.patch +remoteproc-qcom_q6v5_pas-detach-power-domains-on-rem.patch +remoteproc-qcom_q6v5_pas-fix-missing-of_node_put-in-.patch +powerpc-eeh-drop-redundant-spinlock-initialization.patch +powerpc-pseries-eeh-use-correct-api-for-error-log-si.patch +netfilter-flowtable-really-fix-nat-ipv6-offload.patch +rtc-st-lpc-add-missing-clk_disable_unprepare-in-st_r.patch +rtc-pic32-move-devm_rtc_allocate_device-earlier-in-p.patch +rtc-pcf85063-fix-pcf85063_clkout_control.patch +nfsd-remove-spurious-cb_setup_err-tracepoint.patch +nfsd-under-nfsv4.1-fix-double-svc_xprt_put-on-rpc_cr.patch +net-macsec-fix-net-device-access-prior-to-holding-a-.patch +misdn-hfcsusb-don-t-call-dev_kfree_skb-kfree_skb-und.patch +misdn-hfcpci-don-t-call-dev_kfree_skb-kfree_skb-unde.patch +misdn-hfcmulti-don-t-call-dev_kfree_skb-kfree_skb-un.patch +nfc-pn533-clear-nfc_target-before-being-used.patch +r6040-fix-kmemleak-in-probe-and-remove.patch +net-switch-to-storing-kcov-handle-directly-in-sk_buf.patch +net-add-inline-function-skb_csum_is_sctp.patch +net-igc-use-skb_csum_is_sctp-instead-of-protocol-che.patch +net-add-a-helper-to-avoid-issues-with-hw-tx-timestam.patch +igc-enhance-qbv-scheduling-by-using-first-flag-bit.patch +igc-use-strict-cycles-for-qbv-scheduling.patch +igc-add-checking-for-basetime-less-than-zero.patch +igc-recalculate-qbv-end_time-by-considering-cycle-ti.patch +igc-lift-taprio-schedule-restriction.patch +igc-set-qbv-start_time-and-end_time-to-end_time-if-n.patch +rtc-mxc_v2-add-missing-clk_disable_unprepare.patch +selftests-devlink-fix-the-fd-redirect-in-dummy_repor.patch +openvswitch-fix-flow-lookup-to-use-unmasked-key.patch +skbuff-account-for-tail-adjustment-during-pull-opera.patch +mailbox-zynq-ipi-fix-error-handling-while-device_reg.patch +net_sched-reject-tcf_em_simple-case-for-complex-emat.patch +rxrpc-fix-missing-unlock-in-rxrpc_do_sendmsg.patch +myri10ge-fix-an-error-handling-path-in-myri10ge_prob.patch +net-stream-purge-sk_error_queue-in-sk_stream_kill_qu.patch +rcu-fix-__this_cpu_read-lockdep-warning-in-rcu_force.patch +arm64-make-is_ttbrx_addr-noinstr-safe.patch +video-hyperv_fb-avoid-taking-busy-spinlock-on-panic-.patch +x86-hyperv-remove-unregister-syscore-call-from-hyper.patch +binfmt_misc-fix-shift-out-of-bounds-in-check_special.patch +fs-jfs-fix-shift-out-of-bounds-in-dballocag.patch +udf-avoid-double-brelse-in-udf_rename.patch +fs-jfs-fix-shift-out-of-bounds-in-dbdiscardag.patch +acpica-fix-error-code-path-in-acpi_ds_call_control_m.patch +nilfs2-fix-shift-out-of-bounds-overflow-in-nilfs_sb2.patch +nilfs2-fix-shift-out-of-bounds-due-to-too-large-expo.patch +acct-fix-potential-integer-overflow-in-encode_comp_t.patch +hfs-fix-oob-read-in-__hfs_brec_find.patch +drm-etnaviv-add-missing-quirks-for-gc300.patch +brcmfmac-return-error-when-getting-invalid-max_flowr.patch +wifi-ath9k-verify-the-expected-usb_endpoints-are-pre.patch +wifi-ar5523-fix-use-after-free-on-ar5523_cmd-timed-o.patch +asoc-codecs-rt298-add-quirk-for-kbl-r-rvp-platform.patch +ipmi-fix-memleak-when-unload-ipmi-driver.patch +drm-amd-display-prevent-memory-leak.patch +qed-gcc13-use-u16-for-fid-to-be-big-enough.patch +bpf-make-sure-skb-len-0-when-redirecting-to-a-tunnel.patch +net-ethernet-ti-fix-return-type-of-netcp_ndo_start_x.patch +hamradio-baycom_epp-fix-return-type-of-baycom_send_p.patch +wifi-brcmfmac-fix-potential-shift-out-of-bounds-in-b.patch +igb-do-not-free-q_vector-unless-new-one-was-allocate.patch +drm-amdgpu-fix-type-of-second-parameter-in-trans_msg.patch +drm-amdgpu-fix-type-of-second-parameter-in-odn_edit_.patch +s390-ctcm-fix-return-type-of-ctc-mp-m_tx.patch +s390-netiucv-fix-return-type-of-netiucv_tx.patch +s390-lcs-fix-return-type-of-lcs_start_xmit.patch +drm-msm-use-drm_mode_copy.patch +drm-rockchip-use-drm_mode_copy.patch +drm-sti-use-drm_mode_copy.patch +drivers-md-md-bitmap-check-the-return-value-of-md_bi.patch +md-raid1-stop-mdx_raid1-thread-when-raid1-array-run-.patch +drm-amd-display-fix-array-index-out-of-bound-error-i.patch +net-add-atomic_long_t-to-net_device_stats-fields.patch +mrp-introduce-active-flags-to-prevent-uaf-when-appli.patch +ppp-associate-skb-with-a-device-at-tx.patch +bpf-prevent-decl_tag-from-being-referenced-in-func_p.patch +ethtool-avoiding-integer-overflow-in-ethtool_phys_id.patch +media-dvb-frontends-fix-leak-of-memory-fw.patch +media-dvbdev-adopts-refcnt-to-avoid-uaf.patch +media-dvb-usb-fix-memory-leak-in-dvb_usb_adapter_ini.patch +blk-mq-fix-possible-memleak-when-register-hctx-faile.patch +libbpf-avoid-enum-forward-declarations-in-public-api.patch +regulator-core-fix-use_count-leakage-when-handling-b.patch +mmc-f-sdh30-add-quirks-for-broken-timeout-clock-capa.patch +mmc-renesas_sdhi-better-reset-from-hs400-mode.patch +media-si470x-fix-use-after-free-in-si470x_int_in_cal.patch +clk-st-fix-memory-leak-in-st_of_quadfs_setup.patch +hugetlbfs-fix-null-ptr-deref-in-hugetlbfs_parse_para.patch +drm-fsl-dcu-fix-return-type-of-fsl_dcu_drm_connector.patch +drm-sti-fix-return-type-of-sti_-dvo-hda-hdmi-_connec.patch +orangefs-fix-kmemleak-in-orangefs_prepare_debugfs_he.patch +orangefs-fix-kmemleak-in-orangefs_-kernel-client-_de.patch +hwmon-jc42-fix-missing-unlock-on-error-in-jc42_write.patch +alsa-asoc-hda-move-rename-snd_hdac_ext_stop_streams-.patch +alsa-hda-add-snd_hdac_stop_streams-helper.patch +asoc-intel-skylake-fix-driver-hang-during-shutdown.patch +asoc-mediatek-mt8173-rt5650-rt5514-fix-refcount-leak.patch +asoc-audio-graph-card-fix-refcount-leak-of-cpu_ep-in.patch +asoc-rockchip-pdm-add-missing-clk_disable_unprepare-.patch +asoc-wm8994-fix-potential-deadlock.patch +asoc-rockchip-spdif-add-missing-clk_disable_unprepar.patch +asoc-rt5670-remove-unbalanced-pm_runtime_put.patch +loadpin-ignore-the-contents-argument-of-the-lsm-hook.patch +pstore-switch-pmsg_lock-to-an-rt_mutex-to-avoid-prio.patch +perf-debug-set-debug_peo_args-and-redirect_to_stderr.patch +afs-fix-lost-servers_outstanding-count.patch +pstore-make-sure-config_pstore_pmsg-selects-config_r.patch +ima-simplify-ima_lsm_copy_rule.patch +alsa-usb-audio-add-the-quirk-for-kt0206-device.patch +alsa-hda-realtek-add-quirk-for-lenovo-tianyi510pro-14iob.patch +alsa-hda-hdmi-add-hp-device-0x8711-to-force-connect-list.patch +usb-dwc3-fix-race-between-dwc3_set_mode-and-__dwc3_set_mode.patch +usb-dwc3-core-defer-probe-on-ulpi_read_id-timeout.patch +hid-wacom-ensure-bootloader-pid-is-usable-in-hidraw-mode.patch +hid-mcp2221-don-t-connect-hidraw.patch +reiserfs-add-missing-calls-to-reiserfs_security_free.patch +iio-adc-ad_sigma_delta-do-not-use-internal-iio_dev-lock.patch +iio-adc128s052-add-proper-.data-members-in-adc128_of_match-table.patch +regulator-core-fix-deadlock-on-regulator-enable.patch +gcov-add-support-for-checksum-field.patch +ovl-fix-use-inode-directly-in-rcu-walk-mode.patch +media-dvbdev-fix-build-warning-due-to-comments.patch +media-dvbdev-fix-refcnt-bug.patch +pwm-tegra-fix-32-bit-build.patch +usb-dwc3-qcom-fix-memory-leak-in-dwc3_qcom_interconn.patch +cifs-fix-oops-during-encryption.patch +nvme-pci-fix-doorbell-buffer-value-endianness.patch +nvme-pci-fix-mempool-alloc-size.patch +nvme-pci-fix-page-size-checks.patch +ata-ahci-fix-pcs-quirk-application-for-suspend.patch +nvme-fix-the-nvme_cmd_effects_cse_mask-definition.patch +nvmet-don-t-defer-passthrough-commands-with-trivial-.patch +objtool-fix-segfault.patch +powerpc-rtas-avoid-device-tree-lookups-in-rtas_os_te.patch +powerpc-rtas-avoid-scheduling-in-rtas_os_term.patch +hid-multitouch-fix-asus-expertbook-p2-p2451fa-trackp.patch +hid-plantronics-additional-pids-for-double-volume-ke.patch +pstore-zone-use-gfp_atomic-to-allocate-zone-buffer.patch +hfsplus-fix-bug-causing-custom-uid-and-gid-being-unable-to-be-assigned-with-mount.patch +binfmt-fix-error-return-code-in-load_elf_fdpic_binary.patch +ovl-use-ovl-mounter-s-fsuid-and-fsgid-in-ovl_link.patch +alsa-line6-correct-midi-status-byte-when-receiving-data-from-podxt.patch +alsa-line6-fix-stack-overflow-in-line6_midi_transmit.patch +pnode-terminate-at-peers-of-source.patch +md-fix-a-crash-in-mempool_free.patch +mm-compaction-fix-fast_isolate_around-to-stay-within-boundaries.patch +f2fs-should-put-a-page-when-checking-the-summary-info.patch +mmc-vub300-fix-warning-do-not-call-blocking-ops-when-task_running.patch +tpm-acpi-call-acpi_put_table-to-fix-memory-leak.patch +tpm-tpm_crb-add-the-missed-acpi_put_table-to-fix-memory-leak.patch +tpm-tpm_tis-add-the-missed-acpi_put_table-to-fix-memory-leak.patch +sunrpc-don-t-leak-netobj-memory-when-gss_read_proxy_verf-fails.patch +kcsan-instrument-memcpy-memset-memmove-with-newer-clang.patch +kernel-provide-create_io_thread-helper.patch +iov_iter-add-helper-to-save-iov_iter-state.patch +saner-calling-conventions-for-unlazy_child.patch +fs-add-support-for-lookup_cached.patch +fix-handling-of-nd-depth-on-lookup_cached-failures-in-try_to_unlazy.patch +make-sure-nd-path.mnt-and-nd-path.dentry-are-always-valid-pointers.patch +fs-expose-lookup_cached-through-openat2-resolve_cached.patch +tools-headers-uapi-sync-openat2.h-with-the-kernel-sources.patch +net-provide-__sys_shutdown_sock-that-takes-a-socket.patch +net-add-accept-helper-not-installing-fd.patch +signal-add-task_sigpending-helper.patch +fs-make-do_renameat2-take-struct-filename.patch +file-rename-__close_fd_get_file-close_fd_get_file.patch +fs-provide-locked-helper-variant-of-close_fd_get_file.patch +entry-add-support-for-tif_notify_signal.patch +task_work-use-tif_notify_signal-if-available.patch +x86-wire-up-tif_notify_signal.patch +arc-add-support-for-tif_notify_signal.patch +arm64-add-support-for-tif_notify_signal.patch +m68k-add-support-for-tif_notify_signal.patch +nios32-add-support-for-tif_notify_signal.patch +parisc-add-support-for-tif_notify_signal.patch +powerpc-add-support-for-tif_notify_signal.patch +mips-add-support-for-tif_notify_signal.patch +s390-add-support-for-tif_notify_signal.patch +um-add-support-for-tif_notify_signal.patch +sh-add-support-for-tif_notify_signal.patch +openrisc-add-support-for-tif_notify_signal.patch +csky-add-support-for-tif_notify_signal.patch +hexagon-add-support-for-tif_notify_signal.patch +microblaze-add-support-for-tif_notify_signal.patch +arm-add-support-for-tif_notify_signal.patch +xtensa-add-support-for-tif_notify_signal.patch +alpha-add-support-for-tif_notify_signal.patch +c6x-add-support-for-tif_notify_signal.patch +h8300-add-support-for-tif_notify_signal.patch +ia64-add-support-for-tif_notify_signal.patch +nds32-add-support-for-tif_notify_signal.patch +riscv-add-support-for-tif_notify_signal.patch +sparc-add-support-for-tif_notify_signal.patch +ia64-don-t-call-handle_signal-unless-there-s-actually-a-signal-queued.patch +arc-unbork-5.11-bootup-fix-snafu-in-_tif_notify_signal-handling.patch +alpha-fix-tif_notify_signal-handling.patch +task_work-remove-legacy-twa_signal-path.patch +kernel-remove-checking-for-tif_notify_signal.patch +coredump-limit-what-can-interrupt-coredumps.patch +kernel-allow-fork-with-tif_notify_signal-pending.patch +entry-kvm-exit-to-user-mode-when-tif_notify_signal-is-set.patch +arch-setup-pf_io_worker-threads-like-pf_kthread.patch +arch-ensure-parisc-powerpc-handle-pf_io_worker-in-copy_thread.patch +x86-process-setup-io_threads-more-like-normal-user-space-threads.patch +kernel-stop-masking-signals-in-create_io_thread.patch +kernel-don-t-call-do_exit-for-pf_io_worker-threads.patch +task_work-add-helper-for-more-targeted-task_work-canceling.patch +io_uring-import-5.15-stable-io_uring.patch +signal-kill-jobctl_task_work.patch +task_work-unconditionally-run-task_work-from-get_signal.patch +net-remove-cmsg-restriction-from-io_uring-based-send-recvmsg-calls.patch +revert-proc-don-t-allow-async-path-resolution-of-proc-thread-self-components.patch +revert-proc-don-t-allow-async-path-resolution-of-proc-self-components.patch +eventpoll-add-epoll_uring_wake-poll-wakeup-flag.patch +eventfd-provide-a-eventfd_signal_mask-helper.patch +io_uring-pass-in-epoll_uring_wake-for-eventfd-signaling-and-wakeups.patch diff --git a/next-5.10/sh-add-support-for-tif_notify_signal.patch b/next-5.10/sh-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..95e06a5274d --- /dev/null +++ b/next-5.10/sh-add-support-for-tif_notify_signal.patch @@ -0,0 +1,57 @@ +From 1469cd034b065cbe06c2382a0d2aace50e3dc854 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 15:36:35 -0600 +Subject: sh: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit 6d3a273355e3c8471ddf9e8ce9a7cc4472bf1ccc ] + +Wire up TIF_NOTIFY_SIGNAL handling for sh. + +Cc: linux-sh@vger.kernel.org +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/sh/include/asm/thread_info.h | 4 +++- + arch/sh/kernel/signal_32.c | 2 +- + 2 files changed, 4 insertions(+), 2 deletions(-) + +--- a/arch/sh/include/asm/thread_info.h ++++ b/arch/sh/include/asm/thread_info.h +@@ -105,6 +105,7 @@ extern void init_thread_xstate(void); + #define TIF_SYSCALL_TRACE 0 /* syscall trace active */ + #define TIF_SIGPENDING 1 /* signal pending */ + #define TIF_NEED_RESCHED 2 /* rescheduling necessary */ ++#define TIF_NOTIFY_SIGNAL 3 /* signal notifications exist */ + #define TIF_SINGLESTEP 4 /* singlestepping active */ + #define TIF_SYSCALL_AUDIT 5 /* syscall auditing active */ + #define TIF_SECCOMP 6 /* secure computing */ +@@ -116,6 +117,7 @@ extern void init_thread_xstate(void); + #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) + #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) + #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + #define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) + #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) + #define _TIF_SECCOMP (1 << TIF_SECCOMP) +@@ -132,7 +134,7 @@ extern void init_thread_xstate(void); + #define _TIF_ALLWORK_MASK (_TIF_SYSCALL_TRACE | _TIF_SIGPENDING | \ + _TIF_NEED_RESCHED | _TIF_SYSCALL_AUDIT | \ + _TIF_SINGLESTEP | _TIF_NOTIFY_RESUME | \ +- _TIF_SYSCALL_TRACEPOINT) ++ _TIF_SYSCALL_TRACEPOINT | _TIF_NOTIFY_SIGNAL) + + /* work to do on interrupt/exception return */ + #define _TIF_WORK_MASK (_TIF_ALLWORK_MASK & ~(_TIF_SYSCALL_TRACE | \ +--- a/arch/sh/kernel/signal_32.c ++++ b/arch/sh/kernel/signal_32.c +@@ -499,7 +499,7 @@ asmlinkage void do_notify_resume(struct + unsigned long thread_info_flags) + { + /* deal with pending signal delivery */ +- if (thread_info_flags & _TIF_SIGPENDING) ++ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs, save_r0); + + if (thread_info_flags & _TIF_NOTIFY_RESUME) diff --git a/next-5.10/signal-add-task_sigpending-helper.patch b/next-5.10/signal-add-task_sigpending-helper.patch new file mode 100644 index 00000000000..a946e143947 --- /dev/null +++ b/next-5.10/signal-add-task_sigpending-helper.patch @@ -0,0 +1,108 @@ +From a70e1337ccd380cbb8b647c1645ded7f3bf7a136 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Mon, 26 Oct 2020 14:32:27 -0600 +Subject: signal: Add task_sigpending() helper + +From: Jens Axboe + +[ Upstream commit 5c251e9dc0e127bac6fc5b8e6696363d2e35f515 ] + +This is in preparation for maintaining signal_pending() as the decider of +whether or not a schedule() loop should be broken, or continue sleeping. +This is different than the core signal use cases, which really need to know +whether an actual signal is pending or not. task_sigpending() returns +non-zero if TIF_SIGPENDING is set. + +Only core kernel use cases should care about the distinction between +the two, make sure those use the task_sigpending() helper. + +Signed-off-by: Jens Axboe +Signed-off-by: Thomas Gleixner +Reviewed-by: Thomas Gleixner +Reviewed-by: Oleg Nesterov +Link: https://lore.kernel.org/r/20201026203230.386348-2-axboe@kernel.dk +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/sched/signal.h | 9 +++++++-- + kernel/events/uprobes.c | 2 +- + kernel/signal.c | 8 ++++---- + 3 files changed, 12 insertions(+), 7 deletions(-) + +--- a/include/linux/sched/signal.h ++++ b/include/linux/sched/signal.h +@@ -354,11 +354,16 @@ static inline int restart_syscall(void) + return -ERESTARTNOINTR; + } + +-static inline int signal_pending(struct task_struct *p) ++static inline int task_sigpending(struct task_struct *p) + { + return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING)); + } + ++static inline int signal_pending(struct task_struct *p) ++{ ++ return task_sigpending(p); ++} ++ + static inline int __fatal_signal_pending(struct task_struct *p) + { + return unlikely(sigismember(&p->pending.signal, SIGKILL)); +@@ -366,7 +371,7 @@ static inline int __fatal_signal_pending + + static inline int fatal_signal_pending(struct task_struct *p) + { +- return signal_pending(p) && __fatal_signal_pending(p); ++ return task_sigpending(p) && __fatal_signal_pending(p); + } + + static inline int signal_pending_state(long state, struct task_struct *p) +--- a/kernel/events/uprobes.c ++++ b/kernel/events/uprobes.c +@@ -1973,7 +1973,7 @@ bool uprobe_deny_signal(void) + + WARN_ON_ONCE(utask->state != UTASK_SSTEP); + +- if (signal_pending(t)) { ++ if (task_sigpending(t)) { + spin_lock_irq(&t->sighand->siglock); + clear_tsk_thread_flag(t, TIF_SIGPENDING); + spin_unlock_irq(&t->sighand->siglock); +--- a/kernel/signal.c ++++ b/kernel/signal.c +@@ -984,7 +984,7 @@ static inline bool wants_signal(int sig, + if (task_is_stopped_or_traced(p)) + return false; + +- return task_curr(p) || !signal_pending(p); ++ return task_curr(p) || !task_sigpending(p); + } + + static void complete_signal(int sig, struct task_struct *p, enum pid_type type) +@@ -2813,7 +2813,7 @@ static void retarget_shared_pending(stru + /* Remove the signals this thread can handle. */ + sigandsets(&retarget, &retarget, &t->blocked); + +- if (!signal_pending(t)) ++ if (!task_sigpending(t)) + signal_wake_up(t, 0); + + if (sigisemptyset(&retarget)) +@@ -2847,7 +2847,7 @@ void exit_signals(struct task_struct *ts + + cgroup_threadgroup_change_end(tsk); + +- if (!signal_pending(tsk)) ++ if (!task_sigpending(tsk)) + goto out; + + unblocked = tsk->blocked; +@@ -2891,7 +2891,7 @@ long do_no_restart_syscall(struct restar + + static void __set_task_blocked(struct task_struct *tsk, const sigset_t *newset) + { +- if (signal_pending(tsk) && !thread_group_empty(tsk)) { ++ if (task_sigpending(tsk) && !thread_group_empty(tsk)) { + sigset_t newblocked; + /* A set of now blocked but previously unblocked signals. */ + sigandnsets(&newblocked, newset, ¤t->blocked); diff --git a/next-5.10/signal-kill-jobctl_task_work.patch b/next-5.10/signal-kill-jobctl_task_work.patch new file mode 100644 index 00000000000..ba6be83cd84 --- /dev/null +++ b/next-5.10/signal-kill-jobctl_task_work.patch @@ -0,0 +1,69 @@ +From 450fb0845804e4245b90eb155d7ac34cda11429f Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 16:03:01 -0600 +Subject: signal: kill JOBCTL_TASK_WORK + +From: Jens Axboe + +[ Upstream commit 98b89b649fce39dacb9dc036d6d0fdb8caff73f7 ] + +It's no longer used, get rid of it. + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/sched/jobctl.h | 4 +--- + kernel/signal.c | 20 -------------------- + 2 files changed, 1 insertion(+), 23 deletions(-) + +--- a/include/linux/sched/jobctl.h ++++ b/include/linux/sched/jobctl.h +@@ -19,7 +19,6 @@ struct task_struct; + #define JOBCTL_TRAPPING_BIT 21 /* switching to TRACED */ + #define JOBCTL_LISTENING_BIT 22 /* ptracer is listening for events */ + #define JOBCTL_TRAP_FREEZE_BIT 23 /* trap for cgroup freezer */ +-#define JOBCTL_TASK_WORK_BIT 24 /* set by TWA_SIGNAL */ + + #define JOBCTL_STOP_DEQUEUED (1UL << JOBCTL_STOP_DEQUEUED_BIT) + #define JOBCTL_STOP_PENDING (1UL << JOBCTL_STOP_PENDING_BIT) +@@ -29,10 +28,9 @@ struct task_struct; + #define JOBCTL_TRAPPING (1UL << JOBCTL_TRAPPING_BIT) + #define JOBCTL_LISTENING (1UL << JOBCTL_LISTENING_BIT) + #define JOBCTL_TRAP_FREEZE (1UL << JOBCTL_TRAP_FREEZE_BIT) +-#define JOBCTL_TASK_WORK (1UL << JOBCTL_TASK_WORK_BIT) + + #define JOBCTL_TRAP_MASK (JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY) +-#define JOBCTL_PENDING_MASK (JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK | JOBCTL_TASK_WORK) ++#define JOBCTL_PENDING_MASK (JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK) + + extern bool task_set_jobctl_pending(struct task_struct *task, unsigned long mask); + extern void task_clear_jobctl_trapping(struct task_struct *task); +--- a/kernel/signal.c ++++ b/kernel/signal.c +@@ -2544,26 +2544,6 @@ bool get_signal(struct ksignal *ksig) + + relock: + spin_lock_irq(&sighand->siglock); +- /* +- * Make sure we can safely read ->jobctl() in task_work add. As Oleg +- * states: +- * +- * It pairs with mb (implied by cmpxchg) before READ_ONCE. So we +- * roughly have +- * +- * task_work_add: get_signal: +- * STORE(task->task_works, new_work); STORE(task->jobctl); +- * mb(); mb(); +- * LOAD(task->jobctl); LOAD(task->task_works); +- * +- * and we can rely on STORE-MB-LOAD [ in task_work_add]. +- */ +- smp_store_mb(current->jobctl, current->jobctl & ~JOBCTL_TASK_WORK); +- if (unlikely(current->task_works)) { +- spin_unlock_irq(&sighand->siglock); +- task_work_run(); +- goto relock; +- } + + /* + * Every stopped thread goes here after wakeup. Check to see if diff --git a/queue-5.10/skbuff-account-for-tail-adjustment-during-pull-opera.patch b/next-5.10/skbuff-account-for-tail-adjustment-during-pull-opera.patch similarity index 100% rename from queue-5.10/skbuff-account-for-tail-adjustment-during-pull-opera.patch rename to next-5.10/skbuff-account-for-tail-adjustment-during-pull-opera.patch diff --git a/queue-5.10/soc-qcom-apr-add-check-for-idr_alloc-and-of_property.patch b/next-5.10/soc-qcom-apr-add-check-for-idr_alloc-and-of_property.patch similarity index 100% rename from queue-5.10/soc-qcom-apr-add-check-for-idr_alloc-and-of_property.patch rename to next-5.10/soc-qcom-apr-add-check-for-idr_alloc-and-of_property.patch diff --git a/queue-5.10/soc-qcom-apr-make-code-more-reuseable.patch b/next-5.10/soc-qcom-apr-make-code-more-reuseable.patch similarity index 100% rename from queue-5.10/soc-qcom-apr-make-code-more-reuseable.patch rename to next-5.10/soc-qcom-apr-make-code-more-reuseable.patch diff --git a/queue-5.10/soc-qcom-llcc-make-irq-truly-optional.patch b/next-5.10/soc-qcom-llcc-make-irq-truly-optional.patch similarity index 100% rename from queue-5.10/soc-qcom-llcc-make-irq-truly-optional.patch rename to next-5.10/soc-qcom-llcc-make-irq-truly-optional.patch diff --git a/queue-5.10/soc-ti-knav_qmss_queue-fix-pm-disable-depth-imbalanc.patch b/next-5.10/soc-ti-knav_qmss_queue-fix-pm-disable-depth-imbalanc.patch similarity index 100% rename from queue-5.10/soc-ti-knav_qmss_queue-fix-pm-disable-depth-imbalanc.patch rename to next-5.10/soc-ti-knav_qmss_queue-fix-pm-disable-depth-imbalanc.patch diff --git a/queue-5.10/soc-ti-knav_qmss_queue-use-pm_runtime_resume_and_get.patch b/next-5.10/soc-ti-knav_qmss_queue-use-pm_runtime_resume_and_get.patch similarity index 100% rename from queue-5.10/soc-ti-knav_qmss_queue-use-pm_runtime_resume_and_get.patch rename to next-5.10/soc-ti-knav_qmss_queue-use-pm_runtime_resume_and_get.patch diff --git a/queue-5.10/soc-ti-smartreflex-fix-pm-disable-depth-imbalance-in.patch b/next-5.10/soc-ti-smartreflex-fix-pm-disable-depth-imbalance-in.patch similarity index 100% rename from queue-5.10/soc-ti-smartreflex-fix-pm-disable-depth-imbalance-in.patch rename to next-5.10/soc-ti-smartreflex-fix-pm-disable-depth-imbalance-in.patch diff --git a/next-5.10/sparc-add-support-for-tif_notify_signal.patch b/next-5.10/sparc-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..f9112eab075 --- /dev/null +++ b/next-5.10/sparc-add-support-for-tif_notify_signal.patch @@ -0,0 +1,96 @@ +From 12d81984f0d70ca40c14f319ceea586b9351b2cb Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 15:44:37 -0600 +Subject: sparc: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit f50a7052f5e70ee7a6a5e2ed08660994dc3df2a5 ] + +Wire up TIF_NOTIFY_SIGNAL handling for sparc. + +Cc: sparclinux@vger.kernel.org +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/sparc/include/asm/thread_info_32.h | 4 +++- + arch/sparc/include/asm/thread_info_64.h | 6 ++++-- + arch/sparc/kernel/signal_32.c | 2 +- + arch/sparc/kernel/signal_64.c | 2 +- + 4 files changed, 9 insertions(+), 5 deletions(-) + +--- a/arch/sparc/include/asm/thread_info_32.h ++++ b/arch/sparc/include/asm/thread_info_32.h +@@ -104,6 +104,7 @@ register struct thread_info *current_thr + #define TIF_SIGPENDING 2 /* signal pending */ + #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ + #define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */ ++#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ + #define TIF_USEDFPU 8 /* FPU was used by this task + * this quantum (SMP) */ + #define TIF_POLLING_NRFLAG 9 /* true if poll_idle() is polling +@@ -115,11 +116,12 @@ register struct thread_info *current_thr + #define _TIF_NOTIFY_RESUME (1< +Date: Thu, 1 Apr 2021 19:53:29 -0600 +Subject: task_work: add helper for more targeted task_work canceling + +From: Jens Axboe + +[ Upstream commit c7aab1a7c52b82d9afd7e03c398eb03dc2aa0507 ] + +The only exported helper we have right now is task_work_cancel(), which +cancels any task_work from a given task where func matches the queued +work item. This is a bit too coarse for some use cases. Add a +task_work_cancel_match() that allows to more specifically target +individual work items outside of purely the callback function used. + +task_work_cancel() can be trivially implemented on top of that, hence do +so. + +Reviewed-by: Oleg Nesterov +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/task_work.h | 2 ++ + kernel/task_work.c | 35 ++++++++++++++++++++++++++++------- + 2 files changed, 30 insertions(+), 7 deletions(-) + +--- a/include/linux/task_work.h ++++ b/include/linux/task_work.h +@@ -22,6 +22,8 @@ enum task_work_notify_mode { + int task_work_add(struct task_struct *task, struct callback_head *twork, + enum task_work_notify_mode mode); + ++struct callback_head *task_work_cancel_match(struct task_struct *task, ++ bool (*match)(struct callback_head *, void *data), void *data); + struct callback_head *task_work_cancel(struct task_struct *, task_work_func_t); + void task_work_run(void); + +--- a/kernel/task_work.c ++++ b/kernel/task_work.c +@@ -59,18 +59,17 @@ int task_work_add(struct task_struct *ta + } + + /** +- * task_work_cancel - cancel a pending work added by task_work_add() ++ * task_work_cancel_match - cancel a pending work added by task_work_add() + * @task: the task which should execute the work +- * @func: identifies the work to remove +- * +- * Find the last queued pending work with ->func == @func and remove +- * it from queue. ++ * @match: match function to call + * + * RETURNS: + * The found work or NULL if not found. + */ + struct callback_head * +-task_work_cancel(struct task_struct *task, task_work_func_t func) ++task_work_cancel_match(struct task_struct *task, ++ bool (*match)(struct callback_head *, void *data), ++ void *data) + { + struct callback_head **pprev = &task->task_works; + struct callback_head *work; +@@ -86,7 +85,7 @@ task_work_cancel(struct task_struct *tas + */ + raw_spin_lock_irqsave(&task->pi_lock, flags); + while ((work = READ_ONCE(*pprev))) { +- if (work->func != func) ++ if (!match(work, data)) + pprev = &work->next; + else if (cmpxchg(pprev, work, work->next) == work) + break; +@@ -96,6 +95,28 @@ task_work_cancel(struct task_struct *tas + return work; + } + ++static bool task_work_func_match(struct callback_head *cb, void *data) ++{ ++ return cb->func == data; ++} ++ ++/** ++ * task_work_cancel - cancel a pending work added by task_work_add() ++ * @task: the task which should execute the work ++ * @func: identifies the work to remove ++ * ++ * Find the last queued pending work with ->func == @func and remove ++ * it from queue. ++ * ++ * RETURNS: ++ * The found work or NULL if not found. ++ */ ++struct callback_head * ++task_work_cancel(struct task_struct *task, task_work_func_t func) ++{ ++ return task_work_cancel_match(task, task_work_func_match, func); ++} ++ + /** + * task_work_run - execute the works added by task_work_add() + * diff --git a/next-5.10/task_work-remove-legacy-twa_signal-path.patch b/next-5.10/task_work-remove-legacy-twa_signal-path.patch new file mode 100644 index 00000000000..0b9c01742a9 --- /dev/null +++ b/next-5.10/task_work-remove-legacy-twa_signal-path.patch @@ -0,0 +1,63 @@ +From 3e7b055854f9ff08f3c7015c44229147dad5dd08 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 16:01:33 -0600 +Subject: task_work: remove legacy TWA_SIGNAL path + +From: Jens Axboe + +[ Upstream commit 03941ccfda161c2680147fa5ab92aead2a79cac1 ] + +All archs now support TIF_NOTIFY_SIGNAL. + +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + kernel/task_work.c | 30 +----------------------------- + 1 file changed, 1 insertion(+), 29 deletions(-) + +--- a/kernel/task_work.c ++++ b/kernel/task_work.c +@@ -5,34 +5,6 @@ + + static struct callback_head work_exited; /* all we need is ->next == NULL */ + +-/* +- * TWA_SIGNAL signaling - use TIF_NOTIFY_SIGNAL, if available, as it's faster +- * than TIF_SIGPENDING as there's no dependency on ->sighand. The latter is +- * shared for threads, and can cause contention on sighand->lock. Even for +- * the non-threaded case TIF_NOTIFY_SIGNAL is more efficient, as no locking +- * or IRQ disabling is involved for notification (or running) purposes. +- */ +-static void task_work_notify_signal(struct task_struct *task) +-{ +-#if defined(TIF_NOTIFY_SIGNAL) +- set_notify_signal(task); +-#else +- unsigned long flags; +- +- /* +- * Only grab the sighand lock if we don't already have some +- * task_work pending. This pairs with the smp_store_mb() +- * in get_signal(), see comment there. +- */ +- if (!(READ_ONCE(task->jobctl) & JOBCTL_TASK_WORK) && +- lock_task_sighand(task, &flags)) { +- task->jobctl |= JOBCTL_TASK_WORK; +- signal_wake_up(task, 0); +- unlock_task_sighand(task, &flags); +- } +-#endif +-} +- + /** + * task_work_add - ask the @task to execute @work->func() + * @task: the task which should run the callback +@@ -76,7 +48,7 @@ int task_work_add(struct task_struct *ta + set_notify_resume(task); + break; + case TWA_SIGNAL: +- task_work_notify_signal(task); ++ set_notify_signal(task); + break; + default: + WARN_ON_ONCE(1); diff --git a/next-5.10/task_work-unconditionally-run-task_work-from-get_signal.patch b/next-5.10/task_work-unconditionally-run-task_work-from-get_signal.patch new file mode 100644 index 00000000000..90a6ab47f89 --- /dev/null +++ b/next-5.10/task_work-unconditionally-run-task_work-from-get_signal.patch @@ -0,0 +1,44 @@ +From 0f315fd4143c1591a7f902fa08b71b83764f4fa0 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Tue, 5 Jan 2021 11:32:43 -0700 +Subject: task_work: unconditionally run task_work from get_signal() + +From: Jens Axboe + +[ Upstream commit 35d0b389f3b23439ad15b610d6e43fc72fc75779 ] + +Song reported a boot regression in a kvm image with 5.11-rc, and bisected +it down to the below patch. Debugging this issue, turns out that the boot +stalled when a task is waiting on a pipe being released. As we no longer +run task_work from get_signal() unless it's queued with TWA_SIGNAL, the +task goes idle without running the task_work. This prevents ->release() +from being called on the pipe, which another boot task is waiting on. + +For now, re-instate the unconditional task_work run from get_signal(). +For 5.12, we'll collapse TWA_RESUME and TWA_SIGNAL, as it no longer +makes sense to have a distinction between the two. This will turn +task_work notification into a simple boolean, whether to notify or not. + +Fixes: 98b89b649fce ("signal: kill JOBCTL_TASK_WORK") +Reported-by: Song Liu +Tested-by: John Stultz +Tested-by: Douglas Anderson +Tested-by: Sedat Dilek # LLVM/Clang version 11.0.1 +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + kernel/signal.c | 3 +++ + 1 file changed, 3 insertions(+) + +--- a/kernel/signal.c ++++ b/kernel/signal.c +@@ -2520,6 +2520,9 @@ bool get_signal(struct ksignal *ksig) + struct signal_struct *signal = current->signal; + int signr; + ++ if (unlikely(current->task_works)) ++ task_work_run(); ++ + /* + * For non-generic architectures, check for TIF_NOTIFY_SIGNAL so + * that the arch handlers don't all have to do it. If we get here diff --git a/next-5.10/task_work-use-tif_notify_signal-if-available.patch b/next-5.10/task_work-use-tif_notify_signal-if-available.patch new file mode 100644 index 00000000000..41b068d36db --- /dev/null +++ b/next-5.10/task_work-use-tif_notify_signal-if-available.patch @@ -0,0 +1,138 @@ +From fd7614eb55ed1cd0017baea8a7621586d44089be Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Mon, 26 Oct 2020 14:32:30 -0600 +Subject: task_work: Use TIF_NOTIFY_SIGNAL if available + +From: Jens Axboe + +[ Upstream commit 114518eb6430b832d2f9f5a008043b913ccf0e24 ] + +If the arch supports TIF_NOTIFY_SIGNAL, then use that for TWA_SIGNAL as +it's more efficient than using the signal delivery method. This is +especially true on threaded applications, where ->sighand is shared across +threads, but it's also lighter weight on non-shared cases. + +io_uring is a heavy consumer of TWA_SIGNAL based task_work. A test with +threads shows a nice improvement running an io_uring based echo server. + +stock kernel: +0.01% <= 0.1 milliseconds +95.86% <= 0.2 milliseconds +98.27% <= 0.3 milliseconds +99.71% <= 0.4 milliseconds +100.00% <= 0.5 milliseconds +100.00% <= 0.6 milliseconds +100.00% <= 0.7 milliseconds +100.00% <= 0.8 milliseconds +100.00% <= 0.9 milliseconds +100.00% <= 1.0 milliseconds +100.00% <= 1.1 milliseconds +100.00% <= 2 milliseconds +100.00% <= 3 milliseconds +100.00% <= 3 milliseconds +1378930.00 requests per second +~1600% CPU + +1.38M requests/second, and all 16 CPUs are maxed out. + +patched kernel: +0.01% <= 0.1 milliseconds +98.24% <= 0.2 milliseconds +99.47% <= 0.3 milliseconds +99.99% <= 0.4 milliseconds +100.00% <= 0.5 milliseconds +100.00% <= 0.6 milliseconds +100.00% <= 0.7 milliseconds +100.00% <= 0.8 milliseconds +100.00% <= 0.9 milliseconds +100.00% <= 1.2 milliseconds +1666111.38 requests per second +~1450% CPU + +1.67M requests/second, and we're no longer just hammering on the sighand +lock. The original reporter states: + +"For 5.7.15 my benchmark achieves 1.6M qps and system cpu is at ~80%. + for 5.7.16 or later it achieves only 1M qps and the system cpu is is + at ~100%" + +with the only difference there being that TWA_SIGNAL is used +unconditionally in 5.7.16, since it's required to be able to handle the +inability to run task_work if the application is waiting in the kernel +already on an event that needs task_work run to be satisfied. Also see +commit 0ba9c9edcd15. + +Reported-by: Roman Gershman +Signed-off-by: Jens Axboe +Signed-off-by: Thomas Gleixner +Reviewed-by: Oleg Nesterov +Link: https://lore.kernel.org/r/20201026203230.386348-5-axboe@kernel.dk +Signed-off-by: Greg Kroah-Hartman +--- + kernel/task_work.c | 41 +++++++++++++++++++++++++++++------------ + 1 file changed, 29 insertions(+), 12 deletions(-) + +--- a/kernel/task_work.c ++++ b/kernel/task_work.c +@@ -5,6 +5,34 @@ + + static struct callback_head work_exited; /* all we need is ->next == NULL */ + ++/* ++ * TWA_SIGNAL signaling - use TIF_NOTIFY_SIGNAL, if available, as it's faster ++ * than TIF_SIGPENDING as there's no dependency on ->sighand. The latter is ++ * shared for threads, and can cause contention on sighand->lock. Even for ++ * the non-threaded case TIF_NOTIFY_SIGNAL is more efficient, as no locking ++ * or IRQ disabling is involved for notification (or running) purposes. ++ */ ++static void task_work_notify_signal(struct task_struct *task) ++{ ++#if defined(TIF_NOTIFY_SIGNAL) ++ set_notify_signal(task); ++#else ++ unsigned long flags; ++ ++ /* ++ * Only grab the sighand lock if we don't already have some ++ * task_work pending. This pairs with the smp_store_mb() ++ * in get_signal(), see comment there. ++ */ ++ if (!(READ_ONCE(task->jobctl) & JOBCTL_TASK_WORK) && ++ lock_task_sighand(task, &flags)) { ++ task->jobctl |= JOBCTL_TASK_WORK; ++ signal_wake_up(task, 0); ++ unlock_task_sighand(task, &flags); ++ } ++#endif ++} ++ + /** + * task_work_add - ask the @task to execute @work->func() + * @task: the task which should run the callback +@@ -33,7 +61,6 @@ int task_work_add(struct task_struct *ta + enum task_work_notify_mode notify) + { + struct callback_head *head; +- unsigned long flags; + + do { + head = READ_ONCE(task->task_works); +@@ -49,17 +76,7 @@ int task_work_add(struct task_struct *ta + set_notify_resume(task); + break; + case TWA_SIGNAL: +- /* +- * Only grab the sighand lock if we don't already have some +- * task_work pending. This pairs with the smp_store_mb() +- * in get_signal(), see comment there. +- */ +- if (!(READ_ONCE(task->jobctl) & JOBCTL_TASK_WORK) && +- lock_task_sighand(task, &flags)) { +- task->jobctl |= JOBCTL_TASK_WORK; +- signal_wake_up(task, 0); +- unlock_task_sighand(task, &flags); +- } ++ task_work_notify_signal(task); + break; + default: + WARN_ON_ONCE(1); diff --git a/queue-5.10/test_firmware-fix-memory-leak-in-test_firmware_init.patch b/next-5.10/test_firmware-fix-memory-leak-in-test_firmware_init.patch similarity index 100% rename from queue-5.10/test_firmware-fix-memory-leak-in-test_firmware_init.patch rename to next-5.10/test_firmware-fix-memory-leak-in-test_firmware_init.patch diff --git a/queue-5.10/thermal-drivers-imx8mm_thermal-validate-temperature-.patch b/next-5.10/thermal-drivers-imx8mm_thermal-validate-temperature-.patch similarity index 100% rename from queue-5.10/thermal-drivers-imx8mm_thermal-validate-temperature-.patch rename to next-5.10/thermal-drivers-imx8mm_thermal-validate-temperature-.patch diff --git a/queue-5.10/timerqueue-use-rb_entry_safe-in-timerqueue_getnext.patch b/next-5.10/timerqueue-use-rb_entry_safe-in-timerqueue_getnext.patch similarity index 100% rename from queue-5.10/timerqueue-use-rb_entry_safe-in-timerqueue_getnext.patch rename to next-5.10/timerqueue-use-rb_entry_safe-in-timerqueue_getnext.patch diff --git a/next-5.10/tools-headers-uapi-sync-openat2.h-with-the-kernel-sources.patch b/next-5.10/tools-headers-uapi-sync-openat2.h-with-the-kernel-sources.patch new file mode 100644 index 00000000000..94d7b7f22b5 --- /dev/null +++ b/next-5.10/tools-headers-uapi-sync-openat2.h-with-the-kernel-sources.patch @@ -0,0 +1,39 @@ +From 8e9c2ce1060f14cda30808322d9a8dcbd7a3fd77 Mon Sep 17 00:00:00 2001 +From: Arnaldo Carvalho de Melo +Date: Tue, 23 Feb 2021 09:44:37 -0300 +Subject: tools headers UAPI: Sync openat2.h with the kernel sources + +From: Arnaldo Carvalho de Melo + +[ Upstream commit 1e61463cfcd0b3e7a19ba36b8a98c64ebaac5c6e ] + +To pick the changes in: + + 99668f618062816c ("fs: expose LOOKUP_CACHED through openat2() RESOLVE_CACHED") + +That don't result in any change in tooling, only silences this perf +build warning: + + Warning: Kernel ABI header at 'tools/include/uapi/linux/openat2.h' differs from latest version at 'include/uapi/linux/openat2.h' + diff -u tools/include/uapi/linux/openat2.h include/uapi/linux/openat2.h + +Cc: Al Viro +Cc: Jens Axboe +Signed-off-by: Arnaldo Carvalho de Melo +Signed-off-by: Greg Kroah-Hartman +--- + tools/include/uapi/linux/openat2.h | 4 ++++ + 1 file changed, 4 insertions(+) + +--- a/tools/include/uapi/linux/openat2.h ++++ b/tools/include/uapi/linux/openat2.h +@@ -35,5 +35,9 @@ struct open_how { + #define RESOLVE_IN_ROOT 0x10 /* Make all jumps to "/" and ".." + be scoped inside the dirfd + (similar to chroot(2)). */ ++#define RESOLVE_CACHED 0x20 /* Only complete if resolution can be ++ completed through cached lookup. May ++ return -EAGAIN if that's not ++ possible. */ + + #endif /* _UAPI_LINUX_OPENAT2_H */ diff --git a/queue-5.10/tpm-acpi-call-acpi_put_table-to-fix-memory-leak.patch b/next-5.10/tpm-acpi-call-acpi_put_table-to-fix-memory-leak.patch similarity index 100% rename from queue-5.10/tpm-acpi-call-acpi_put_table-to-fix-memory-leak.patch rename to next-5.10/tpm-acpi-call-acpi_put_table-to-fix-memory-leak.patch diff --git a/queue-5.10/tpm-tpm_crb-add-the-missed-acpi_put_table-to-fix-memory-leak.patch b/next-5.10/tpm-tpm_crb-add-the-missed-acpi_put_table-to-fix-memory-leak.patch similarity index 100% rename from queue-5.10/tpm-tpm_crb-add-the-missed-acpi_put_table-to-fix-memory-leak.patch rename to next-5.10/tpm-tpm_crb-add-the-missed-acpi_put_table-to-fix-memory-leak.patch diff --git a/queue-5.10/tpm-tpm_crb-fix-error-message-in-__crb_relinquish_lo.patch b/next-5.10/tpm-tpm_crb-fix-error-message-in-__crb_relinquish_lo.patch similarity index 100% rename from queue-5.10/tpm-tpm_crb-fix-error-message-in-__crb_relinquish_lo.patch rename to next-5.10/tpm-tpm_crb-fix-error-message-in-__crb_relinquish_lo.patch diff --git a/queue-5.10/tpm-tpm_ftpm_tee-fix-error-handling-in-ftpm_mod_init.patch b/next-5.10/tpm-tpm_ftpm_tee-fix-error-handling-in-ftpm_mod_init.patch similarity index 100% rename from queue-5.10/tpm-tpm_ftpm_tee-fix-error-handling-in-ftpm_mod_init.patch rename to next-5.10/tpm-tpm_ftpm_tee-fix-error-handling-in-ftpm_mod_init.patch diff --git a/queue-5.10/tpm-tpm_tis-add-the-missed-acpi_put_table-to-fix-memory-leak.patch b/next-5.10/tpm-tpm_tis-add-the-missed-acpi_put_table-to-fix-memory-leak.patch similarity index 100% rename from queue-5.10/tpm-tpm_tis-add-the-missed-acpi_put_table-to-fix-memory-leak.patch rename to next-5.10/tpm-tpm_tis-add-the-missed-acpi_put_table-to-fix-memory-leak.patch diff --git a/queue-5.10/tracing-hist-fix-issue-of-losting-command-info-in-er.patch b/next-5.10/tracing-hist-fix-issue-of-losting-command-info-in-er.patch similarity index 100% rename from queue-5.10/tracing-hist-fix-issue-of-losting-command-info-in-er.patch rename to next-5.10/tracing-hist-fix-issue-of-losting-command-info-in-er.patch diff --git a/queue-5.10/tty-serial-altera_uart_-r-t-x_chars-need-only-uart_p.patch b/next-5.10/tty-serial-altera_uart_-r-t-x_chars-need-only-uart_p.patch similarity index 100% rename from queue-5.10/tty-serial-altera_uart_-r-t-x_chars-need-only-uart_p.patch rename to next-5.10/tty-serial-altera_uart_-r-t-x_chars-need-only-uart_p.patch diff --git a/queue-5.10/tty-serial-clean-up-stop-tx-part-in-altera_uart_tx_c.patch b/next-5.10/tty-serial-clean-up-stop-tx-part-in-altera_uart_tx_c.patch similarity index 100% rename from queue-5.10/tty-serial-clean-up-stop-tx-part-in-altera_uart_tx_c.patch rename to next-5.10/tty-serial-clean-up-stop-tx-part-in-altera_uart_tx_c.patch diff --git a/queue-5.10/udf-avoid-double-brelse-in-udf_rename.patch b/next-5.10/udf-avoid-double-brelse-in-udf_rename.patch similarity index 100% rename from queue-5.10/udf-avoid-double-brelse-in-udf_rename.patch rename to next-5.10/udf-avoid-double-brelse-in-udf_rename.patch diff --git a/queue-5.10/uio-uio_dmem_genirq-fix-deadlock-between-irq-config-.patch b/next-5.10/uio-uio_dmem_genirq-fix-deadlock-between-irq-config-.patch similarity index 100% rename from queue-5.10/uio-uio_dmem_genirq-fix-deadlock-between-irq-config-.patch rename to next-5.10/uio-uio_dmem_genirq-fix-deadlock-between-irq-config-.patch diff --git a/queue-5.10/uio-uio_dmem_genirq-fix-missing-unlock-in-irq-config.patch b/next-5.10/uio-uio_dmem_genirq-fix-missing-unlock-in-irq-config.patch similarity index 100% rename from queue-5.10/uio-uio_dmem_genirq-fix-missing-unlock-in-irq-config.patch rename to next-5.10/uio-uio_dmem_genirq-fix-missing-unlock-in-irq-config.patch diff --git a/next-5.10/um-add-support-for-tif_notify_signal.patch b/next-5.10/um-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..d561142d6f7 --- /dev/null +++ b/next-5.10/um-add-support-for-tif_notify_signal.patch @@ -0,0 +1,50 @@ +From 9b4d7860ca092440b774c82827455f53e649ee0f Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 15:47:28 -0600 +Subject: um: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit a5b3cd32ff238b87e94d47b927aff117e22d13c0 ] + +Wire up TIF_NOTIFY_SIGNAL handling for um. + +Cc: linux-um@lists.infradead.org +Acked-By: Anton Ivanov +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/um/include/asm/thread_info.h | 2 ++ + arch/um/kernel/process.c | 3 ++- + 2 files changed, 4 insertions(+), 1 deletion(-) + +--- a/arch/um/include/asm/thread_info.h ++++ b/arch/um/include/asm/thread_info.h +@@ -57,6 +57,7 @@ static inline struct thread_info *curren + #define TIF_SYSCALL_TRACE 0 /* syscall trace active */ + #define TIF_SIGPENDING 1 /* signal pending */ + #define TIF_NEED_RESCHED 2 /* rescheduling necessary */ ++#define TIF_NOTIFY_SIGNAL 3 /* signal notifications exist */ + #define TIF_RESTART_BLOCK 4 + #define TIF_MEMDIE 5 /* is terminating due to OOM killer */ + #define TIF_SYSCALL_AUDIT 6 +@@ -68,6 +69,7 @@ static inline struct thread_info *curren + #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) + #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) + #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + #define _TIF_MEMDIE (1 << TIF_MEMDIE) + #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) + #define _TIF_SECCOMP (1 << TIF_SECCOMP) +--- a/arch/um/kernel/process.c ++++ b/arch/um/kernel/process.c +@@ -99,7 +99,8 @@ void interrupt_end(void) + + if (need_resched()) + schedule(); +- if (test_thread_flag(TIF_SIGPENDING)) ++ if (test_thread_flag(TIF_SIGPENDING) || ++ test_thread_flag(TIF_NOTIFY_SIGNAL)) + do_signal(regs); + if (test_thread_flag(TIF_NOTIFY_RESUME)) + tracehook_notify_resume(regs); diff --git a/queue-5.10/uprobes-x86-allow-to-probe-a-nop-instruction-with-0x.patch b/next-5.10/uprobes-x86-allow-to-probe-a-nop-instruction-with-0x.patch similarity index 100% rename from queue-5.10/uprobes-x86-allow-to-probe-a-nop-instruction-with-0x.patch rename to next-5.10/uprobes-x86-allow-to-probe-a-nop-instruction-with-0x.patch diff --git a/queue-5.10/usb-dwc3-core-defer-probe-on-ulpi_read_id-timeout.patch b/next-5.10/usb-dwc3-core-defer-probe-on-ulpi_read_id-timeout.patch similarity index 100% rename from queue-5.10/usb-dwc3-core-defer-probe-on-ulpi_read_id-timeout.patch rename to next-5.10/usb-dwc3-core-defer-probe-on-ulpi_read_id-timeout.patch diff --git a/queue-5.10/usb-dwc3-fix-race-between-dwc3_set_mode-and-__dwc3_set_mode.patch b/next-5.10/usb-dwc3-fix-race-between-dwc3_set_mode-and-__dwc3_set_mode.patch similarity index 100% rename from queue-5.10/usb-dwc3-fix-race-between-dwc3_set_mode-and-__dwc3_set_mode.patch rename to next-5.10/usb-dwc3-fix-race-between-dwc3_set_mode-and-__dwc3_set_mode.patch diff --git a/queue-5.10/usb-dwc3-qcom-fix-memory-leak-in-dwc3_qcom_interconn.patch b/next-5.10/usb-dwc3-qcom-fix-memory-leak-in-dwc3_qcom_interconn.patch similarity index 100% rename from queue-5.10/usb-dwc3-qcom-fix-memory-leak-in-dwc3_qcom_interconn.patch rename to next-5.10/usb-dwc3-qcom-fix-memory-leak-in-dwc3_qcom_interconn.patch diff --git a/queue-5.10/usb-fotg210-udc-fix-ages-old-endianness-issues.patch b/next-5.10/usb-fotg210-udc-fix-ages-old-endianness-issues.patch similarity index 100% rename from queue-5.10/usb-fotg210-udc-fix-ages-old-endianness-issues.patch rename to next-5.10/usb-fotg210-udc-fix-ages-old-endianness-issues.patch diff --git a/queue-5.10/usb-gadget-f_hid-fix-f_hidg-lifetime-vs-cdev.patch b/next-5.10/usb-gadget-f_hid-fix-f_hidg-lifetime-vs-cdev.patch similarity index 100% rename from queue-5.10/usb-gadget-f_hid-fix-f_hidg-lifetime-vs-cdev.patch rename to next-5.10/usb-gadget-f_hid-fix-f_hidg-lifetime-vs-cdev.patch diff --git a/queue-5.10/usb-gadget-f_hid-fix-refcount-leak-on-error-path.patch b/next-5.10/usb-gadget-f_hid-fix-refcount-leak-on-error-path.patch similarity index 100% rename from queue-5.10/usb-gadget-f_hid-fix-refcount-leak-on-error-path.patch rename to next-5.10/usb-gadget-f_hid-fix-refcount-leak-on-error-path.patch diff --git a/queue-5.10/usb-gadget-f_hid-optional-setup-set_report-mode.patch b/next-5.10/usb-gadget-f_hid-optional-setup-set_report-mode.patch similarity index 100% rename from queue-5.10/usb-gadget-f_hid-optional-setup-set_report-mode.patch rename to next-5.10/usb-gadget-f_hid-optional-setup-set_report-mode.patch diff --git a/queue-5.10/usb-musb-remove-extra-check-in-musb_gadget_vbus_draw.patch b/next-5.10/usb-musb-remove-extra-check-in-musb_gadget_vbus_draw.patch similarity index 100% rename from queue-5.10/usb-musb-remove-extra-check-in-musb_gadget_vbus_draw.patch rename to next-5.10/usb-musb-remove-extra-check-in-musb_gadget_vbus_draw.patch diff --git a/queue-5.10/usb-roles-fix-of-node-refcount-leak-in-usb_role_swit.patch b/next-5.10/usb-roles-fix-of-node-refcount-leak-in-usb_role_swit.patch similarity index 100% rename from queue-5.10/usb-roles-fix-of-node-refcount-leak-in-usb_role_swit.patch rename to next-5.10/usb-roles-fix-of-node-refcount-leak-in-usb_role_swit.patch diff --git a/queue-5.10/usb-storage-add-check-for-kcalloc.patch b/next-5.10/usb-storage-add-check-for-kcalloc.patch similarity index 100% rename from queue-5.10/usb-storage-add-check-for-kcalloc.patch rename to next-5.10/usb-storage-add-check-for-kcalloc.patch diff --git a/queue-5.10/usb-typec-check-for-ops-exit-instead-of-ops-enter-in.patch b/next-5.10/usb-typec-check-for-ops-exit-instead-of-ops-enter-in.patch similarity index 100% rename from queue-5.10/usb-typec-check-for-ops-exit-instead-of-ops-enter-in.patch rename to next-5.10/usb-typec-check-for-ops-exit-instead-of-ops-enter-in.patch diff --git a/queue-5.10/usb-typec-tcpci-fix-of-node-refcount-leak-in-tcpci_r.patch b/next-5.10/usb-typec-tcpci-fix-of-node-refcount-leak-in-tcpci_r.patch similarity index 100% rename from queue-5.10/usb-typec-tcpci-fix-of-node-refcount-leak-in-tcpci_r.patch rename to next-5.10/usb-typec-tcpci-fix-of-node-refcount-leak-in-tcpci_r.patch diff --git a/queue-5.10/usb-typec-tipd-fix-spurious-fwnode_handle_put-in-err.patch b/next-5.10/usb-typec-tipd-fix-spurious-fwnode_handle_put-in-err.patch similarity index 100% rename from queue-5.10/usb-typec-tipd-fix-spurious-fwnode_handle_put-in-err.patch rename to next-5.10/usb-typec-tipd-fix-spurious-fwnode_handle_put-in-err.patch diff --git a/queue-5.10/venus-pm_helpers-fix-error-check-in-vcodec_domains_g.patch b/next-5.10/venus-pm_helpers-fix-error-check-in-vcodec_domains_g.patch similarity index 100% rename from queue-5.10/venus-pm_helpers-fix-error-check-in-vcodec_domains_g.patch rename to next-5.10/venus-pm_helpers-fix-error-check-in-vcodec_domains_g.patch diff --git a/queue-5.10/vfio-platform-do-not-pass-return-buffer-to-acpi-_rst.patch b/next-5.10/vfio-platform-do-not-pass-return-buffer-to-acpi-_rst.patch similarity index 100% rename from queue-5.10/vfio-platform-do-not-pass-return-buffer-to-acpi-_rst.patch rename to next-5.10/vfio-platform-do-not-pass-return-buffer-to-acpi-_rst.patch diff --git a/queue-5.10/video-hyperv_fb-avoid-taking-busy-spinlock-on-panic-.patch b/next-5.10/video-hyperv_fb-avoid-taking-busy-spinlock-on-panic-.patch similarity index 100% rename from queue-5.10/video-hyperv_fb-avoid-taking-busy-spinlock-on-panic-.patch rename to next-5.10/video-hyperv_fb-avoid-taking-busy-spinlock-on-panic-.patch diff --git a/queue-5.10/vme-fix-error-not-catched-in-fake_init.patch b/next-5.10/vme-fix-error-not-catched-in-fake_init.patch similarity index 100% rename from queue-5.10/vme-fix-error-not-catched-in-fake_init.patch rename to next-5.10/vme-fix-error-not-catched-in-fake_init.patch diff --git a/queue-5.10/wifi-ar5523-fix-use-after-free-on-ar5523_cmd-timed-o.patch b/next-5.10/wifi-ar5523-fix-use-after-free-on-ar5523_cmd-timed-o.patch similarity index 100% rename from queue-5.10/wifi-ar5523-fix-use-after-free-on-ar5523_cmd-timed-o.patch rename to next-5.10/wifi-ar5523-fix-use-after-free-on-ar5523_cmd-timed-o.patch diff --git a/queue-5.10/wifi-ath10k-fix-return-value-in-ath10k_pci_init.patch b/next-5.10/wifi-ath10k-fix-return-value-in-ath10k_pci_init.patch similarity index 100% rename from queue-5.10/wifi-ath10k-fix-return-value-in-ath10k_pci_init.patch rename to next-5.10/wifi-ath10k-fix-return-value-in-ath10k_pci_init.patch diff --git a/queue-5.10/wifi-ath9k-hif_usb-fix-memory-leak-of-urbs-in-ath9k_.patch b/next-5.10/wifi-ath9k-hif_usb-fix-memory-leak-of-urbs-in-ath9k_.patch similarity index 100% rename from queue-5.10/wifi-ath9k-hif_usb-fix-memory-leak-of-urbs-in-ath9k_.patch rename to next-5.10/wifi-ath9k-hif_usb-fix-memory-leak-of-urbs-in-ath9k_.patch diff --git a/queue-5.10/wifi-ath9k-hif_usb-fix-use-after-free-in-ath9k_hif_u.patch b/next-5.10/wifi-ath9k-hif_usb-fix-use-after-free-in-ath9k_hif_u.patch similarity index 100% rename from queue-5.10/wifi-ath9k-hif_usb-fix-use-after-free-in-ath9k_hif_u.patch rename to next-5.10/wifi-ath9k-hif_usb-fix-use-after-free-in-ath9k_hif_u.patch diff --git a/queue-5.10/wifi-ath9k-verify-the-expected-usb_endpoints-are-pre.patch b/next-5.10/wifi-ath9k-verify-the-expected-usb_endpoints-are-pre.patch similarity index 100% rename from queue-5.10/wifi-ath9k-verify-the-expected-usb_endpoints-are-pre.patch rename to next-5.10/wifi-ath9k-verify-the-expected-usb_endpoints-are-pre.patch diff --git a/queue-5.10/wifi-brcmfmac-fix-error-return-code-in-brcmf_sdio_do.patch b/next-5.10/wifi-brcmfmac-fix-error-return-code-in-brcmf_sdio_do.patch similarity index 100% rename from queue-5.10/wifi-brcmfmac-fix-error-return-code-in-brcmf_sdio_do.patch rename to next-5.10/wifi-brcmfmac-fix-error-return-code-in-brcmf_sdio_do.patch diff --git a/queue-5.10/wifi-brcmfmac-fix-potential-shift-out-of-bounds-in-b.patch b/next-5.10/wifi-brcmfmac-fix-potential-shift-out-of-bounds-in-b.patch similarity index 100% rename from queue-5.10/wifi-brcmfmac-fix-potential-shift-out-of-bounds-in-b.patch rename to next-5.10/wifi-brcmfmac-fix-potential-shift-out-of-bounds-in-b.patch diff --git a/queue-5.10/wifi-cfg80211-fix-not-unregister-reg_pdev-when-load_.patch b/next-5.10/wifi-cfg80211-fix-not-unregister-reg_pdev-when-load_.patch similarity index 100% rename from queue-5.10/wifi-cfg80211-fix-not-unregister-reg_pdev-when-load_.patch rename to next-5.10/wifi-cfg80211-fix-not-unregister-reg_pdev-when-load_.patch diff --git a/queue-5.10/wifi-iwlwifi-mvm-fix-double-free-on-tx-path.patch b/next-5.10/wifi-iwlwifi-mvm-fix-double-free-on-tx-path.patch similarity index 100% rename from queue-5.10/wifi-iwlwifi-mvm-fix-double-free-on-tx-path.patch rename to next-5.10/wifi-iwlwifi-mvm-fix-double-free-on-tx-path.patch diff --git a/queue-5.10/wifi-mac80211-fix-memory-leak-in-ieee80211_if_add.patch b/next-5.10/wifi-mac80211-fix-memory-leak-in-ieee80211_if_add.patch similarity index 100% rename from queue-5.10/wifi-mac80211-fix-memory-leak-in-ieee80211_if_add.patch rename to next-5.10/wifi-mac80211-fix-memory-leak-in-ieee80211_if_add.patch diff --git a/queue-5.10/wifi-mt76-fix-coverity-overrun-call-in-mt76_get_txpo.patch b/next-5.10/wifi-mt76-fix-coverity-overrun-call-in-mt76_get_txpo.patch similarity index 100% rename from queue-5.10/wifi-mt76-fix-coverity-overrun-call-in-mt76_get_txpo.patch rename to next-5.10/wifi-mt76-fix-coverity-overrun-call-in-mt76_get_txpo.patch diff --git a/queue-5.10/wifi-rsi-fix-handling-of-802.3-eapol-frames-sent-via.patch b/next-5.10/wifi-rsi-fix-handling-of-802.3-eapol-frames-sent-via.patch similarity index 100% rename from queue-5.10/wifi-rsi-fix-handling-of-802.3-eapol-frames-sent-via.patch rename to next-5.10/wifi-rsi-fix-handling-of-802.3-eapol-frames-sent-via.patch diff --git a/queue-5.10/wifi-rtl8xxxu-add-__packed-to-struct-rtl8723bu_c2h.patch b/next-5.10/wifi-rtl8xxxu-add-__packed-to-struct-rtl8723bu_c2h.patch similarity index 100% rename from queue-5.10/wifi-rtl8xxxu-add-__packed-to-struct-rtl8723bu_c2h.patch rename to next-5.10/wifi-rtl8xxxu-add-__packed-to-struct-rtl8723bu_c2h.patch diff --git a/queue-5.10/wifi-rtl8xxxu-fix-reading-the-vendor-of-combo-chips.patch b/next-5.10/wifi-rtl8xxxu-fix-reading-the-vendor-of-combo-chips.patch similarity index 100% rename from queue-5.10/wifi-rtl8xxxu-fix-reading-the-vendor-of-combo-chips.patch rename to next-5.10/wifi-rtl8xxxu-fix-reading-the-vendor-of-combo-chips.patch diff --git a/queue-5.10/wifi-rtl8xxxu-fix-the-channel-width-reporting.patch b/next-5.10/wifi-rtl8xxxu-fix-the-channel-width-reporting.patch similarity index 100% rename from queue-5.10/wifi-rtl8xxxu-fix-the-channel-width-reporting.patch rename to next-5.10/wifi-rtl8xxxu-fix-the-channel-width-reporting.patch diff --git a/queue-5.10/x86-hyperv-remove-unregister-syscore-call-from-hyper.patch b/next-5.10/x86-hyperv-remove-unregister-syscore-call-from-hyper.patch similarity index 100% rename from queue-5.10/x86-hyperv-remove-unregister-syscore-call-from-hyper.patch rename to next-5.10/x86-hyperv-remove-unregister-syscore-call-from-hyper.patch diff --git a/next-5.10/x86-process-setup-io_threads-more-like-normal-user-space-threads.patch b/next-5.10/x86-process-setup-io_threads-more-like-normal-user-space-threads.patch new file mode 100644 index 00000000000..7bac5de85ea --- /dev/null +++ b/next-5.10/x86-process-setup-io_threads-more-like-normal-user-space-threads.patch @@ -0,0 +1,94 @@ +From f757cc1efe0ec578f54dc857f1684dcd6541b464 Mon Sep 17 00:00:00 2001 +From: Stefan Metzmacher +Date: Wed, 5 May 2021 13:03:10 +0200 +Subject: x86/process: setup io_threads more like normal user space threads + +From: Stefan Metzmacher + +[ Upstream commit 50b7b6f29de3e18e9d6c09641256a0296361cfee ] + +As io_threads are fully set up USER threads it's clearer to separate the +code path from the KTHREAD logic. + +The only remaining difference to user space threads is that io_threads +never return to user space again. Instead they loop within the given +worker function. + +The fact that they never return to user space means they don't have an +user space thread stack. In order to indicate that to tools like gdb we +reset the stack and instruction pointers to 0. + +This allows gdb attach to user space processes using io-uring, which like +means that they have io_threads, without printing worrying message like +this: + + warning: Selected architecture i386:x86-64 is not compatible with reported target architecture i386 + + warning: Architecture rejected target-supplied description + +The output will be something like this: + + (gdb) info threads + Id Target Id Frame + * 1 LWP 4863 "io_uring-cp-for" syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 + 2 LWP 4864 "iou-mgr-4863" 0x0000000000000000 in ?? () + 3 LWP 4865 "iou-wrk-4863" 0x0000000000000000 in ?? () + (gdb) thread 3 + [Switching to thread 3 (LWP 4865)] + #0 0x0000000000000000 in ?? () + (gdb) bt + #0 0x0000000000000000 in ?? () + Backtrace stopped: Cannot access memory at address 0x0 + +Fixes: 4727dc20e042 ("arch: setup PF_IO_WORKER threads like PF_KTHREAD") +Link: https://lore.kernel.org/io-uring/044d0bad-6888-a211-e1d3-159a4aeed52d@polymtl.ca/T/#m1bbf5727e3d4e839603f6ec7ed79c7eebfba6267 +Signed-off-by: Stefan Metzmacher +cc: Linus Torvalds +cc: Jens Axboe +cc: Andy Lutomirski +cc: linux-kernel@vger.kernel.org +cc: io-uring@vger.kernel.org +cc: x86@kernel.org +Link: https://lore.kernel.org/r/20210505110310.237537-1-metze@samba.org +Reviewed-by: Thomas Gleixner +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/kernel/process.c | 19 ++++++++++++++++++- + 1 file changed, 18 insertions(+), 1 deletion(-) + +--- a/arch/x86/kernel/process.c ++++ b/arch/x86/kernel/process.c +@@ -162,7 +162,7 @@ int copy_thread(unsigned long clone_flag + #endif + + /* Kernel thread ? */ +- if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { ++ if (unlikely(p->flags & PF_KTHREAD)) { + memset(childregs, 0, sizeof(struct pt_regs)); + kthread_frame_init(frame, sp, arg); + return 0; +@@ -178,6 +178,23 @@ int copy_thread(unsigned long clone_flag + task_user_gs(p) = get_user_gs(current_pt_regs()); + #endif + ++ if (unlikely(p->flags & PF_IO_WORKER)) { ++ /* ++ * An IO thread is a user space thread, but it doesn't ++ * return to ret_after_fork(). ++ * ++ * In order to indicate that to tools like gdb, ++ * we reset the stack and instruction pointers. ++ * ++ * It does the same kernel frame setup to return to a kernel ++ * function that a kernel thread does. ++ */ ++ childregs->sp = 0; ++ childregs->ip = 0; ++ kthread_frame_init(frame, sp, arg); ++ return 0; ++ } ++ + /* Set a new TLS for the child thread? */ + if (clone_flags & CLONE_SETTLS) + ret = set_new_tls(p, tls); diff --git a/next-5.10/x86-wire-up-tif_notify_signal.patch b/next-5.10/x86-wire-up-tif_notify_signal.patch new file mode 100644 index 00000000000..15aa0ad1c61 --- /dev/null +++ b/next-5.10/x86-wire-up-tif_notify_signal.patch @@ -0,0 +1,40 @@ +From f8f5fdc79f0e480230c501bc5d744bc08defec6c Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Mon, 26 Oct 2020 14:32:29 -0600 +Subject: x86: Wire up TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit c8d5ed67936fddbe2ae845fc80397718006322d7 ] + +The generic entry code has support for TIF_NOTIFY_SIGNAL already. Just +provide the TIF bit. + +[ tglx: Adopted to other TIF changes in x86 ] + +Signed-off-by: Jens Axboe +Signed-off-by: Thomas Gleixner +Link: https://lore.kernel.org/r/20201026203230.386348-4-axboe@kernel.dk +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/include/asm/thread_info.h | 2 ++ + 1 file changed, 2 insertions(+) + +--- a/arch/x86/include/asm/thread_info.h ++++ b/arch/x86/include/asm/thread_info.h +@@ -93,6 +93,7 @@ struct thread_info { + #define TIF_NOTSC 16 /* TSC is not accessible in userland */ + #define TIF_IA32 17 /* IA32 compatibility process */ + #define TIF_SLD 18 /* Restore split lock detection on context switch */ ++#define TIF_NOTIFY_SIGNAL 19 /* signal notifications exist */ + #define TIF_MEMDIE 20 /* is terminating due to OOM killer */ + #define TIF_POLLING_NRFLAG 21 /* idle is polling for TIF_NEED_RESCHED */ + #define TIF_IO_BITMAP 22 /* uses I/O bitmap */ +@@ -121,6 +122,7 @@ struct thread_info { + #define _TIF_NOCPUID (1 << TIF_NOCPUID) + #define _TIF_NOTSC (1 << TIF_NOTSC) + #define _TIF_IA32 (1 << TIF_IA32) ++#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + #define _TIF_SLD (1 << TIF_SLD) + #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) + #define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) diff --git a/queue-5.10/x86-xen-fix-memory-leak-in-xen_init_lock_cpu.patch b/next-5.10/x86-xen-fix-memory-leak-in-xen_init_lock_cpu.patch similarity index 100% rename from queue-5.10/x86-xen-fix-memory-leak-in-xen_init_lock_cpu.patch rename to next-5.10/x86-xen-fix-memory-leak-in-xen_init_lock_cpu.patch diff --git a/queue-5.10/x86-xen-fix-memory-leak-in-xen_smp_intr_init-_pv.patch b/next-5.10/x86-xen-fix-memory-leak-in-xen_smp_intr_init-_pv.patch similarity index 100% rename from queue-5.10/x86-xen-fix-memory-leak-in-xen_smp_intr_init-_pv.patch rename to next-5.10/x86-xen-fix-memory-leak-in-xen_smp_intr_init-_pv.patch diff --git a/queue-5.10/xen-privcmd-fix-a-possible-warning-in-privcmd_ioctl_.patch b/next-5.10/xen-privcmd-fix-a-possible-warning-in-privcmd_ioctl_.patch similarity index 100% rename from queue-5.10/xen-privcmd-fix-a-possible-warning-in-privcmd_ioctl_.patch rename to next-5.10/xen-privcmd-fix-a-possible-warning-in-privcmd_ioctl_.patch diff --git a/queue-5.10/xprtrdma-fix-regbuf-data-not-freed-in-rpcrdma_req_cr.patch b/next-5.10/xprtrdma-fix-regbuf-data-not-freed-in-rpcrdma_req_cr.patch similarity index 100% rename from queue-5.10/xprtrdma-fix-regbuf-data-not-freed-in-rpcrdma_req_cr.patch rename to next-5.10/xprtrdma-fix-regbuf-data-not-freed-in-rpcrdma_req_cr.patch diff --git a/next-5.10/xtensa-add-support-for-tif_notify_signal.patch b/next-5.10/xtensa-add-support-for-tif_notify_signal.patch new file mode 100644 index 00000000000..851aa45e5fc --- /dev/null +++ b/next-5.10/xtensa-add-support-for-tif_notify_signal.patch @@ -0,0 +1,72 @@ +From 4a20887d142c23ea4f6e0dc1d7c39756ec71d327 Mon Sep 17 00:00:00 2001 +From: Jens Axboe +Date: Fri, 9 Oct 2020 15:56:07 -0600 +Subject: xtensa: add support for TIF_NOTIFY_SIGNAL + +From: Jens Axboe + +[ Upstream commit bec58f40d6c5372d812c93cc3947f3bc97440e57 ] + +Wire up TIF_NOTIFY_SIGNAL handling for xtensa. + +Thanks to Max Filippov for making the asm correct. + +Cc: linux-xtensa@linux-xtensa.org +Signed-off-by: Jens Axboe +Signed-off-by: Greg Kroah-Hartman +--- + arch/xtensa/include/asm/thread_info.h | 5 ++++- + arch/xtensa/kernel/entry.S | 4 ++-- + arch/xtensa/kernel/signal.c | 3 ++- + 3 files changed, 8 insertions(+), 4 deletions(-) + +--- a/arch/xtensa/include/asm/thread_info.h ++++ b/arch/xtensa/include/asm/thread_info.h +@@ -111,18 +111,21 @@ static inline struct thread_info *curren + #define TIF_NEED_RESCHED 2 /* rescheduling necessary */ + #define TIF_SINGLESTEP 3 /* restore singlestep on return to user mode */ + #define TIF_SYSCALL_TRACEPOINT 4 /* syscall tracepoint instrumentation */ +-#define TIF_MEMDIE 5 /* is terminating due to OOM killer */ ++#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ + #define TIF_RESTORE_SIGMASK 6 /* restore signal mask in do_signal() */ + #define TIF_NOTIFY_RESUME 7 /* callback before returning to user */ + #define TIF_DB_DISABLED 8 /* debug trap disabled for syscall */ + #define TIF_SYSCALL_AUDIT 9 /* syscall auditing active */ + #define TIF_SECCOMP 10 /* secure computing */ ++#define TIF_MEMDIE 11 /* is terminating due to OOM killer */ + + #define _TIF_SYSCALL_TRACE (1< /* Work to do on interrupt/exception return. */ --- a/arch/alpha/kernel/entry.S +++ b/arch/alpha/kernel/entry.S -@@ -546,7 +546,7 @@ $ret_success: +@@ -544,7 +544,7 @@ $ret_success: .align 4 .type work_pending, @function work_pending: diff --git a/queue-5.10/series b/queue-5.10/series index ba4de0dadb8..8d082643e49 100644 --- a/queue-5.10/series +++ b/queue-5.10/series @@ -1,603 +1,3 @@ -usb-musb-remove-extra-check-in-musb_gadget_vbus_draw.patch -arm64-dts-qcom-ipq6018-cp01-c1-use-blspi1-pins.patch -arm64-dts-qcom-msm8996-fix-gpu-opp-table.patch -arm-dts-qcom-apq8064-fix-coresight-compatible.patch -arm64-dts-qcom-sdm630-fix-uart1-pin-bias.patch -arm64-dts-qcom-sdm845-cheza-fix-ap-suspend-pin-bias.patch -arm64-dts-qcom-msm8916-drop-mss-fallback-compatible.patch -objtool-kcsan-add-volatile-read-write-instrumentatio.patch -arm-dts-stm32-drop-stm32mp15xc.dtsi-from-avenger96.patch -arm-dts-stm32-fix-av96-wlan-regulator-gpio-property.patch -drivers-soc-ti-knav_qmss_queue-mark-knav_acc_firmwar.patch -soc-qcom-llcc-make-irq-truly-optional.patch -soc-qcom-apr-make-code-more-reuseable.patch -soc-qcom-apr-add-check-for-idr_alloc-and-of_property.patch -arm-dts-spear600-fix-clcd-interrupt.patch -soc-ti-knav_qmss_queue-use-pm_runtime_resume_and_get.patch -soc-ti-knav_qmss_queue-fix-pm-disable-depth-imbalanc.patch -soc-ti-smartreflex-fix-pm-disable-depth-imbalance-in.patch -perf-arm_dsu-fix-hotplug-callback-leak-in-dsu_pmu_in.patch -perf-smmuv3-fix-hotplug-callback-leak-in-arm_smmu_pm.patch -arm64-dts-ti-k3-am65-main-drop-dma-coherent-in-crypt.patch -arm64-dts-ti-k3-j721e-main-drop-dma-coherent-in-cryp.patch -arm64-dts-mt2712e-fix-unit_address_vs_reg-warning-fo.patch -arm64-dts-mt2712e-fix-unit-address-for-pinctrl-node.patch -arm64-dts-mt2712-evb-fix-vproc-fixed-regulators-unit.patch -arm64-dts-mt2712-evb-fix-usb-vbus-regulators-unit-na.patch -arm64-dts-mediatek-pumpkin-common-fix-devicetree-war.patch -arm64-dts-mediatek-mt6797-fix-26m-oscillator-unit-na.patch -arm-dts-dove-fix-assigned-addresses-for-every-pcie-r.patch -arm-dts-armada-370-fix-assigned-addresses-for-every-.patch -arm-dts-armada-xp-fix-assigned-addresses-for-every-p.patch -arm-dts-armada-375-fix-assigned-addresses-for-every-.patch -arm-dts-armada-38x-fix-assigned-addresses-for-every-.patch -arm-dts-armada-39x-fix-assigned-addresses-for-every-.patch -arm-dts-turris-omnia-add-ethernet-aliases.patch -arm-dts-turris-omnia-add-switch-port-6-node.patch -arm64-dts-armada-3720-turris-mox-add-missing-interru.patch -pstore-ram-fix-error-return-code-in-ramoops_probe.patch -arm-mmp-fix-timer_read-delay.patch -pstore-avoid-kcore-oops-by-vmap-ing-with-vm_ioremap.patch -tpm-tpm_ftpm_tee-fix-error-handling-in-ftpm_mod_init.patch -tpm-tpm_crb-fix-error-message-in-__crb_relinquish_lo.patch -sched-fair-cleanup-task_util-and-capacity-type.patch -sched-uclamp-fix-relationship-between-uclamp-and-mig.patch -cpuidle-dt-return-the-correct-numbers-of-parsed-idle.patch -alpha-fix-syscall-entry-in-audut_syscall-case.patch -pm-hibernate-fix-mistake-in-kerneldoc-comment.patch -fs-don-t-audit-the-capability-check-in-simple_xattr_.patch -cpufreq-qcom-hw-fix-memory-leak-in-qcom_cpufreq_hw_r.patch -selftests-ftrace-event_triggers-wait-longer-for-test.patch -perf-fix-possible-memleak-in-pmu_dev_alloc.patch -lib-debugobjects-fix-stat-count-and-optimize-debug_o.patch -platform-x86-huawei-wmi-fix-return-value-calculation.patch -timerqueue-use-rb_entry_safe-in-timerqueue_getnext.patch -proc-fixup-uptime-selftest.patch -lib-fonts-fix-undefined-behavior-in-bit-shift-for-ge.patch -ocfs2-fix-memory-leak-in-ocfs2_stack_glue_init.patch -mips-vpe-mt-fix-possible-memory-leak-while-module-ex.patch -mips-vpe-cmp-fix-possible-memory-leak-while-module-e.patch -selftests-efivarfs-add-checking-of-the-test-return-v.patch -pnp-fix-name-memory-leak-in-pnp_alloc_dev.patch -perf-x86-intel-uncore-fix-reference-count-leak-in-hs.patch -perf-x86-intel-uncore-fix-reference-count-leak-in-sn.patch -perf-x86-intel-uncore-fix-reference-count-leak-in-__.patch -platform-chrome-cros_usbpd_notify-fix-error-handling.patch -irqchip-gic-pm-use-pm_runtime_resume_and_get-in-gic_.patch -edac-i10nm-fix-refcount-leak-in-pci_get_dev_wrapper.patch -nfsd-don-t-call-nfsd_file_put-from-client-states-seq.patch -genirq-irqdesc-don-t-try-to-remove-non-existing-sysf.patch -cpufreq-amd_freq_sensitivity-add-missing-pci_dev_put.patch -libfs-add-define_simple_attribute_signed-for-signed-.patch -lib-notifier-error-inject-fix-error-when-writing-err.patch -docs-fault-injection-fix-non-working-usage-of-negati.patch -debugfs-fix-error-when-writing-negative-value-to-ato.patch -ocfs2-ocfs2_mount_volume-does-cleanup-job-before-ret.patch -ocfs2-rewrite-error-handling-of-ocfs2_fill_super.patch -ocfs2-fix-memory-leak-in-ocfs2_mount_volume.patch -rapidio-fix-possible-name-leaks-when-rio_add_device-.patch -rapidio-rio-fix-possible-name-leak-in-rio_register_m.patch -clocksource-drivers-sh_cmt-make-sure-channel-clock-s.patch -clocksource-drivers-sh_cmt-access-registers-accordin.patch -futex-move-to-kernel-futex.patch -futex-resend-potentially-swallowed-owner-death-notif.patch -cpu-hotplug-make-target_store-a-nop-when-target-stat.patch -clocksource-drivers-timer-ti-dm-fix-missing-clk_disa.patch -acpica-fix-use-after-free-in-acpi_ut_copy_ipackage_t.patch -uprobes-x86-allow-to-probe-a-nop-instruction-with-0x.patch -x86-xen-fix-memory-leak-in-xen_smp_intr_init-_pv.patch -x86-xen-fix-memory-leak-in-xen_init_lock_cpu.patch -xen-privcmd-fix-a-possible-warning-in-privcmd_ioctl_.patch -pm-runtime-improve-path-in-rpm_idle-when-no-callback.patch -pm-runtime-do-not-call-__rpm_callback-from-rpm_idle.patch -platform-x86-mxm-wmi-fix-memleak-in-mxm_wmi_call_mx-.patch -platform-x86-intel_scu_ipc-fix-possible-name-leak-in.patch -mips-bcm63xx-add-check-for-null-for-clk-in-clk_enabl.patch -mips-octeon-warn-only-once-if-deprecated-link-status.patch -fs-sysv-fix-sysv_nblocks-returns-wrong-value.patch -rapidio-fix-possible-uaf-when-kfifo_alloc-fails.patch -eventfd-change-int-to-__u64-in-eventfd_signal-ifndef.patch -relay-fix-type-mismatch-when-allocating-memory-in-re.patch -hfs-fix-oob-write-in-hfs_asc2mac.patch -rapidio-devices-fix-missing-put_device-in-mport_cdev.patch -wifi-ath9k-hif_usb-fix-memory-leak-of-urbs-in-ath9k_.patch -wifi-ath9k-hif_usb-fix-use-after-free-in-ath9k_hif_u.patch -wifi-rtl8xxxu-fix-reading-the-vendor-of-combo-chips.patch -drm-bridge-adv7533-remove-dynamic-lane-switching-fro.patch -libbpf-fix-use-after-free-in-btf_dump_name_dups.patch -libbpf-fix-null-pointer-dereference-in-find_prog_by_.patch -pata_ipx4xx_cf-fix-unsigned-comparison-with-less-tha.patch -media-coda-jpeg-add-check-for-kmalloc.patch -media-i2c-ad5820-fix-error-path.patch -venus-pm_helpers-fix-error-check-in-vcodec_domains_g.patch -media-exynos4-is-use-v4l2_async_notifier_add_fwnode_.patch -media-exynos4-is-don-t-rely-on-the-v4l2_async_subdev.patch -can-kvaser_usb-do-not-increase-tx-statistics-when-se.patch -can-kvaser_usb-kvaser_usb_leaf-get-capabilities-from.patch -can-kvaser_usb-kvaser_usb_leaf-rename-leaf-usbcan-_c.patch -can-kvaser_usb-kvaser_usb_leaf-handle-cmd_error_even.patch -can-kvaser_usb_leaf-set-warning-state-even-without-b.patch -can-kvaser_usb_leaf-fix-improved-state-not-being-rep.patch -can-kvaser_usb_leaf-fix-wrong-can-state-after-stoppi.patch -can-kvaser_usb_leaf-fix-bogus-restart-events.patch -can-kvaser_usb-add-struct-kvaser_usb_busparams.patch -can-kvaser_usb-compare-requested-bittiming-parameter.patch -drm-rockchip-lvds-fix-pm-usage-counter-unbalance-in-.patch -clk-renesas-r9a06g032-repair-grave-increment-error.patch -spi-update-reference-to-struct-spi_controller.patch -drm-panel-panel-sitronix-st7701-remove-panel-on-dsi-.patch -ima-fix-fall-through-warnings-for-clang.patch -ima-handle-estale-returned-by-ima_filter_rule_match.patch -drm-msm-hdmi-switch-to-drm_bridge_connector.patch -drm-msm-hdmi-drop-unused-gpio-support.patch -bpf-fix-slot-type-check-in-check_stack_write_var_off.patch -media-vivid-fix-compose-size-exceed-boundary.patch -media-platform-exynos4-is-fix-return-value-check-in-.patch -bpf-propagate-precision-in-alu-alu64-operations.patch -bpf-check-the-other-end-of-slot_type-for-stack_spill.patch -bpf-propagate-precision-across-all-frames-not-just-t.patch -clk-qcom-gcc-sm8250-use-retention-mode-for-usb-gdscs.patch -mtd-fix-device-name-leak-when-register-device-failed.patch -input-joystick-fix-kconfig-warning-for-joystick_adc.patch -wifi-rsi-fix-handling-of-802.3-eapol-frames-sent-via.patch -media-camss-clean-up-received-buffers-on-failed-star.patch -net-proc-provide-proc_fs-n-fallback-for-proc_create_.patch -rxrpc-fix-ack.buffersize-to-be-0-when-generating-an-.patch -drm-radeon-add-the-missed-acpi_put_table-to-fix-memo.patch -drm-mediatek-modify-dpi-power-on-off-sequence.patch -asoc-pxa-fix-null-pointer-dereference-in-filter.patch -regulator-core-fix-unbalanced-of-node-refcount-in-re.patch -amdgpu-pm-prevent-array-underflow-in-vega20_odn_edit.patch -drm-fourcc-add-packed-10bit-yuv-4-2-0-format.patch -drm-fourcc-fix-vsub-hsub-for-q410-and-q401.patch -integrity-fix-memory-leakage-in-keyring-allocation-e.patch -ima-fix-misuse-of-dereference-of-pointer-in-template.patch -wifi-ath10k-fix-return-value-in-ath10k_pci_init.patch -mtd-lpddr2_nvm-fix-possible-null-ptr-deref.patch -input-elants_i2c-properly-handle-the-reset-gpio-when.patch -media-vidtv-fix-use-after-free-in-vidtv_bridge_dvb_i.patch -media-solo6x10-fix-possible-memory-leak-in-solo_sysf.patch -media-platform-exynos4-is-fix-error-handling-in-fimc.patch -media-videobuf-dma-contig-use-dma_mmap_coherent.patch -inet-add-read_once-sk-sk_bound_dev_if-in-inet_csk_bi.patch -bpf-move-skb-len-0-checks-into-__bpf_redirect.patch -hid-hid-sensor-custom-set-fixed-size-for-custom-attr.patch -alsa-pcm-fix-undefined-behavior-in-bit-shift-for-snd.patch -alsa-seq-fix-undefined-behavior-in-bit-shift-for-snd.patch -regulator-core-use-kfree_const-to-free-space-conditi.patch -clk-rockchip-fix-memory-leak-in-rockchip_clk_registe.patch -drm-amdgpu-fix-pci-device-refcount-leak.patch -bonding-fix-link-recovery-in-mode-2-when-updelay-is-.patch -mtd-maps-pxa2xx-flash-fix-memory-leak-in-probe.patch -drbd-fix-an-invalid-memory-access-caused-by-incorrec.patch -asoc-qcom-add-checks-for-devm_kcalloc.patch -media-vimc-fix-wrong-function-called-when-vimc_init-.patch -media-imon-fix-a-race-condition-in-send_packet.patch -clk-imx-replace-osc_hdmi-with-dummy.patch -pinctrl-pinconf-generic-add-missing-of_node_put.patch -media-dvb-core-fix-ignored-return-value-in-dvb_regis.patch -media-dvb-usb-az6027-fix-null-ptr-deref-in-az6027_i2.patch -media-s5p-mfc-add-variant-data-for-mfc-v7-hardware-f.patch -drm-tegra-add-missing-clk_disable_unprepare-in-tegra.patch -asoc-dt-bindings-wcd9335-fix-reset-line-polarity-in-.patch -asoc-mediatek-mtk-btcvsd-add-checks-for-write-and-re.patch -nfsv4.2-clear-fattr4_word2_security_label-when-done-.patch -nfsv4.2-fix-a-memory-stomp-in-decode_attr_security_l.patch -nfsv4.2-fix-initialisation-of-struct-nfs4_label.patch -nfsv4-fix-a-deadlock-between-nfs4_open_recover_helpe.patch -nfs-fix-an-oops-in-nfs_d_automount.patch -alsa-asihpi-fix-missing-pci_disable_device.patch -wifi-iwlwifi-mvm-fix-double-free-on-tx-path.patch -asoc-mediatek-mt8173-fix-debugfs-registration-for-co.patch -asoc-mediatek-mt8173-enable-irq-when-pdata-is-ready.patch -drm-amd-pm-smu11-baco-is-supported-when-it-s-in-baco.patch -drm-radeon-fix-pci-device-refcount-leak-in-radeon_at.patch -drm-amdgpu-fix-pci-device-refcount-leak-in-amdgpu_at.patch -asoc-pcm512x-fix-pm-disable-depth-imbalance-in-pcm51.patch -netfilter-conntrack-set-icmpv6-redirects-as-related.patch -bpf-sockmap-fix-repeated-calls-to-sock_put-when-msg-.patch -bpf-sockmap-fix-data-loss-caused-by-using-apply_byte.patch -bonding-uninitialized-variable-in-bond_miimon_inspec.patch -spi-spidev-mask-spi_cs_high-in-spi_ioc_rd_mode.patch -wifi-mac80211-fix-memory-leak-in-ieee80211_if_add.patch -wifi-cfg80211-fix-not-unregister-reg_pdev-when-load_.patch -wifi-mt76-fix-coverity-overrun-call-in-mt76_get_txpo.patch -regulator-core-fix-module-refcount-leak-in-set_suppl.patch -clk-qcom-clk-krait-fix-wrong-div2-functions.patch -hsr-add-a-rcu-read-lock-to-hsr_forward_skb.patch -net-hsr-generate-supervision-frame-without-hsr-prp-t.patch -hsr-disable-netpoll.patch -hsr-synchronize-sending-frames-to-have-always-increm.patch -hsr-synchronize-sequence-number-updates.patch -configfs-fix-possible-memory-leak-in-configfs_create.patch -regulator-core-fix-resource-leak-in-regulator_regist.patch -hwmon-jc42-convert-register-access-and-caching-to-re.patch -hwmon-jc42-restore-the-min-max-critical-temperatures.patch -bpf-sockmap-fix-race-in-sock_map_free.patch -alsa-pcm-set-missing-stop_operating-flag-at-undoing-.patch -media-saa7164-fix-missing-pci_disable_device.patch -alsa-mts64-fix-possible-null-ptr-defer-in-snd_mts64_.patch -xprtrdma-fix-regbuf-data-not-freed-in-rpcrdma_req_cr.patch -sunrpc-fix-missing-release-socket-in-rpc_sockname.patch -nfsv4.x-fail-client-initialisation-if-state-manager-.patch -mmc-alcor-fix-return-value-check-of-mmc_add_host.patch -mmc-moxart-fix-return-value-check-of-mmc_add_host.patch -mmc-mxcmmc-fix-return-value-check-of-mmc_add_host.patch -mmc-pxamci-fix-return-value-check-of-mmc_add_host.patch -mmc-rtsx_usb_sdmmc-fix-return-value-check-of-mmc_add.patch -mmc-toshsd-fix-return-value-check-of-mmc_add_host.patch -mmc-vub300-fix-return-value-check-of-mmc_add_host.patch -mmc-wmt-sdmmc-fix-return-value-check-of-mmc_add_host.patch -mmc-atmel-mci-fix-return-value-check-of-mmc_add_host.patch -mmc-omap_hsmmc-fix-return-value-check-of-mmc_add_hos.patch -mmc-meson-gx-fix-return-value-check-of-mmc_add_host.patch -mmc-via-sdmmc-fix-return-value-check-of-mmc_add_host.patch -mmc-wbsd-fix-return-value-check-of-mmc_add_host.patch -mmc-mmci-fix-return-value-check-of-mmc_add_host.patch -media-c8sectpfe-add-of_node_put-when-breaking-out-of.patch -media-coda-add-check-for-dcoda_iram_alloc.patch -media-coda-add-check-for-kmalloc.patch -clk-samsung-fix-memory-leak-in-_samsung_clk_register.patch -spi-spi-gpio-don-t-set-mosi-as-an-input-if-not-3wire.patch -wifi-rtl8xxxu-add-__packed-to-struct-rtl8723bu_c2h.patch -wifi-rtl8xxxu-fix-the-channel-width-reporting.patch -wifi-brcmfmac-fix-error-return-code-in-brcmf_sdio_do.patch -blktrace-fix-output-non-blktrace-event-when-blk_clas.patch -clk-socfpga-clk-pll-remove-unused-variable-rc.patch -clk-socfpga-use-clk_hw_register-for-a5-c5.patch -clk-socfpga-fix-memory-leak-in-socfpga_gate_init.patch -net-vmw_vsock-vmci-check-memcpy_from_msg.patch -net-defxx-fix-missing-err-handling-in-dfx_init.patch -net-stmmac-selftests-fix-potential-memleak-in-stmmac.patch -drivers-net-qlcnic-fix-potential-memory-leak-in-qlcn.patch -of-overlay-fix-null-pointer-dereferencing-in-find_du.patch -ethernet-s2io-don-t-call-dev_kfree_skb-under-spin_lo.patch -net-farsync-fix-kmemleak-when-rmmods-farsync.patch -net-tunnel-wait-until-all-sk_user_data-reader-finish.patch -net-apple-mace-don-t-call-dev_kfree_skb-under-spin_l.patch -net-apple-bmac-don-t-call-dev_kfree_skb-under-spin_l.patch -net-emaclite-don-t-call-dev_kfree_skb-under-spin_loc.patch -net-ethernet-dnet-don-t-call-dev_kfree_skb-under-spi.patch -hamradio-don-t-call-dev_kfree_skb-under-spin_lock_ir.patch -net-amd-lance-don-t-call-dev_kfree_skb-under-spin_lo.patch -net-amd-xgbe-fix-logic-around-active-and-passive-cab.patch -net-amd-xgbe-check-only-the-minimum-speed-for-active.patch -can-tcan4x5x-remove-invalid-write-in-clear_interrupt.patch -net-lan9303-fix-read-error-execution-path.patch -ntb_netdev-use-dev_kfree_skb_any-in-interrupt-contex.patch -sctp-sysctl-make-extra-pointers-netns-aware.patch -bluetooth-btusb-don-t-call-kfree_skb-under-spin_lock.patch -bluetooth-hci_qca-don-t-call-kfree_skb-under-spin_lo.patch -bluetooth-hci_ll-don-t-call-kfree_skb-under-spin_loc.patch -bluetooth-hci_h5-don-t-call-kfree_skb-under-spin_loc.patch -bluetooth-hci_bcsp-don-t-call-kfree_skb-under-spin_l.patch -bluetooth-hci_core-don-t-call-kfree_skb-under-spin_l.patch -bluetooth-rfcomm-don-t-call-kfree_skb-under-spin_loc.patch -stmmac-fix-potential-division-by-0.patch -apparmor-fix-a-memleak-in-multi_transaction_new.patch -apparmor-fix-lockdep-warning-when-removing-a-namespa.patch -apparmor-fix-abi-check-to-include-v8-abi.patch -crypto-sun8i-ss-use-dma_addr-instead-u32.patch -crypto-nitrox-avoid-double-free-on-error-path-in-nit.patch -scsi-core-fix-a-race-between-scsi_done-and-scsi_time.patch -apparmor-use-pointer-to-struct-aa_label-for-lbs_cred.patch -pci-dwc-fix-n_fts-array-overrun.patch -rdma-core-fix-order-of-nldev_exit-call.patch -pci-pci-epf-test-register-notifier-if-only-core_init.patch -f2fs-fix-the-race-condition-of-resize-flag-between-r.patch -crypto-rockchip-do-not-do-custom-power-management.patch -crypto-rockchip-do-not-store-mode-globally.patch -crypto-rockchip-add-fallback-for-cipher.patch -crypto-rockchip-add-fallback-for-ahash.patch -crypto-rockchip-better-handle-cipher-key.patch -crypto-rockchip-remove-non-aligned-handling.patch -crypto-rockchip-delete-unneeded-variable-initializat.patch -crypto-rockchip-rework-by-using-crypto_engine.patch -apparmor-fix-memleak-in-alloc_ns.patch -f2fs-fix-normal-discard-process.patch -rdma-siw-fix-immediate-work-request-flush-to-complet.patch -rdma-nldev-return-eagain-if-the-cm_id-isn-t-from-exp.patch -rdma-siw-set-defined-status-for-work-completion-with.patch -scsi-scsi_debug-fix-a-warning-in-resp_write_scat.patch -crypto-ccree-remove-debugfs-when-platform_driver_reg.patch -crypto-cryptd-use-request-context-instead-of-stack-f.patch -crypto-hisilicon-qm-add-missing-pci_dev_put-in-q_num.patch -rdma-hns-repacing-dseg_len-by-macros-in-fill_ext_sge.patch -rdma-hns-fix-ext_sge-num-error-when-post-send.patch -pci-check-for-alloc-failure-in-pci_request_irq.patch -rdma-hfi-decrease-pci-device-reference-count-in-erro.patch -crypto-ccree-make-cc_debugfs_global_fini-available-f.patch -rdma-hns-fix-memory-leak-in-hns_roce_alloc_mr.patch -rdma-rxe-fix-null-ptr-deref-in-rxe_qp_do_cleanup-whe.patch -scsi-hpsa-fix-possible-memory-leak-in-hpsa_init_one.patch -crypto-tcrypt-fix-multibuffer-skcipher-speed-test-me.patch -padata-always-leave-bhs-disabled-when-running-parall.patch -padata-fix-list-iterator-in-padata_do_serial.patch -scsi-mpt3sas-fix-possible-resource-leaks-in-mpt3sas_.patch -scsi-hpsa-fix-error-handling-in-hpsa_add_sas_host.patch -scsi-hpsa-fix-possible-memory-leak-in-hpsa_add_sas_d.patch -scsi-scsi_debug-fix-a-warning-in-resp_verify.patch -scsi-scsi_debug-fix-a-warning-in-resp_report_zones.patch -scsi-fcoe-fix-possible-name-leak-when-device_registe.patch -scsi-scsi_debug-fix-possible-name-leak-in-sdebug_add.patch -scsi-ipr-fix-warning-in-ipr_init.patch -scsi-fcoe-fix-transport-not-deattached-when-fcoe_if_.patch -scsi-snic-fix-possible-uaf-in-snic_tgt_create.patch -rdma-nldev-add-checks-for-nla_nest_start-in-fill_sta.patch -f2fs-avoid-victim-selection-from-previous-victim-sec.patch -rdma-nldev-fix-failure-to-send-large-messages.patch -crypto-amlogic-remove-kcalloc-without-check.patch -crypto-omap-sham-use-pm_runtime_resume_and_get-in-om.patch -riscv-mm-add-arch-hook-arch_clear_hugepage_flags.patch -rdma-hfi1-fix-error-return-code-in-parse_platform_co.patch -rdma-srp-fix-error-return-code-in-srp_parse_options.patch -orangefs-fix-sysfs-not-cleanup-when-dev-init-failed.patch -rdma-hns-fix-pbl-page-mtr-find.patch -rdma-hns-fix-page-size-cap-from-firmware.patch -crypto-img-hash-fix-variable-dereferenced-before-che.patch -hwrng-amd-fix-pci-device-refcount-leak.patch -hwrng-geode-fix-pci-device-refcount-leak.patch -ib-ipoib-fix-queue-count-inconsistency-for-pkey-chil.patch -drivers-dio-fix-possible-memory-leak-in-dio_init.patch -serial-tegra-read-dma-status-before-terminating.patch -class-fix-possible-memory-leak-in-__class_register.patch -vfio-platform-do-not-pass-return-buffer-to-acpi-_rst.patch -uio-uio_dmem_genirq-fix-missing-unlock-in-irq-config.patch -uio-uio_dmem_genirq-fix-deadlock-between-irq-config-.patch -usb-fotg210-udc-fix-ages-old-endianness-issues.patch -staging-vme_user-fix-possible-uaf-in-tsi148_dma_list.patch -usb-typec-check-for-ops-exit-instead-of-ops-enter-in.patch -usb-typec-tcpci-fix-of-node-refcount-leak-in-tcpci_r.patch -usb-typec-tipd-fix-spurious-fwnode_handle_put-in-err.patch -serial-amba-pl011-avoid-sbsa-uart-accessing-dmacr-re.patch -serial-pl011-do-not-clear-rx-fifo-rx-interrupt-in-un.patch -serial-pch-fix-pci-device-refcount-leak-in-pch_reque.patch -tty-serial-clean-up-stop-tx-part-in-altera_uart_tx_c.patch -tty-serial-altera_uart_-r-t-x_chars-need-only-uart_p.patch -serial-altera_uart-fix-locking-in-polling-mode.patch -serial-sunsab-fix-error-handling-in-sunsab_init.patch -test_firmware-fix-memory-leak-in-test_firmware_init.patch -misc-ocxl-fix-possible-name-leak-in-ocxl_file_regist.patch -ocxl-fix-pci-device-refcount-leak-when-calling-get_f.patch -misc-tifm-fix-possible-memory-leak-in-tifm_7xx1_swit.patch -misc-sgi-gru-fix-use-after-free-error-in-gru_set_con.patch -firmware-raspberrypi-fix-possible-memory-leak-in-rpi.patch -cxl-fix-possible-null-ptr-deref-in-cxl_guest_init_af.patch -cxl-fix-possible-null-ptr-deref-in-cxl_pci_init_afu-.patch -iio-temperature-ltc2983-make-bulk-write-buffer-dma-s.patch -genirq-add-irqf_no_autoen-for-request_irq-nmi.patch -iio-imu-adis-use-irqf_no_autoen-instead-of-irq-reque.patch -iio-adis-handle-devices-that-cannot-unmask-the-drdy-.patch -iio-adis-stylistic-changes.patch -iio-imu-adis-move-exports-into-iio_adislib-namespace.patch -iio-adis-add-__adis_enable_irq-implementation.patch -counter-stm32-lptimer-cnt-fix-the-check-on-arr-and-c.patch -usb-roles-fix-of-node-refcount-leak-in-usb_role_swit.patch -usb-gadget-f_hid-optional-setup-set_report-mode.patch -usb-gadget-f_hid-fix-f_hidg-lifetime-vs-cdev.patch -usb-gadget-f_hid-fix-refcount-leak-on-error-path.patch -drivers-mcb-fix-resource-leak-in-mcb_probe.patch -mcb-mcb-parse-fix-error-handing-in-chameleon_parse_g.patch -chardev-fix-error-handling-in-cdev_device_add.patch -i2c-pxa-pci-fix-missing-pci_disable_device-on-error-.patch -staging-rtl8192u-fix-use-after-free-in-ieee80211_rx.patch -staging-rtl8192e-fix-potential-use-after-free-in-rtl.patch -vme-fix-error-not-catched-in-fake_init.patch -gpiolib-get-rid-of-redundant-else.patch -gpiolib-cdev-fix-null-pointer-dereferences.patch -i2c-mux-reg-check-return-value-after-calling-platfor.patch -i2c-ismt-fix-an-out-of-bounds-bug-in-ismt_access.patch -usb-storage-add-check-for-kcalloc.patch -tracing-hist-fix-issue-of-losting-command-info-in-er.patch -samples-vfio-mdev-fix-missing-pci_disable_device-in-.patch -thermal-drivers-imx8mm_thermal-validate-temperature-.patch -fbdev-ssd1307fb-drop-optional-dependency.patch -fbdev-pm2fb-fix-missing-pci_disable_device.patch -fbdev-via-fix-error-in-via_core_init.patch -fbdev-vermilion-decrease-reference-count-in-error-pa.patch -fbdev-uvesafb-fixes-an-error-handling-path-in-uvesaf.patch -hsi-omap_ssi_core-fix-unbalanced-pm_runtime_disable.patch -hsi-omap_ssi_core-fix-possible-memory-leak-in-ssi_pr.patch -power-supply-fix-residue-sysfs-file-in-error-handle-.patch -perf-trace-return-error-if-a-system-call-doesn-t-exi.patch -perf-trace-use-macro-raw_syscall_args_num-to-replace.patch -perf-trace-handle-failure-when-trace-point-folder-is.patch -perf-symbol-correction-while-adjusting-symbol.patch -hsi-omap_ssi_core-fix-error-handling-in-ssi_init.patch -power-supply-fix-null-pointer-dereferencing-in-power.patch -rdma-siw-fix-pointer-cast-warning.patch -iommu-sun50i-fix-reset-release.patch -iommu-sun50i-consider-all-fault-sources-for-reset.patch -iommu-sun50i-fix-r-w-permission-check.patch -iommu-sun50i-fix-flush-size.patch -phy-usb-s2-wol-wakeup_count-not-incremented-for-usb-.patch -include-uapi-linux-swab-fix-potentially-missing-__al.patch -pwm-tegra-improve-required-rate-calculation.patch -dmaengine-idxd-fix-crc_val-field-for-completion-reco.patch -rtc-rtc-cmos-do-not-check-acpi_fadt_low_power_s0.patch -rtc-cmos-fix-event-handler-registration-ordering-iss.patch -rtc-cmos-fix-wake-alarm-breakage.patch -rtc-cmos-fix-build-on-non-acpi-platforms.patch -rtc-cmos-call-cmos_wake_setup-from-cmos_do_probe.patch -rtc-cmos-call-rtc_wake_setup-from-cmos_do_probe.patch -rtc-cmos-eliminate-forward-declarations-of-some-func.patch -rtc-cmos-rename-acpi-related-functions.patch -rtc-cmos-disable-acpi-rtc-event-on-removal.patch -rtc-snvs-allow-a-time-difference-on-clock-register-r.patch -rtc-pcf85063-fix-reading-alarm.patch -iommu-amd-fix-pci-device-refcount-leak-in-ppr_notifi.patch -iommu-fsl_pamu-fix-resource-leak-in-fsl_pamu_probe.patch -macintosh-fix-possible-memory-leak-in-macio_add_one_.patch -macintosh-macio-adb-check-the-return-value-of-iorema.patch -powerpc-52xx-fix-a-resource-leak-in-an-error-handlin.patch -cxl-fix-refcount-leak-in-cxl_calc_capp_routing.patch -powerpc-xmon-enable-breakpoints-on-8xx.patch -powerpc-xmon-fix-wswitch-unreachable-warning-in-bpt_.patch -powerpc-xive-add-missing-iounmap-in-error-path-in-xi.patch -kbuild-remove-unneeded-mkdir-for-external-modules_in.patch -kbuild-unify-modules-_install-for-in-tree-and-extern.patch -kbuild-refactor-single-builds-of-.ko.patch -powerpc-perf-callchain-validate-kernel-stack-pointer.patch -powerpc-83xx-mpc832x_rdb-call-platform_device_put-in.patch -powerpc-hv-gpci-fix-hv_gpci-event-list.patch -selftests-powerpc-fix-resource-leaks.patch -iommu-sun50i-remove-iommu_domain_identity.patch -pwm-sifive-call-pwm_sifive_update_clock-while-mutex-.patch -remoteproc-sysmon-fix-memory-leak-in-qcom_add_sysmon.patch -remoteproc-qcom_q6v5_pas-disable-wakeup-on-probe-fai.patch -remoteproc-qcom_q6v5_pas-detach-power-domains-on-rem.patch -remoteproc-qcom_q6v5_pas-fix-missing-of_node_put-in-.patch -powerpc-eeh-drop-redundant-spinlock-initialization.patch -powerpc-pseries-eeh-use-correct-api-for-error-log-si.patch -netfilter-flowtable-really-fix-nat-ipv6-offload.patch -rtc-st-lpc-add-missing-clk_disable_unprepare-in-st_r.patch -rtc-pic32-move-devm_rtc_allocate_device-earlier-in-p.patch -rtc-pcf85063-fix-pcf85063_clkout_control.patch -nfsd-remove-spurious-cb_setup_err-tracepoint.patch -nfsd-under-nfsv4.1-fix-double-svc_xprt_put-on-rpc_cr.patch -net-macsec-fix-net-device-access-prior-to-holding-a-.patch -misdn-hfcsusb-don-t-call-dev_kfree_skb-kfree_skb-und.patch -misdn-hfcpci-don-t-call-dev_kfree_skb-kfree_skb-unde.patch -misdn-hfcmulti-don-t-call-dev_kfree_skb-kfree_skb-un.patch -nfc-pn533-clear-nfc_target-before-being-used.patch -r6040-fix-kmemleak-in-probe-and-remove.patch -net-switch-to-storing-kcov-handle-directly-in-sk_buf.patch -net-add-inline-function-skb_csum_is_sctp.patch -net-igc-use-skb_csum_is_sctp-instead-of-protocol-che.patch -net-add-a-helper-to-avoid-issues-with-hw-tx-timestam.patch -igc-enhance-qbv-scheduling-by-using-first-flag-bit.patch -igc-use-strict-cycles-for-qbv-scheduling.patch -igc-add-checking-for-basetime-less-than-zero.patch -igc-recalculate-qbv-end_time-by-considering-cycle-ti.patch -igc-lift-taprio-schedule-restriction.patch -igc-set-qbv-start_time-and-end_time-to-end_time-if-n.patch -rtc-mxc_v2-add-missing-clk_disable_unprepare.patch -selftests-devlink-fix-the-fd-redirect-in-dummy_repor.patch -openvswitch-fix-flow-lookup-to-use-unmasked-key.patch -skbuff-account-for-tail-adjustment-during-pull-opera.patch -mailbox-zynq-ipi-fix-error-handling-while-device_reg.patch -net_sched-reject-tcf_em_simple-case-for-complex-emat.patch -rxrpc-fix-missing-unlock-in-rxrpc_do_sendmsg.patch -myri10ge-fix-an-error-handling-path-in-myri10ge_prob.patch -net-stream-purge-sk_error_queue-in-sk_stream_kill_qu.patch -rcu-fix-__this_cpu_read-lockdep-warning-in-rcu_force.patch -arm64-make-is_ttbrx_addr-noinstr-safe.patch -video-hyperv_fb-avoid-taking-busy-spinlock-on-panic-.patch -x86-hyperv-remove-unregister-syscore-call-from-hyper.patch -binfmt_misc-fix-shift-out-of-bounds-in-check_special.patch -fs-jfs-fix-shift-out-of-bounds-in-dballocag.patch -udf-avoid-double-brelse-in-udf_rename.patch -fs-jfs-fix-shift-out-of-bounds-in-dbdiscardag.patch -acpica-fix-error-code-path-in-acpi_ds_call_control_m.patch -nilfs2-fix-shift-out-of-bounds-overflow-in-nilfs_sb2.patch -nilfs2-fix-shift-out-of-bounds-due-to-too-large-expo.patch -acct-fix-potential-integer-overflow-in-encode_comp_t.patch -hfs-fix-oob-read-in-__hfs_brec_find.patch -drm-etnaviv-add-missing-quirks-for-gc300.patch -brcmfmac-return-error-when-getting-invalid-max_flowr.patch -wifi-ath9k-verify-the-expected-usb_endpoints-are-pre.patch -wifi-ar5523-fix-use-after-free-on-ar5523_cmd-timed-o.patch -asoc-codecs-rt298-add-quirk-for-kbl-r-rvp-platform.patch -ipmi-fix-memleak-when-unload-ipmi-driver.patch -drm-amd-display-prevent-memory-leak.patch -qed-gcc13-use-u16-for-fid-to-be-big-enough.patch -bpf-make-sure-skb-len-0-when-redirecting-to-a-tunnel.patch -net-ethernet-ti-fix-return-type-of-netcp_ndo_start_x.patch -hamradio-baycom_epp-fix-return-type-of-baycom_send_p.patch -wifi-brcmfmac-fix-potential-shift-out-of-bounds-in-b.patch -igb-do-not-free-q_vector-unless-new-one-was-allocate.patch -drm-amdgpu-fix-type-of-second-parameter-in-trans_msg.patch -drm-amdgpu-fix-type-of-second-parameter-in-odn_edit_.patch -s390-ctcm-fix-return-type-of-ctc-mp-m_tx.patch -s390-netiucv-fix-return-type-of-netiucv_tx.patch -s390-lcs-fix-return-type-of-lcs_start_xmit.patch -drm-msm-use-drm_mode_copy.patch -drm-rockchip-use-drm_mode_copy.patch -drm-sti-use-drm_mode_copy.patch -drivers-md-md-bitmap-check-the-return-value-of-md_bi.patch -md-raid1-stop-mdx_raid1-thread-when-raid1-array-run-.patch -drm-amd-display-fix-array-index-out-of-bound-error-i.patch -net-add-atomic_long_t-to-net_device_stats-fields.patch -mrp-introduce-active-flags-to-prevent-uaf-when-appli.patch -ppp-associate-skb-with-a-device-at-tx.patch -bpf-prevent-decl_tag-from-being-referenced-in-func_p.patch -ethtool-avoiding-integer-overflow-in-ethtool_phys_id.patch -media-dvb-frontends-fix-leak-of-memory-fw.patch -media-dvbdev-adopts-refcnt-to-avoid-uaf.patch -media-dvb-usb-fix-memory-leak-in-dvb_usb_adapter_ini.patch -blk-mq-fix-possible-memleak-when-register-hctx-faile.patch -libbpf-avoid-enum-forward-declarations-in-public-api.patch -regulator-core-fix-use_count-leakage-when-handling-b.patch -mmc-f-sdh30-add-quirks-for-broken-timeout-clock-capa.patch -mmc-renesas_sdhi-better-reset-from-hs400-mode.patch -media-si470x-fix-use-after-free-in-si470x_int_in_cal.patch -clk-st-fix-memory-leak-in-st_of_quadfs_setup.patch -hugetlbfs-fix-null-ptr-deref-in-hugetlbfs_parse_para.patch -drm-fsl-dcu-fix-return-type-of-fsl_dcu_drm_connector.patch -drm-sti-fix-return-type-of-sti_-dvo-hda-hdmi-_connec.patch -orangefs-fix-kmemleak-in-orangefs_prepare_debugfs_he.patch -orangefs-fix-kmemleak-in-orangefs_-kernel-client-_de.patch -hwmon-jc42-fix-missing-unlock-on-error-in-jc42_write.patch -alsa-asoc-hda-move-rename-snd_hdac_ext_stop_streams-.patch -alsa-hda-add-snd_hdac_stop_streams-helper.patch -asoc-intel-skylake-fix-driver-hang-during-shutdown.patch -asoc-mediatek-mt8173-rt5650-rt5514-fix-refcount-leak.patch -asoc-audio-graph-card-fix-refcount-leak-of-cpu_ep-in.patch -asoc-rockchip-pdm-add-missing-clk_disable_unprepare-.patch -asoc-wm8994-fix-potential-deadlock.patch -asoc-rockchip-spdif-add-missing-clk_disable_unprepar.patch -asoc-rt5670-remove-unbalanced-pm_runtime_put.patch -loadpin-ignore-the-contents-argument-of-the-lsm-hook.patch -pstore-switch-pmsg_lock-to-an-rt_mutex-to-avoid-prio.patch -perf-debug-set-debug_peo_args-and-redirect_to_stderr.patch -afs-fix-lost-servers_outstanding-count.patch -pstore-make-sure-config_pstore_pmsg-selects-config_r.patch -ima-simplify-ima_lsm_copy_rule.patch -alsa-usb-audio-add-the-quirk-for-kt0206-device.patch -alsa-hda-realtek-add-quirk-for-lenovo-tianyi510pro-14iob.patch -alsa-hda-hdmi-add-hp-device-0x8711-to-force-connect-list.patch -usb-dwc3-fix-race-between-dwc3_set_mode-and-__dwc3_set_mode.patch -usb-dwc3-core-defer-probe-on-ulpi_read_id-timeout.patch -hid-wacom-ensure-bootloader-pid-is-usable-in-hidraw-mode.patch -hid-mcp2221-don-t-connect-hidraw.patch -reiserfs-add-missing-calls-to-reiserfs_security_free.patch -iio-adc-ad_sigma_delta-do-not-use-internal-iio_dev-lock.patch -iio-adc128s052-add-proper-.data-members-in-adc128_of_match-table.patch -regulator-core-fix-deadlock-on-regulator-enable.patch -gcov-add-support-for-checksum-field.patch -ovl-fix-use-inode-directly-in-rcu-walk-mode.patch -media-dvbdev-fix-build-warning-due-to-comments.patch -media-dvbdev-fix-refcnt-bug.patch -pwm-tegra-fix-32-bit-build.patch -usb-dwc3-qcom-fix-memory-leak-in-dwc3_qcom_interconn.patch -cifs-fix-oops-during-encryption.patch -nvme-pci-fix-doorbell-buffer-value-endianness.patch -nvme-pci-fix-mempool-alloc-size.patch -nvme-pci-fix-page-size-checks.patch -ata-ahci-fix-pcs-quirk-application-for-suspend.patch -nvme-fix-the-nvme_cmd_effects_cse_mask-definition.patch -nvmet-don-t-defer-passthrough-commands-with-trivial-.patch -objtool-fix-segfault.patch -powerpc-rtas-avoid-device-tree-lookups-in-rtas_os_te.patch -powerpc-rtas-avoid-scheduling-in-rtas_os_term.patch -hid-multitouch-fix-asus-expertbook-p2-p2451fa-trackp.patch -hid-plantronics-additional-pids-for-double-volume-ke.patch -pstore-zone-use-gfp_atomic-to-allocate-zone-buffer.patch -hfsplus-fix-bug-causing-custom-uid-and-gid-being-unable-to-be-assigned-with-mount.patch -binfmt-fix-error-return-code-in-load_elf_fdpic_binary.patch -ovl-use-ovl-mounter-s-fsuid-and-fsgid-in-ovl_link.patch -alsa-line6-correct-midi-status-byte-when-receiving-data-from-podxt.patch -alsa-line6-fix-stack-overflow-in-line6_midi_transmit.patch -pnode-terminate-at-peers-of-source.patch -md-fix-a-crash-in-mempool_free.patch -mm-compaction-fix-fast_isolate_around-to-stay-within-boundaries.patch -f2fs-should-put-a-page-when-checking-the-summary-info.patch -mmc-vub300-fix-warning-do-not-call-blocking-ops-when-task_running.patch -tpm-acpi-call-acpi_put_table-to-fix-memory-leak.patch -tpm-tpm_crb-add-the-missed-acpi_put_table-to-fix-memory-leak.patch -tpm-tpm_tis-add-the-missed-acpi_put_table-to-fix-memory-leak.patch -sunrpc-don-t-leak-netobj-memory-when-gss_read_proxy_verf-fails.patch -kcsan-instrument-memcpy-memset-memmove-with-newer-clang.patch kernel-provide-create_io_thread-helper.patch iov_iter-add-helper-to-save-iov_iter-state.patch saner-calling-conventions-for-unlazy_child.patch