From fcda2419d9fa83e20ba74da5b0f46db2a66b4316 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 16 Oct 2023 10:03:55 +0200 Subject: [PATCH] 5.15-stable patches added patches: arm64-allow-kprobes-on-el0-handlers.patch arm64-armv8_deprecated-fix-unused-function-error.patch arm64-armv8_deprecated-fold-ops-into-insn_emulation.patch arm64-armv8_deprecated-move-aarch32-helper-earlier.patch arm64-armv8_deprecated-move-emulation-functions.patch arm64-armv8_deprecated-rework-deprected-instruction-handling.patch arm64-consistently-pass-esr_elx-to-die.patch arm64-die-pass-err-as-long.patch arm64-factor-insn-read-out-of-call_undef_hook.patch arm64-factor-out-el1-ssbs-emulation-hook.patch arm64-report-el1-undefs-better.patch arm64-rework-bti-exception-handling.patch arm64-rework-el0-mrs-emulation.patch arm64-rework-fpac-exception-handling.patch arm64-split-el0-el1-undef-handlers.patch --- .../arm64-allow-kprobes-on-el0-handlers.patch | 104 ++++ ...deprecated-fix-unused-function-error.patch | 45 ++ ...recated-fold-ops-into-insn_emulation.patch | 241 ++++++++ ...precated-move-aarch32-helper-earlier.patch | 100 +++ ..._deprecated-move-emulation-functions.patch | 445 +++++++++++++ ...ework-deprected-instruction-handling.patch | 582 ++++++++++++++++++ ...m64-consistently-pass-esr_elx-to-die.patch | 174 ++++++ queue-5.15/arm64-die-pass-err-as-long.patch | 80 +++ ...tor-insn-read-out-of-call_undef_hook.patch | 118 ++++ ...4-factor-out-el1-ssbs-emulation-hook.patch | 167 +++++ .../arm64-report-el1-undefs-better.patch | 113 ++++ .../arm64-rework-bti-exception-handling.patch | 185 ++++++ .../arm64-rework-el0-mrs-emulation.patch | 144 +++++ ...arm64-rework-fpac-exception-handling.patch | 173 ++++++ .../arm64-split-el0-el1-undef-handlers.patch | 177 ++++++ queue-5.15/series | 15 + 16 files changed, 2863 insertions(+) create mode 100644 queue-5.15/arm64-allow-kprobes-on-el0-handlers.patch create mode 100644 queue-5.15/arm64-armv8_deprecated-fix-unused-function-error.patch create mode 100644 queue-5.15/arm64-armv8_deprecated-fold-ops-into-insn_emulation.patch create mode 100644 queue-5.15/arm64-armv8_deprecated-move-aarch32-helper-earlier.patch create mode 100644 queue-5.15/arm64-armv8_deprecated-move-emulation-functions.patch create mode 100644 queue-5.15/arm64-armv8_deprecated-rework-deprected-instruction-handling.patch create mode 100644 queue-5.15/arm64-consistently-pass-esr_elx-to-die.patch create mode 100644 queue-5.15/arm64-die-pass-err-as-long.patch create mode 100644 queue-5.15/arm64-factor-insn-read-out-of-call_undef_hook.patch create mode 100644 queue-5.15/arm64-factor-out-el1-ssbs-emulation-hook.patch create mode 100644 queue-5.15/arm64-report-el1-undefs-better.patch create mode 100644 queue-5.15/arm64-rework-bti-exception-handling.patch create mode 100644 queue-5.15/arm64-rework-el0-mrs-emulation.patch create mode 100644 queue-5.15/arm64-rework-fpac-exception-handling.patch create mode 100644 queue-5.15/arm64-split-el0-el1-undef-handlers.patch diff --git a/queue-5.15/arm64-allow-kprobes-on-el0-handlers.patch b/queue-5.15/arm64-allow-kprobes-on-el0-handlers.patch new file mode 100644 index 00000000000..f80602ba181 --- /dev/null +++ b/queue-5.15/arm64-allow-kprobes-on-el0-handlers.patch @@ -0,0 +1,104 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:43 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:46 +0000 +Subject: arm64: allow kprobes on EL0 handlers +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-7-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit b3a0c010e900a9f89dcd99f10bd8f7538d21b0a9 upstream. + +Currently do_sysinstr() and do_cp15instr() are marked with +NOKPROBE_SYMBOL(). However, these are only called for exceptions taken +from EL0, and there is no risk of recursion in kprobes, so this is not +necessary. + +Remove the NOKPROBE_SYMBOL() annotation, and rename the two functions to +more clearly indicate that these are solely for exceptions taken from +EL0, better matching the names used by the lower level entry points in +entry-common.c. + +Signed-off-by: Mark Rutland +Cc: Catalin Marinas +Cc: James Morse +Cc: Joey Gouly +Cc: Peter Zijlstra +Cc: Will Deacon +Link: https://lore.kernel.org/r/20221019144123.612388-2-mark.rutland@arm.com +Signed-off-by: Will Deacon +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/exception.h | 4 ++-- + arch/arm64/kernel/entry-common.c | 4 ++-- + arch/arm64/kernel/traps.c | 6 ++---- + 3 files changed, 6 insertions(+), 8 deletions(-) + +--- a/arch/arm64/include/asm/exception.h ++++ b/arch/arm64/include/asm/exception.h +@@ -66,10 +66,10 @@ void do_debug_exception(unsigned long ad + void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs); + void do_sve_acc(unsigned long esr, struct pt_regs *regs); + void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs); +-void do_sysinstr(unsigned long esr, struct pt_regs *regs); ++void do_el0_sys(unsigned long esr, struct pt_regs *regs); + void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs); + void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr); +-void do_cp15instr(unsigned long esr, struct pt_regs *regs); ++void do_el0_cp15(unsigned long esr, struct pt_regs *regs); + void do_el0_svc(struct pt_regs *regs); + void do_el0_svc_compat(struct pt_regs *regs); + void do_el0_fpac(struct pt_regs *regs, unsigned long esr); +--- a/arch/arm64/kernel/entry-common.c ++++ b/arch/arm64/kernel/entry-common.c +@@ -541,7 +541,7 @@ static void noinstr el0_sys(struct pt_re + { + enter_from_user_mode(regs); + local_daif_restore(DAIF_PROCCTX); +- do_sysinstr(esr, regs); ++ do_el0_sys(esr, regs); + exit_to_user_mode(regs); + } + +@@ -728,7 +728,7 @@ static void noinstr el0_cp15(struct pt_r + { + enter_from_user_mode(regs); + local_daif_restore(DAIF_PROCCTX); +- do_cp15instr(esr, regs); ++ do_el0_cp15(esr, regs); + exit_to_user_mode(regs); + } + +--- a/arch/arm64/kernel/traps.c ++++ b/arch/arm64/kernel/traps.c +@@ -742,7 +742,7 @@ static const struct sys64_hook cp15_64_h + {}, + }; + +-void do_cp15instr(unsigned long esr, struct pt_regs *regs) ++void do_el0_cp15(unsigned long esr, struct pt_regs *regs) + { + const struct sys64_hook *hook, *hook_base; + +@@ -780,10 +780,9 @@ void do_cp15instr(unsigned long esr, str + */ + do_undefinstr(regs, esr); + } +-NOKPROBE_SYMBOL(do_cp15instr); + #endif + +-void do_sysinstr(unsigned long esr, struct pt_regs *regs) ++void do_el0_sys(unsigned long esr, struct pt_regs *regs) + { + const struct sys64_hook *hook; + +@@ -800,7 +799,6 @@ void do_sysinstr(unsigned long esr, stru + */ + do_undefinstr(regs, esr); + } +-NOKPROBE_SYMBOL(do_sysinstr); + + static const char *esr_class_str[] = { + [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC", diff --git a/queue-5.15/arm64-armv8_deprecated-fix-unused-function-error.patch b/queue-5.15/arm64-armv8_deprecated-fix-unused-function-error.patch new file mode 100644 index 00000000000..9d72c3d2f1e --- /dev/null +++ b/queue-5.15/arm64-armv8_deprecated-fix-unused-function-error.patch @@ -0,0 +1,45 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:54 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:55 +0000 +Subject: arm64: armv8_deprecated: fix unused-function error +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-16-ruanjinjie@huawei.com> + +From: Ren Zhijie + +commit 223d3a0d30b6e9f979f5642e430e1753d3e29f89 upstream. + +If CONFIG_SWP_EMULATION is not set and +CONFIG_CP15_BARRIER_EMULATION is not set, +aarch64-linux-gnu complained about unused-function : + +arch/arm64/kernel/armv8_deprecated.c:67:21: error: ‘aarch32_check_condition’ defined but not used [-Werror=unused-function] + static unsigned int aarch32_check_condition(u32 opcode, u32 psr) + ^~~~~~~~~~~~~~~~~~~~~~~ +cc1: all warnings being treated as errors + +To fix this warning, modify aarch32_check_condition() with __maybe_unused. + +Fixes: 0c5f416219da ("arm64: armv8_deprecated: move aarch32 helper earlier") +Signed-off-by: Ren Zhijie +Acked-by: Mark Rutland +Link: https://lore.kernel.org/r/20221124022429.19024-1-renzhijie2@huawei.com +Signed-off-by: Will Deacon +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/kernel/armv8_deprecated.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +--- a/arch/arm64/kernel/armv8_deprecated.c ++++ b/arch/arm64/kernel/armv8_deprecated.c +@@ -64,7 +64,7 @@ struct insn_emulation { + + #define ARM_OPCODE_CONDITION_UNCOND 0xf + +-static unsigned int aarch32_check_condition(u32 opcode, u32 psr) ++static unsigned int __maybe_unused aarch32_check_condition(u32 opcode, u32 psr) + { + u32 cc_bits = opcode >> 28; + diff --git a/queue-5.15/arm64-armv8_deprecated-fold-ops-into-insn_emulation.patch b/queue-5.15/arm64-armv8_deprecated-fold-ops-into-insn_emulation.patch new file mode 100644 index 00000000000..068d55a3a54 --- /dev/null +++ b/queue-5.15/arm64-armv8_deprecated-fold-ops-into-insn_emulation.patch @@ -0,0 +1,241 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:48 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:51 +0000 +Subject: arm64: armv8_deprecated: fold ops into insn_emulation +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-12-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit b4453cc8a7ebbd45436a8cd3ffeaa069ceac146f upstream. + +The code for emulating deprecated instructions has two related +structures: struct insn_emulation_ops and struct insn_emulation, where +each struct insn_emulation_ops is associated 1-1 with a struct +insn_emulation. + +It would be simpler to combine the two into a single structure, removing +the need for (unconditional) dynamic allocation at boot time, and +simplifying some runtime pointer chasing. + +This patch merges the two structures together. + +There should be no functional change as a result of this patch. + +Signed-off-by: Mark Rutland +Cc: Catalin Marinas +Cc: James Morse +Cc: Joey Gouly +Cc: Peter Zijlstra +Cc: Will Deacon +Link: https://lore.kernel.org/r/20221019144123.612388-7-mark.rutland@arm.com +Signed-off-by: Will Deacon +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/kernel/armv8_deprecated.c | 76 +++++++++++++++-------------------- + 1 file changed, 33 insertions(+), 43 deletions(-) + +--- a/arch/arm64/kernel/armv8_deprecated.c ++++ b/arch/arm64/kernel/armv8_deprecated.c +@@ -41,16 +41,12 @@ enum legacy_insn_status { + INSN_OBSOLETE, + }; + +-struct insn_emulation_ops { +- const char *name; +- enum legacy_insn_status status; +- struct undef_hook *hooks; +- int (*set_hw_mode)(bool enable); +-}; +- + struct insn_emulation { +- struct list_head node; +- struct insn_emulation_ops *ops; ++ const char *name; ++ struct list_head node; ++ enum legacy_insn_status status; ++ struct undef_hook *hooks; ++ int (*set_hw_mode)(bool enable); + int current_mode; + int min; + int max; +@@ -61,48 +57,48 @@ static int nr_insn_emulated __initdata; + static DEFINE_RAW_SPINLOCK(insn_emulation_lock); + static DEFINE_MUTEX(insn_emulation_mutex); + +-static void register_emulation_hooks(struct insn_emulation_ops *ops) ++static void register_emulation_hooks(struct insn_emulation *insn) + { + struct undef_hook *hook; + +- BUG_ON(!ops->hooks); ++ BUG_ON(!insn->hooks); + +- for (hook = ops->hooks; hook->instr_mask; hook++) ++ for (hook = insn->hooks; hook->instr_mask; hook++) + register_undef_hook(hook); + +- pr_notice("Registered %s emulation handler\n", ops->name); ++ pr_notice("Registered %s emulation handler\n", insn->name); + } + +-static void remove_emulation_hooks(struct insn_emulation_ops *ops) ++static void remove_emulation_hooks(struct insn_emulation *insn) + { + struct undef_hook *hook; + +- BUG_ON(!ops->hooks); ++ BUG_ON(!insn->hooks); + +- for (hook = ops->hooks; hook->instr_mask; hook++) ++ for (hook = insn->hooks; hook->instr_mask; hook++) + unregister_undef_hook(hook); + +- pr_notice("Removed %s emulation handler\n", ops->name); ++ pr_notice("Removed %s emulation handler\n", insn->name); + } + + static void enable_insn_hw_mode(void *data) + { + struct insn_emulation *insn = (struct insn_emulation *)data; +- if (insn->ops->set_hw_mode) +- insn->ops->set_hw_mode(true); ++ if (insn->set_hw_mode) ++ insn->set_hw_mode(true); + } + + static void disable_insn_hw_mode(void *data) + { + struct insn_emulation *insn = (struct insn_emulation *)data; +- if (insn->ops->set_hw_mode) +- insn->ops->set_hw_mode(false); ++ if (insn->set_hw_mode) ++ insn->set_hw_mode(false); + } + + /* Run set_hw_mode(mode) on all active CPUs */ + static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable) + { +- if (!insn->ops->set_hw_mode) ++ if (!insn->set_hw_mode) + return -EINVAL; + if (enable) + on_each_cpu(enable_insn_hw_mode, (void *)insn, true); +@@ -126,9 +122,9 @@ static int run_all_insn_set_hw_mode(unsi + raw_spin_lock_irqsave(&insn_emulation_lock, flags); + list_for_each_entry(insn, &insn_emulation, node) { + bool enable = (insn->current_mode == INSN_HW); +- if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) { ++ if (insn->set_hw_mode && insn->set_hw_mode(enable)) { + pr_warn("CPU[%u] cannot support the emulation of %s", +- cpu, insn->ops->name); ++ cpu, insn->name); + rc = -EINVAL; + } + } +@@ -145,11 +141,11 @@ static int update_insn_emulation_mode(st + case INSN_UNDEF: /* Nothing to be done */ + break; + case INSN_EMULATE: +- remove_emulation_hooks(insn->ops); ++ remove_emulation_hooks(insn); + break; + case INSN_HW: + if (!run_all_cpu_set_hw_mode(insn, false)) +- pr_notice("Disabled %s support\n", insn->ops->name); ++ pr_notice("Disabled %s support\n", insn->name); + break; + } + +@@ -157,31 +153,25 @@ static int update_insn_emulation_mode(st + case INSN_UNDEF: + break; + case INSN_EMULATE: +- register_emulation_hooks(insn->ops); ++ register_emulation_hooks(insn); + break; + case INSN_HW: + ret = run_all_cpu_set_hw_mode(insn, true); + if (!ret) +- pr_notice("Enabled %s support\n", insn->ops->name); ++ pr_notice("Enabled %s support\n", insn->name); + break; + } + + return ret; + } + +-static void __init register_insn_emulation(struct insn_emulation_ops *ops) ++static void __init register_insn_emulation(struct insn_emulation *insn) + { + unsigned long flags; +- struct insn_emulation *insn; +- +- insn = kzalloc(sizeof(*insn), GFP_KERNEL); +- if (!insn) +- return; + +- insn->ops = ops; + insn->min = INSN_UNDEF; + +- switch (ops->status) { ++ switch (insn->status) { + case INSN_DEPRECATED: + insn->current_mode = INSN_EMULATE; + /* Disable the HW mode if it was turned on at early boot time */ +@@ -247,7 +237,7 @@ static void __init register_insn_emulati + sysctl->mode = 0644; + sysctl->maxlen = sizeof(int); + +- sysctl->procname = insn->ops->name; ++ sysctl->procname = insn->name; + sysctl->data = &insn->current_mode; + sysctl->extra1 = &insn->min; + sysctl->extra2 = &insn->max; +@@ -451,7 +441,7 @@ static struct undef_hook swp_hooks[] = { + { } + }; + +-static struct insn_emulation_ops swp_ops = { ++static struct insn_emulation insn_swp = { + .name = "swp", + .status = INSN_OBSOLETE, + .hooks = swp_hooks, +@@ -538,7 +528,7 @@ static struct undef_hook cp15_barrier_ho + { } + }; + +-static struct insn_emulation_ops cp15_barrier_ops = { ++static struct insn_emulation insn_cp15_barrier = { + .name = "cp15_barrier", + .status = INSN_DEPRECATED, + .hooks = cp15_barrier_hooks, +@@ -611,7 +601,7 @@ static struct undef_hook setend_hooks[] + {} + }; + +-static struct insn_emulation_ops setend_ops = { ++static struct insn_emulation insn_setend = { + .name = "setend", + .status = INSN_DEPRECATED, + .hooks = setend_hooks, +@@ -625,14 +615,14 @@ static struct insn_emulation_ops setend_ + static int __init armv8_deprecated_init(void) + { + if (IS_ENABLED(CONFIG_SWP_EMULATION)) +- register_insn_emulation(&swp_ops); ++ register_insn_emulation(&insn_swp); + + if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION)) +- register_insn_emulation(&cp15_barrier_ops); ++ register_insn_emulation(&insn_cp15_barrier); + + if (IS_ENABLED(CONFIG_SETEND_EMULATION)) { + if (system_supports_mixed_endian_el0()) +- register_insn_emulation(&setend_ops); ++ register_insn_emulation(&insn_setend); + else + pr_info("setend instruction emulation is not supported on this system\n"); + } diff --git a/queue-5.15/arm64-armv8_deprecated-move-aarch32-helper-earlier.patch b/queue-5.15/arm64-armv8_deprecated-move-aarch32-helper-earlier.patch new file mode 100644 index 00000000000..cb15b73510b --- /dev/null +++ b/queue-5.15/arm64-armv8_deprecated-move-aarch32-helper-earlier.patch @@ -0,0 +1,100 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:51 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:53 +0000 +Subject: arm64: armv8_deprecated: move aarch32 helper earlier +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-14-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit 0c5f416219da3795dc8b33e5bb7865a6b3c4e55c upstream. + +Subsequent patches will rework the logic in armv8_deprecated.c. + +In preparation for subsequent changes, this patch moves some shared logic +earlier in the file. This will make subsequent diffs simpler and easier to +read. + +At the same time, drop the `__kprobes` annotation from +aarch32_check_condition(), as this is only used for traps from compat +userspace, and has no risk of recursion within kprobes. As this is the +last kprobes annotation in armve8_deprecated.c, we no longer need to +include . + +Signed-off-by: Mark Rutland +Cc: Catalin Marinas +Cc: James Morse +Cc: Joey Gouly +Cc: Peter Zijlstra +Cc: Will Deacon +Link: https://lore.kernel.org/r/20221019144123.612388-9-mark.rutland@arm.com +Signed-off-by: Will Deacon +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/kernel/armv8_deprecated.c | 39 +++++++++++++++++------------------ + 1 file changed, 19 insertions(+), 20 deletions(-) + +--- a/arch/arm64/kernel/armv8_deprecated.c ++++ b/arch/arm64/kernel/armv8_deprecated.c +@@ -17,7 +17,6 @@ + #include + #include + #include +-#include + + #define CREATE_TRACE_POINTS + #include "trace-events-emulation.h" +@@ -52,6 +51,25 @@ struct insn_emulation { + int max; + }; + ++#define ARM_OPCODE_CONDTEST_FAIL 0 ++#define ARM_OPCODE_CONDTEST_PASS 1 ++#define ARM_OPCODE_CONDTEST_UNCOND 2 ++ ++#define ARM_OPCODE_CONDITION_UNCOND 0xf ++ ++static unsigned int aarch32_check_condition(u32 opcode, u32 psr) ++{ ++ u32 cc_bits = opcode >> 28; ++ ++ if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) { ++ if ((*aarch32_opcode_cond_checks[cc_bits])(psr)) ++ return ARM_OPCODE_CONDTEST_PASS; ++ else ++ return ARM_OPCODE_CONDTEST_FAIL; ++ } ++ return ARM_OPCODE_CONDTEST_UNCOND; ++} ++ + /* + * Implement emulation of the SWP/SWPB instructions using load-exclusive and + * store-exclusive. +@@ -138,25 +156,6 @@ static int emulate_swpX(unsigned int add + return res; + } + +-#define ARM_OPCODE_CONDTEST_FAIL 0 +-#define ARM_OPCODE_CONDTEST_PASS 1 +-#define ARM_OPCODE_CONDTEST_UNCOND 2 +- +-#define ARM_OPCODE_CONDITION_UNCOND 0xf +- +-static unsigned int __kprobes aarch32_check_condition(u32 opcode, u32 psr) +-{ +- u32 cc_bits = opcode >> 28; +- +- if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) { +- if ((*aarch32_opcode_cond_checks[cc_bits])(psr)) +- return ARM_OPCODE_CONDTEST_PASS; +- else +- return ARM_OPCODE_CONDTEST_FAIL; +- } +- return ARM_OPCODE_CONDTEST_UNCOND; +-} +- + /* + * swp_handler logs the id of calling process, dissects the instruction, sanity + * checks the memory location, calls emulate_swpX for the actual operation and diff --git a/queue-5.15/arm64-armv8_deprecated-move-emulation-functions.patch b/queue-5.15/arm64-armv8_deprecated-move-emulation-functions.patch new file mode 100644 index 00000000000..93729c8be8b --- /dev/null +++ b/queue-5.15/arm64-armv8_deprecated-move-emulation-functions.patch @@ -0,0 +1,445 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:08:21 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:52 +0000 +Subject: arm64: armv8_deprecated move emulation functions +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-13-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit 25eeac0cfe7c97ade1be07340e11e7143aab57a6 upstream. + +Subsequent patches will rework the logic in armv8_deprecated.c. + +In preparation for subsequent changes, this patch moves the emulation +logic earlier in the file, and moves the infrastructure later in the +file. This will make subsequent diffs simpler and easier to read. + +This is purely a move. There should be no functional change as a result +of this patch. + +Signed-off-by: Mark Rutland +Cc: Catalin Marinas +Cc: James Morse +Cc: Joey Gouly +Cc: Peter Zijlstra +Cc: Will Deacon +Link: https://lore.kernel.org/r/20221019144123.612388-8-mark.rutland@arm.com +Signed-off-by: Will Deacon +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/kernel/armv8_deprecated.c | 394 +++++++++++++++++------------------ + 1 file changed, 197 insertions(+), 197 deletions(-) + +--- a/arch/arm64/kernel/armv8_deprecated.c ++++ b/arch/arm64/kernel/armv8_deprecated.c +@@ -52,203 +52,6 @@ struct insn_emulation { + int max; + }; + +-static LIST_HEAD(insn_emulation); +-static int nr_insn_emulated __initdata; +-static DEFINE_RAW_SPINLOCK(insn_emulation_lock); +-static DEFINE_MUTEX(insn_emulation_mutex); +- +-static void register_emulation_hooks(struct insn_emulation *insn) +-{ +- struct undef_hook *hook; +- +- BUG_ON(!insn->hooks); +- +- for (hook = insn->hooks; hook->instr_mask; hook++) +- register_undef_hook(hook); +- +- pr_notice("Registered %s emulation handler\n", insn->name); +-} +- +-static void remove_emulation_hooks(struct insn_emulation *insn) +-{ +- struct undef_hook *hook; +- +- BUG_ON(!insn->hooks); +- +- for (hook = insn->hooks; hook->instr_mask; hook++) +- unregister_undef_hook(hook); +- +- pr_notice("Removed %s emulation handler\n", insn->name); +-} +- +-static void enable_insn_hw_mode(void *data) +-{ +- struct insn_emulation *insn = (struct insn_emulation *)data; +- if (insn->set_hw_mode) +- insn->set_hw_mode(true); +-} +- +-static void disable_insn_hw_mode(void *data) +-{ +- struct insn_emulation *insn = (struct insn_emulation *)data; +- if (insn->set_hw_mode) +- insn->set_hw_mode(false); +-} +- +-/* Run set_hw_mode(mode) on all active CPUs */ +-static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable) +-{ +- if (!insn->set_hw_mode) +- return -EINVAL; +- if (enable) +- on_each_cpu(enable_insn_hw_mode, (void *)insn, true); +- else +- on_each_cpu(disable_insn_hw_mode, (void *)insn, true); +- return 0; +-} +- +-/* +- * Run set_hw_mode for all insns on a starting CPU. +- * Returns: +- * 0 - If all the hooks ran successfully. +- * -EINVAL - At least one hook is not supported by the CPU. +- */ +-static int run_all_insn_set_hw_mode(unsigned int cpu) +-{ +- int rc = 0; +- unsigned long flags; +- struct insn_emulation *insn; +- +- raw_spin_lock_irqsave(&insn_emulation_lock, flags); +- list_for_each_entry(insn, &insn_emulation, node) { +- bool enable = (insn->current_mode == INSN_HW); +- if (insn->set_hw_mode && insn->set_hw_mode(enable)) { +- pr_warn("CPU[%u] cannot support the emulation of %s", +- cpu, insn->name); +- rc = -EINVAL; +- } +- } +- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); +- return rc; +-} +- +-static int update_insn_emulation_mode(struct insn_emulation *insn, +- enum insn_emulation_mode prev) +-{ +- int ret = 0; +- +- switch (prev) { +- case INSN_UNDEF: /* Nothing to be done */ +- break; +- case INSN_EMULATE: +- remove_emulation_hooks(insn); +- break; +- case INSN_HW: +- if (!run_all_cpu_set_hw_mode(insn, false)) +- pr_notice("Disabled %s support\n", insn->name); +- break; +- } +- +- switch (insn->current_mode) { +- case INSN_UNDEF: +- break; +- case INSN_EMULATE: +- register_emulation_hooks(insn); +- break; +- case INSN_HW: +- ret = run_all_cpu_set_hw_mode(insn, true); +- if (!ret) +- pr_notice("Enabled %s support\n", insn->name); +- break; +- } +- +- return ret; +-} +- +-static void __init register_insn_emulation(struct insn_emulation *insn) +-{ +- unsigned long flags; +- +- insn->min = INSN_UNDEF; +- +- switch (insn->status) { +- case INSN_DEPRECATED: +- insn->current_mode = INSN_EMULATE; +- /* Disable the HW mode if it was turned on at early boot time */ +- run_all_cpu_set_hw_mode(insn, false); +- insn->max = INSN_HW; +- break; +- case INSN_OBSOLETE: +- insn->current_mode = INSN_UNDEF; +- insn->max = INSN_EMULATE; +- break; +- } +- +- raw_spin_lock_irqsave(&insn_emulation_lock, flags); +- list_add(&insn->node, &insn_emulation); +- nr_insn_emulated++; +- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); +- +- /* Register any handlers if required */ +- update_insn_emulation_mode(insn, INSN_UNDEF); +-} +- +-static int emulation_proc_handler(struct ctl_table *table, int write, +- void *buffer, size_t *lenp, +- loff_t *ppos) +-{ +- int ret = 0; +- struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode); +- enum insn_emulation_mode prev_mode = insn->current_mode; +- +- mutex_lock(&insn_emulation_mutex); +- ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); +- +- if (ret || !write || prev_mode == insn->current_mode) +- goto ret; +- +- ret = update_insn_emulation_mode(insn, prev_mode); +- if (ret) { +- /* Mode change failed, revert to previous mode. */ +- insn->current_mode = prev_mode; +- update_insn_emulation_mode(insn, INSN_UNDEF); +- } +-ret: +- mutex_unlock(&insn_emulation_mutex); +- return ret; +-} +- +-static void __init register_insn_emulation_sysctl(void) +-{ +- unsigned long flags; +- int i = 0; +- struct insn_emulation *insn; +- struct ctl_table *insns_sysctl, *sysctl; +- +- insns_sysctl = kcalloc(nr_insn_emulated + 1, sizeof(*sysctl), +- GFP_KERNEL); +- if (!insns_sysctl) +- return; +- +- raw_spin_lock_irqsave(&insn_emulation_lock, flags); +- list_for_each_entry(insn, &insn_emulation, node) { +- sysctl = &insns_sysctl[i]; +- +- sysctl->mode = 0644; +- sysctl->maxlen = sizeof(int); +- +- sysctl->procname = insn->name; +- sysctl->data = &insn->current_mode; +- sysctl->extra1 = &insn->min; +- sysctl->extra2 = &insn->max; +- sysctl->proc_handler = emulation_proc_handler; +- i++; +- } +- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); +- +- register_sysctl("abi", insns_sysctl); +-} +- + /* + * Implement emulation of the SWP/SWPB instructions using load-exclusive and + * store-exclusive. +@@ -608,6 +411,203 @@ static struct insn_emulation insn_setend + .set_hw_mode = setend_set_hw_mode, + }; + ++static LIST_HEAD(insn_emulation); ++static int nr_insn_emulated __initdata; ++static DEFINE_RAW_SPINLOCK(insn_emulation_lock); ++static DEFINE_MUTEX(insn_emulation_mutex); ++ ++static void register_emulation_hooks(struct insn_emulation *insn) ++{ ++ struct undef_hook *hook; ++ ++ BUG_ON(!insn->hooks); ++ ++ for (hook = insn->hooks; hook->instr_mask; hook++) ++ register_undef_hook(hook); ++ ++ pr_notice("Registered %s emulation handler\n", insn->name); ++} ++ ++static void remove_emulation_hooks(struct insn_emulation *insn) ++{ ++ struct undef_hook *hook; ++ ++ BUG_ON(!insn->hooks); ++ ++ for (hook = insn->hooks; hook->instr_mask; hook++) ++ unregister_undef_hook(hook); ++ ++ pr_notice("Removed %s emulation handler\n", insn->name); ++} ++ ++static void enable_insn_hw_mode(void *data) ++{ ++ struct insn_emulation *insn = (struct insn_emulation *)data; ++ if (insn->set_hw_mode) ++ insn->set_hw_mode(true); ++} ++ ++static void disable_insn_hw_mode(void *data) ++{ ++ struct insn_emulation *insn = (struct insn_emulation *)data; ++ if (insn->set_hw_mode) ++ insn->set_hw_mode(false); ++} ++ ++/* Run set_hw_mode(mode) on all active CPUs */ ++static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable) ++{ ++ if (!insn->set_hw_mode) ++ return -EINVAL; ++ if (enable) ++ on_each_cpu(enable_insn_hw_mode, (void *)insn, true); ++ else ++ on_each_cpu(disable_insn_hw_mode, (void *)insn, true); ++ return 0; ++} ++ ++/* ++ * Run set_hw_mode for all insns on a starting CPU. ++ * Returns: ++ * 0 - If all the hooks ran successfully. ++ * -EINVAL - At least one hook is not supported by the CPU. ++ */ ++static int run_all_insn_set_hw_mode(unsigned int cpu) ++{ ++ int rc = 0; ++ unsigned long flags; ++ struct insn_emulation *insn; ++ ++ raw_spin_lock_irqsave(&insn_emulation_lock, flags); ++ list_for_each_entry(insn, &insn_emulation, node) { ++ bool enable = (insn->current_mode == INSN_HW); ++ if (insn->set_hw_mode && insn->set_hw_mode(enable)) { ++ pr_warn("CPU[%u] cannot support the emulation of %s", ++ cpu, insn->name); ++ rc = -EINVAL; ++ } ++ } ++ raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); ++ return rc; ++} ++ ++static int update_insn_emulation_mode(struct insn_emulation *insn, ++ enum insn_emulation_mode prev) ++{ ++ int ret = 0; ++ ++ switch (prev) { ++ case INSN_UNDEF: /* Nothing to be done */ ++ break; ++ case INSN_EMULATE: ++ remove_emulation_hooks(insn); ++ break; ++ case INSN_HW: ++ if (!run_all_cpu_set_hw_mode(insn, false)) ++ pr_notice("Disabled %s support\n", insn->name); ++ break; ++ } ++ ++ switch (insn->current_mode) { ++ case INSN_UNDEF: ++ break; ++ case INSN_EMULATE: ++ register_emulation_hooks(insn); ++ break; ++ case INSN_HW: ++ ret = run_all_cpu_set_hw_mode(insn, true); ++ if (!ret) ++ pr_notice("Enabled %s support\n", insn->name); ++ break; ++ } ++ ++ return ret; ++} ++ ++static void __init register_insn_emulation(struct insn_emulation *insn) ++{ ++ unsigned long flags; ++ ++ insn->min = INSN_UNDEF; ++ ++ switch (insn->status) { ++ case INSN_DEPRECATED: ++ insn->current_mode = INSN_EMULATE; ++ /* Disable the HW mode if it was turned on at early boot time */ ++ run_all_cpu_set_hw_mode(insn, false); ++ insn->max = INSN_HW; ++ break; ++ case INSN_OBSOLETE: ++ insn->current_mode = INSN_UNDEF; ++ insn->max = INSN_EMULATE; ++ break; ++ } ++ ++ raw_spin_lock_irqsave(&insn_emulation_lock, flags); ++ list_add(&insn->node, &insn_emulation); ++ nr_insn_emulated++; ++ raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); ++ ++ /* Register any handlers if required */ ++ update_insn_emulation_mode(insn, INSN_UNDEF); ++} ++ ++static int emulation_proc_handler(struct ctl_table *table, int write, ++ void *buffer, size_t *lenp, ++ loff_t *ppos) ++{ ++ int ret = 0; ++ struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode); ++ enum insn_emulation_mode prev_mode = insn->current_mode; ++ ++ mutex_lock(&insn_emulation_mutex); ++ ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); ++ ++ if (ret || !write || prev_mode == insn->current_mode) ++ goto ret; ++ ++ ret = update_insn_emulation_mode(insn, prev_mode); ++ if (ret) { ++ /* Mode change failed, revert to previous mode. */ ++ insn->current_mode = prev_mode; ++ update_insn_emulation_mode(insn, INSN_UNDEF); ++ } ++ret: ++ mutex_unlock(&insn_emulation_mutex); ++ return ret; ++} ++ ++static void __init register_insn_emulation_sysctl(void) ++{ ++ unsigned long flags; ++ int i = 0; ++ struct insn_emulation *insn; ++ struct ctl_table *insns_sysctl, *sysctl; ++ ++ insns_sysctl = kcalloc(nr_insn_emulated + 1, sizeof(*sysctl), ++ GFP_KERNEL); ++ if (!insns_sysctl) ++ return; ++ ++ raw_spin_lock_irqsave(&insn_emulation_lock, flags); ++ list_for_each_entry(insn, &insn_emulation, node) { ++ sysctl = &insns_sysctl[i]; ++ ++ sysctl->mode = 0644; ++ sysctl->maxlen = sizeof(int); ++ ++ sysctl->procname = insn->name; ++ sysctl->data = &insn->current_mode; ++ sysctl->extra1 = &insn->min; ++ sysctl->extra2 = &insn->max; ++ sysctl->proc_handler = emulation_proc_handler; ++ i++; ++ } ++ raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); ++ ++ register_sysctl("abi", insns_sysctl); ++} ++ + /* + * Invoked as core_initcall, which guarantees that the instruction + * emulation is ready for userspace. diff --git a/queue-5.15/arm64-armv8_deprecated-rework-deprected-instruction-handling.patch b/queue-5.15/arm64-armv8_deprecated-rework-deprected-instruction-handling.patch new file mode 100644 index 00000000000..946abbbbba5 --- /dev/null +++ b/queue-5.15/arm64-armv8_deprecated-rework-deprected-instruction-handling.patch @@ -0,0 +1,582 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:52 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:54 +0000 +Subject: arm64: armv8_deprecated: rework deprected instruction handling +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-15-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit 124c49b1b5d947b7180c5d6cbb09ddf76ea45ea2 upstream. + +Support for deprecated instructions can be enabled or disabled at +runtime. To handle this, the code in armv8_deprecated.c registers and +unregisters undef_hooks, and makes cross CPU calls to configure HW +support. This is rather complicated, and the synchronization required to +make this safe ends up serializing the handling of instructions which +have been trapped. + +This patch simplifies the deprecated instruction handling by removing +the dynamic registration and unregistration, and changing the trap +handling code to determine whether a handler should be invoked. This +removes the need for dynamic list management, and simplifies the locking +requirements, making it possible to handle trapped instructions entirely +in parallel. + +Where changing the emulation state requires a cross-call, this is +serialized by locally disabling interrupts, ensuring that the CPU is not +left in an inconsistent state. + +To simplify sysctl management, each insn_emulation is given a separate +sysctl table, permitting these to be registered separately. The core +sysctl code will iterate over all of these when walking sysfs. + +I've tested this with userspace programs which use each of the +deprecated instructions, and I've concurrently modified the support +level for each of the features back-and-forth between HW and emulated to +check that there are no spurious SIGILLs sent to userspace when the +support level is changed. + +Signed-off-by: Mark Rutland +Cc: Catalin Marinas +Cc: James Morse +Cc: Joey Gouly +Cc: Peter Zijlstra +Cc: Will Deacon +Link: https://lore.kernel.org/r/20221019144123.612388-10-mark.rutland@arm.com +Signed-off-by: Will Deacon +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/traps.h | 19 +- + arch/arm64/kernel/armv8_deprecated.c | 291 +++++++++++++++++------------------ + arch/arm64/kernel/traps.c | 40 ---- + 3 files changed, 156 insertions(+), 194 deletions(-) + +--- a/arch/arm64/include/asm/traps.h ++++ b/arch/arm64/include/asm/traps.h +@@ -13,17 +13,16 @@ + + struct pt_regs; + +-struct undef_hook { +- struct list_head node; +- u32 instr_mask; +- u32 instr_val; +- u64 pstate_mask; +- u64 pstate_val; +- int (*fn)(struct pt_regs *regs, u32 instr); +-}; ++#ifdef CONFIG_ARMV8_DEPRECATED ++bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn); ++#else ++static inline bool ++try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn) ++{ ++ return false; ++} ++#endif /* CONFIG_ARMV8_DEPRECATED */ + +-void register_undef_hook(struct undef_hook *hook); +-void unregister_undef_hook(struct undef_hook *hook); + void force_signal_inject(int signal, int code, unsigned long address, unsigned long err); + void arm64_notify_segfault(unsigned long addr); + void arm64_force_sig_fault(int signo, int code, unsigned long far, const char *str); +--- a/arch/arm64/kernel/armv8_deprecated.c ++++ b/arch/arm64/kernel/armv8_deprecated.c +@@ -38,17 +38,24 @@ enum insn_emulation_mode { + enum legacy_insn_status { + INSN_DEPRECATED, + INSN_OBSOLETE, ++ INSN_UNAVAILABLE, + }; + + struct insn_emulation { + const char *name; +- struct list_head node; + enum legacy_insn_status status; +- struct undef_hook *hooks; ++ bool (*try_emulate)(struct pt_regs *regs, ++ u32 insn); + int (*set_hw_mode)(bool enable); ++ + int current_mode; + int min; + int max; ++ ++ /* ++ * sysctl for this emulation + a sentinal entry. ++ */ ++ struct ctl_table sysctl[2]; + }; + + #define ARM_OPCODE_CONDTEST_FAIL 0 +@@ -70,6 +77,7 @@ static unsigned int aarch32_check_condit + return ARM_OPCODE_CONDTEST_UNCOND; + } + ++#ifdef CONFIG_SWP_EMULATION + /* + * Implement emulation of the SWP/SWPB instructions using load-exclusive and + * store-exclusive. +@@ -228,28 +236,27 @@ fault: + return 0; + } + +-/* +- * Only emulate SWP/SWPB executed in ARM state/User mode. +- * The kernel must be SWP free and SWP{B} does not exist in Thumb. +- */ +-static struct undef_hook swp_hooks[] = { +- { +- .instr_mask = 0x0fb00ff0, +- .instr_val = 0x01000090, +- .pstate_mask = PSR_AA32_MODE_MASK, +- .pstate_val = PSR_AA32_MODE_USR, +- .fn = swp_handler +- }, +- { } +-}; ++static bool try_emulate_swp(struct pt_regs *regs, u32 insn) ++{ ++ /* SWP{B} only exists in ARM state and does not exist in Thumb */ ++ if (!compat_user_mode(regs) || compat_thumb_mode(regs)) ++ return false; ++ ++ if ((insn & 0x0fb00ff0) != 0x01000090) ++ return false; ++ ++ return swp_handler(regs, insn) == 0; ++} + + static struct insn_emulation insn_swp = { + .name = "swp", + .status = INSN_OBSOLETE, +- .hooks = swp_hooks, ++ .try_emulate = try_emulate_swp, + .set_hw_mode = NULL, + }; ++#endif /* CONFIG_SWP_EMULATION */ + ++#ifdef CONFIG_CP15_BARRIER_EMULATION + static int cp15barrier_handler(struct pt_regs *regs, u32 instr) + { + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); +@@ -312,31 +319,29 @@ static int cp15_barrier_set_hw_mode(bool + return 0; + } + +-static struct undef_hook cp15_barrier_hooks[] = { +- { +- .instr_mask = 0x0fff0fdf, +- .instr_val = 0x0e070f9a, +- .pstate_mask = PSR_AA32_MODE_MASK, +- .pstate_val = PSR_AA32_MODE_USR, +- .fn = cp15barrier_handler, +- }, +- { +- .instr_mask = 0x0fff0fff, +- .instr_val = 0x0e070f95, +- .pstate_mask = PSR_AA32_MODE_MASK, +- .pstate_val = PSR_AA32_MODE_USR, +- .fn = cp15barrier_handler, +- }, +- { } +-}; ++static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn) ++{ ++ if (!compat_user_mode(regs) || compat_thumb_mode(regs)) ++ return false; ++ ++ if ((insn & 0x0fff0fdf) == 0x0e070f9a) ++ return cp15barrier_handler(regs, insn) == 0; ++ ++ if ((insn & 0x0fff0fff) == 0x0e070f95) ++ return cp15barrier_handler(regs, insn) == 0; ++ ++ return false; ++} + + static struct insn_emulation insn_cp15_barrier = { + .name = "cp15_barrier", + .status = INSN_DEPRECATED, +- .hooks = cp15_barrier_hooks, ++ .try_emulate = try_emulate_cp15_barrier, + .set_hw_mode = cp15_barrier_set_hw_mode, + }; ++#endif /* CONFIG_CP15_BARRIER_EMULATION */ + ++#ifdef CONFIG_SETEND_EMULATION + static int setend_set_hw_mode(bool enable) + { + if (!cpu_supports_mixed_endian_el0()) +@@ -384,60 +389,40 @@ static int t16_setend_handler(struct pt_ + return rc; + } + +-static struct undef_hook setend_hooks[] = { +- { +- .instr_mask = 0xfffffdff, +- .instr_val = 0xf1010000, +- .pstate_mask = PSR_AA32_MODE_MASK, +- .pstate_val = PSR_AA32_MODE_USR, +- .fn = a32_setend_handler, +- }, +- { +- /* Thumb mode */ +- .instr_mask = 0xfffffff7, +- .instr_val = 0x0000b650, +- .pstate_mask = (PSR_AA32_T_BIT | PSR_AA32_MODE_MASK), +- .pstate_val = (PSR_AA32_T_BIT | PSR_AA32_MODE_USR), +- .fn = t16_setend_handler, +- }, +- {} +-}; ++static bool try_emulate_setend(struct pt_regs *regs, u32 insn) ++{ ++ if (compat_thumb_mode(regs) && ++ (insn & 0xfffffff7) == 0x0000b650) ++ return t16_setend_handler(regs, insn) == 0; ++ ++ if (compat_user_mode(regs) && ++ (insn & 0xfffffdff) == 0xf1010000) ++ return a32_setend_handler(regs, insn) == 0; ++ ++ return false; ++} + + static struct insn_emulation insn_setend = { + .name = "setend", + .status = INSN_DEPRECATED, +- .hooks = setend_hooks, ++ .try_emulate = try_emulate_setend, + .set_hw_mode = setend_set_hw_mode, + }; ++#endif /* CONFIG_SETEND_EMULATION */ + +-static LIST_HEAD(insn_emulation); +-static int nr_insn_emulated __initdata; +-static DEFINE_RAW_SPINLOCK(insn_emulation_lock); +-static DEFINE_MUTEX(insn_emulation_mutex); +- +-static void register_emulation_hooks(struct insn_emulation *insn) +-{ +- struct undef_hook *hook; +- +- BUG_ON(!insn->hooks); +- +- for (hook = insn->hooks; hook->instr_mask; hook++) +- register_undef_hook(hook); +- +- pr_notice("Registered %s emulation handler\n", insn->name); +-} +- +-static void remove_emulation_hooks(struct insn_emulation *insn) +-{ +- struct undef_hook *hook; +- +- BUG_ON(!insn->hooks); +- +- for (hook = insn->hooks; hook->instr_mask; hook++) +- unregister_undef_hook(hook); ++static struct insn_emulation *insn_emulations[] = { ++#ifdef CONFIG_SWP_EMULATION ++ &insn_swp, ++#endif ++#ifdef CONFIG_CP15_BARRIER_EMULATION ++ &insn_cp15_barrier, ++#endif ++#ifdef CONFIG_SETEND_EMULATION ++ &insn_setend, ++#endif ++}; + +- pr_notice("Removed %s emulation handler\n", insn->name); +-} ++static DEFINE_MUTEX(insn_emulation_mutex); + + static void enable_insn_hw_mode(void *data) + { +@@ -473,20 +458,27 @@ static int run_all_cpu_set_hw_mode(struc + */ + static int run_all_insn_set_hw_mode(unsigned int cpu) + { ++ int i; + int rc = 0; + unsigned long flags; +- struct insn_emulation *insn; + +- raw_spin_lock_irqsave(&insn_emulation_lock, flags); +- list_for_each_entry(insn, &insn_emulation, node) { +- bool enable = (insn->current_mode == INSN_HW); ++ /* ++ * Disable IRQs to serialize against an IPI from ++ * run_all_cpu_set_hw_mode(), ensuring the HW is programmed to the most ++ * recent enablement state if the two race with one another. ++ */ ++ local_irq_save(flags); ++ for (i = 0; i < ARRAY_SIZE(insn_emulations); i++) { ++ struct insn_emulation *insn = insn_emulations[i]; ++ bool enable = READ_ONCE(insn->current_mode) == INSN_HW; + if (insn->set_hw_mode && insn->set_hw_mode(enable)) { + pr_warn("CPU[%u] cannot support the emulation of %s", + cpu, insn->name); + rc = -EINVAL; + } + } +- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); ++ local_irq_restore(flags); ++ + return rc; + } + +@@ -499,7 +491,6 @@ static int update_insn_emulation_mode(st + case INSN_UNDEF: /* Nothing to be done */ + break; + case INSN_EMULATE: +- remove_emulation_hooks(insn); + break; + case INSN_HW: + if (!run_all_cpu_set_hw_mode(insn, false)) +@@ -511,7 +502,6 @@ static int update_insn_emulation_mode(st + case INSN_UNDEF: + break; + case INSN_EMULATE: +- register_emulation_hooks(insn); + break; + case INSN_HW: + ret = run_all_cpu_set_hw_mode(insn, true); +@@ -523,34 +513,6 @@ static int update_insn_emulation_mode(st + return ret; + } + +-static void __init register_insn_emulation(struct insn_emulation *insn) +-{ +- unsigned long flags; +- +- insn->min = INSN_UNDEF; +- +- switch (insn->status) { +- case INSN_DEPRECATED: +- insn->current_mode = INSN_EMULATE; +- /* Disable the HW mode if it was turned on at early boot time */ +- run_all_cpu_set_hw_mode(insn, false); +- insn->max = INSN_HW; +- break; +- case INSN_OBSOLETE: +- insn->current_mode = INSN_UNDEF; +- insn->max = INSN_EMULATE; +- break; +- } +- +- raw_spin_lock_irqsave(&insn_emulation_lock, flags); +- list_add(&insn->node, &insn_emulation); +- nr_insn_emulated++; +- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); +- +- /* Register any handlers if required */ +- update_insn_emulation_mode(insn, INSN_UNDEF); +-} +- + static int emulation_proc_handler(struct ctl_table *table, int write, + void *buffer, size_t *lenp, + loff_t *ppos) +@@ -568,7 +530,7 @@ static int emulation_proc_handler(struct + ret = update_insn_emulation_mode(insn, prev_mode); + if (ret) { + /* Mode change failed, revert to previous mode. */ +- insn->current_mode = prev_mode; ++ WRITE_ONCE(insn->current_mode, prev_mode); + update_insn_emulation_mode(insn, INSN_UNDEF); + } + ret: +@@ -576,21 +538,34 @@ ret: + return ret; + } + +-static void __init register_insn_emulation_sysctl(void) ++static void __init register_insn_emulation(struct insn_emulation *insn) + { +- unsigned long flags; +- int i = 0; +- struct insn_emulation *insn; +- struct ctl_table *insns_sysctl, *sysctl; +- +- insns_sysctl = kcalloc(nr_insn_emulated + 1, sizeof(*sysctl), +- GFP_KERNEL); +- if (!insns_sysctl) +- return; +- +- raw_spin_lock_irqsave(&insn_emulation_lock, flags); +- list_for_each_entry(insn, &insn_emulation, node) { +- sysctl = &insns_sysctl[i]; ++ struct ctl_table *sysctl; ++ ++ insn->min = INSN_UNDEF; ++ ++ switch (insn->status) { ++ case INSN_DEPRECATED: ++ insn->current_mode = INSN_EMULATE; ++ /* Disable the HW mode if it was turned on at early boot time */ ++ run_all_cpu_set_hw_mode(insn, false); ++ insn->max = INSN_HW; ++ break; ++ case INSN_OBSOLETE: ++ insn->current_mode = INSN_UNDEF; ++ insn->max = INSN_EMULATE; ++ break; ++ case INSN_UNAVAILABLE: ++ insn->current_mode = INSN_UNDEF; ++ insn->max = INSN_UNDEF; ++ break; ++ } ++ ++ /* Program the HW if required */ ++ update_insn_emulation_mode(insn, INSN_UNDEF); ++ ++ if (insn->status != INSN_UNAVAILABLE) { ++ sysctl = &insn->sysctl[0]; + + sysctl->mode = 0644; + sysctl->maxlen = sizeof(int); +@@ -600,11 +575,34 @@ static void __init register_insn_emulati + sysctl->extra1 = &insn->min; + sysctl->extra2 = &insn->max; + sysctl->proc_handler = emulation_proc_handler; +- i++; ++ ++ register_sysctl("abi", sysctl); ++ } ++} ++ ++bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(insn_emulations); i++) { ++ struct insn_emulation *ie = insn_emulations[i]; ++ ++ if (ie->status == INSN_UNAVAILABLE) ++ continue; ++ ++ /* ++ * A trap may race with the mode being changed ++ * INSN_EMULATE<->INSN_HW. Try to emulate the instruction to ++ * avoid a spurious UNDEF. ++ */ ++ if (READ_ONCE(ie->current_mode) == INSN_UNDEF) ++ continue; ++ ++ if (ie->try_emulate(regs, insn)) ++ return true; + } +- raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); + +- register_sysctl("abi", insns_sysctl); ++ return false; + } + + /* +@@ -613,24 +611,27 @@ static void __init register_insn_emulati + */ + static int __init armv8_deprecated_init(void) + { +- if (IS_ENABLED(CONFIG_SWP_EMULATION)) +- register_insn_emulation(&insn_swp); ++ int i; + +- if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION)) +- register_insn_emulation(&insn_cp15_barrier); ++#ifdef CONFIG_SETEND_EMULATION ++ if (!system_supports_mixed_endian_el0()) { ++ insn_setend.status = INSN_UNAVAILABLE; ++ pr_info("setend instruction emulation is not supported on this system\n"); ++ } + +- if (IS_ENABLED(CONFIG_SETEND_EMULATION)) { +- if (system_supports_mixed_endian_el0()) +- register_insn_emulation(&insn_setend); +- else +- pr_info("setend instruction emulation is not supported on this system\n"); ++#endif ++ for (i = 0; i < ARRAY_SIZE(insn_emulations); i++) { ++ struct insn_emulation *ie = insn_emulations[i]; ++ ++ if (ie->status == INSN_UNAVAILABLE) ++ continue; ++ ++ register_insn_emulation(ie); + } + + cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING, + "arm64/isndep:starting", + run_all_insn_set_hw_mode, NULL); +- register_insn_emulation_sysctl(); +- + return 0; + } + +--- a/arch/arm64/kernel/traps.c ++++ b/arch/arm64/kernel/traps.c +@@ -373,27 +373,6 @@ void arm64_skip_faulting_instruction(str + regs->pstate &= ~PSR_BTYPE_MASK; + } + +-static LIST_HEAD(undef_hook); +-static DEFINE_RAW_SPINLOCK(undef_lock); +- +-void register_undef_hook(struct undef_hook *hook) +-{ +- unsigned long flags; +- +- raw_spin_lock_irqsave(&undef_lock, flags); +- list_add(&hook->node, &undef_hook); +- raw_spin_unlock_irqrestore(&undef_lock, flags); +-} +- +-void unregister_undef_hook(struct undef_hook *hook) +-{ +- unsigned long flags; +- +- raw_spin_lock_irqsave(&undef_lock, flags); +- list_del(&hook->node); +- raw_spin_unlock_irqrestore(&undef_lock, flags); +-} +- + static int user_insn_read(struct pt_regs *regs, u32 *insnp) + { + u32 instr; +@@ -425,23 +404,6 @@ static int user_insn_read(struct pt_regs + return 0; + } + +-static int call_undef_hook(struct pt_regs *regs, u32 instr) +-{ +- struct undef_hook *hook; +- unsigned long flags; +- int (*fn)(struct pt_regs *regs, u32 instr) = NULL; +- +- raw_spin_lock_irqsave(&undef_lock, flags); +- list_for_each_entry(hook, &undef_hook, node) +- if ((instr & hook->instr_mask) == hook->instr_val && +- (regs->pstate & hook->pstate_mask) == hook->pstate_val) +- fn = hook->fn; +- +- raw_spin_unlock_irqrestore(&undef_lock, flags); +- +- return fn ? fn(regs, instr) : 1; +-} +- + void force_signal_inject(int signal, int code, unsigned long address, unsigned long err) + { + const char *desc; +@@ -502,7 +464,7 @@ void do_el0_undef(struct pt_regs *regs, + if (try_emulate_mrs(regs, insn)) + return; + +- if (call_undef_hook(regs, insn) == 0) ++ if (try_emulate_armv8_deprecated(regs, insn)) + return; + + out_err: diff --git a/queue-5.15/arm64-consistently-pass-esr_elx-to-die.patch b/queue-5.15/arm64-consistently-pass-esr_elx-to-die.patch new file mode 100644 index 00000000000..bdbc8f50ac9 --- /dev/null +++ b/queue-5.15/arm64-consistently-pass-esr_elx-to-die.patch @@ -0,0 +1,174 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:40 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:43 +0000 +Subject: arm64: consistently pass ESR_ELx to die() +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-4-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit 0f2cb928a1547ae8f89e80a4b8df2c6c02ae5f96 upstream. + +Currently, bug_handler() and kasan_handler() call die() with '0' as the +'err' value, whereas die_kernel_fault() passes the ESR_ELx value. + +For consistency, this patch ensures we always pass the ESR_ELx value to +die(). As this is only called for exceptions taken from kernel mode, +there should be no user-visible change as a result of this patch. + +For UNDEFINED exceptions, I've had to modify do_undefinstr() and its +callers to pass the ESR_ELx value. In all cases the ESR_ELx value had +already been read and was available. + +Signed-off-by: Mark Rutland +Cc: Mark Brown +Cc: Alexandru Elisei +Cc: Amit Daniel Kachhap +Cc: James Morse +Cc: Will Deacon +Reviewed-by: Anshuman Khandual +Reviewed-by: Mark Brown +Link: https://lore.kernel.org/r/20220913101732.3925290-4-mark.rutland@arm.com +Signed-off-by: Catalin Marinas +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/exception.h | 2 +- + arch/arm64/kernel/entry-common.c | 14 +++++++------- + arch/arm64/kernel/traps.c | 14 +++++++------- + 3 files changed, 15 insertions(+), 15 deletions(-) + +--- a/arch/arm64/include/asm/exception.h ++++ b/arch/arm64/include/asm/exception.h +@@ -58,7 +58,7 @@ asmlinkage void call_on_irq_stack(struct + asmlinkage void asm_exit_to_user_mode(struct pt_regs *regs); + + void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs); +-void do_undefinstr(struct pt_regs *regs); ++void do_undefinstr(struct pt_regs *regs, unsigned long esr); + void do_bti(struct pt_regs *regs); + void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr, + struct pt_regs *regs); +--- a/arch/arm64/kernel/entry-common.c ++++ b/arch/arm64/kernel/entry-common.c +@@ -371,11 +371,11 @@ static void noinstr el1_pc(struct pt_reg + exit_to_kernel_mode(regs); + } + +-static void noinstr el1_undef(struct pt_regs *regs) ++static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) + { + enter_from_kernel_mode(regs); + local_daif_inherit(regs); +- do_undefinstr(regs); ++ do_undefinstr(regs, esr); + local_daif_mask(); + exit_to_kernel_mode(regs); + } +@@ -417,7 +417,7 @@ asmlinkage void noinstr el1h_64_sync_han + break; + case ESR_ELx_EC_SYS64: + case ESR_ELx_EC_UNKNOWN: +- el1_undef(regs); ++ el1_undef(regs, esr); + break; + case ESR_ELx_EC_BREAKPT_CUR: + case ESR_ELx_EC_SOFTSTP_CUR: +@@ -554,11 +554,11 @@ static void noinstr el0_sp(struct pt_reg + exit_to_user_mode(regs); + } + +-static void noinstr el0_undef(struct pt_regs *regs) ++static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr) + { + enter_from_user_mode(regs); + local_daif_restore(DAIF_PROCCTX); +- do_undefinstr(regs); ++ do_undefinstr(regs, esr); + exit_to_user_mode(regs); + } + +@@ -639,7 +639,7 @@ asmlinkage void noinstr el0t_64_sync_han + el0_pc(regs, esr); + break; + case ESR_ELx_EC_UNKNOWN: +- el0_undef(regs); ++ el0_undef(regs, esr); + break; + case ESR_ELx_EC_BTI: + el0_bti(regs); +@@ -755,7 +755,7 @@ asmlinkage void noinstr el0t_32_sync_han + case ESR_ELx_EC_CP14_MR: + case ESR_ELx_EC_CP14_LS: + case ESR_ELx_EC_CP14_64: +- el0_undef(regs); ++ el0_undef(regs, esr); + break; + case ESR_ELx_EC_CP15_32: + case ESR_ELx_EC_CP15_64: +--- a/arch/arm64/kernel/traps.c ++++ b/arch/arm64/kernel/traps.c +@@ -486,7 +486,7 @@ void arm64_notify_segfault(unsigned long + force_signal_inject(SIGSEGV, code, addr, 0); + } + +-void do_undefinstr(struct pt_regs *regs) ++void do_undefinstr(struct pt_regs *regs, unsigned long esr) + { + /* check for AArch32 breakpoint instructions */ + if (!aarch32_break_handler(regs)) +@@ -496,7 +496,7 @@ void do_undefinstr(struct pt_regs *regs) + return; + + if (!user_mode(regs)) +- die("Oops - Undefined instruction", regs, 0); ++ die("Oops - Undefined instruction", regs, esr); + + force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); + } +@@ -755,7 +755,7 @@ void do_cp15instr(unsigned long esr, str + hook_base = cp15_64_hooks; + break; + default: +- do_undefinstr(regs); ++ do_undefinstr(regs, esr); + return; + } + +@@ -770,7 +770,7 @@ void do_cp15instr(unsigned long esr, str + * EL0. Fall back to our usual undefined instruction handler + * so that we handle these consistently. + */ +- do_undefinstr(regs); ++ do_undefinstr(regs, esr); + } + NOKPROBE_SYMBOL(do_cp15instr); + #endif +@@ -790,7 +790,7 @@ void do_sysinstr(unsigned long esr, stru + * back to our usual undefined instruction handler so that we handle + * these consistently. + */ +- do_undefinstr(regs); ++ do_undefinstr(regs, esr); + } + NOKPROBE_SYMBOL(do_sysinstr); + +@@ -966,7 +966,7 @@ static int bug_handler(struct pt_regs *r + { + switch (report_bug(regs->pc, regs)) { + case BUG_TRAP_TYPE_BUG: +- die("Oops - BUG", regs, 0); ++ die("Oops - BUG", regs, esr); + break; + + case BUG_TRAP_TYPE_WARN: +@@ -1034,7 +1034,7 @@ static int kasan_handler(struct pt_regs + * This is something that might be fixed at some point in the future. + */ + if (!recover) +- die("Oops - KASAN", regs, 0); ++ die("Oops - KASAN", regs, esr); + + /* If thread survives, skip over the brk instruction and continue: */ + arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); diff --git a/queue-5.15/arm64-die-pass-err-as-long.patch b/queue-5.15/arm64-die-pass-err-as-long.patch new file mode 100644 index 00000000000..defb78952f8 --- /dev/null +++ b/queue-5.15/arm64-die-pass-err-as-long.patch @@ -0,0 +1,80 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:39 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:42 +0000 +Subject: arm64: die(): pass 'err' as long +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-3-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit 18906ff9af6517c20763ed63dab602a4150794f7 upstream. + +Recently, we reworked a lot of code to consistentlt pass ESR_ELx as a +64-bit quantity. However, we missed that this can be passed into die() +and __die() as the 'err' parameter where it is truncated to a 32-bit +int. + +As notify_die() already takes 'err' as a long, this patch changes die() +and __die() to also take 'err' as a long, ensuring that the full value +of ESR_ELx is retained. + +At the same time, die() is updated to consistently log 'err' as a +zero-padded 64-bit quantity. + +Subsequent patches will pass the ESR_ELx value to die() for a number of +exceptions. + +Signed-off-by: Mark Rutland +Reviewed-by: Mark Brown +Reviewed-by: Anshuman Khandual +Cc: Alexandru Elisei +Cc: Amit Daniel Kachhap +Cc: James Morse +Cc: Will Deacon +Link: https://lore.kernel.org/r/20220913101732.3925290-3-mark.rutland@arm.com +Signed-off-by: Catalin Marinas +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/system_misc.h | 2 +- + arch/arm64/kernel/traps.c | 6 +++--- + 2 files changed, 4 insertions(+), 4 deletions(-) + +--- a/arch/arm64/include/asm/system_misc.h ++++ b/arch/arm64/include/asm/system_misc.h +@@ -18,7 +18,7 @@ + + struct pt_regs; + +-void die(const char *msg, struct pt_regs *regs, int err); ++void die(const char *msg, struct pt_regs *regs, long err); + + struct siginfo; + void arm64_notify_die(const char *str, struct pt_regs *regs, +--- a/arch/arm64/kernel/traps.c ++++ b/arch/arm64/kernel/traps.c +@@ -181,12 +181,12 @@ static void dump_kernel_instr(const char + + #define S_SMP " SMP" + +-static int __die(const char *str, int err, struct pt_regs *regs) ++static int __die(const char *str, long err, struct pt_regs *regs) + { + static int die_counter; + int ret; + +- pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n", ++ pr_emerg("Internal error: %s: %016lx [#%d]" S_PREEMPT S_SMP "\n", + str, err, ++die_counter); + + /* trap and error numbers are mostly meaningless on ARM */ +@@ -207,7 +207,7 @@ static DEFINE_RAW_SPINLOCK(die_lock); + /* + * This function is protected against re-entrancy. + */ +-void die(const char *str, struct pt_regs *regs, int err) ++void die(const char *str, struct pt_regs *regs, long err) + { + int ret; + unsigned long flags; diff --git a/queue-5.15/arm64-factor-insn-read-out-of-call_undef_hook.patch b/queue-5.15/arm64-factor-insn-read-out-of-call_undef_hook.patch new file mode 100644 index 00000000000..fcf6db3c067 --- /dev/null +++ b/queue-5.15/arm64-factor-insn-read-out-of-call_undef_hook.patch @@ -0,0 +1,118 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:46 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:49 +0000 +Subject: arm64: factor insn read out of call_undef_hook() +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-10-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit dbfbd87efa79575491af0ba1a87bf567eaea6cae upstream. + +Subsequent patches will rework EL0 UNDEF handling, removing the need for +struct undef_hook and call_undef_hook. In preparation for those changes, +this patch factors the logic for reading user instructions out of +call_undef_hook() and into a new user_insn_read() helper, matching the +style of the existing aarch64_insn_read() helper used for reading kernel +instructions. + +There should be no functional change as a result of this patch. + +Signed-off-by: Mark Rutland +Cc: Catalin Marinas +Cc: James Morse +Cc: Joey Gouly +Cc: Peter Zijlstra +Cc: Will Deacon +Link: https://lore.kernel.org/r/20221019144123.612388-5-mark.rutland@arm.com +Signed-off-by: Will Deacon +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/kernel/traps.c | 31 ++++++++++++++++++++++--------- + 1 file changed, 22 insertions(+), 9 deletions(-) + +--- a/arch/arm64/kernel/traps.c ++++ b/arch/arm64/kernel/traps.c +@@ -394,25 +394,22 @@ void unregister_undef_hook(struct undef_ + raw_spin_unlock_irqrestore(&undef_lock, flags); + } + +-static int call_undef_hook(struct pt_regs *regs) ++static int user_insn_read(struct pt_regs *regs, u32 *insnp) + { +- struct undef_hook *hook; +- unsigned long flags; + u32 instr; +- int (*fn)(struct pt_regs *regs, u32 instr) = NULL; + void __user *pc = (void __user *)instruction_pointer(regs); + + if (compat_thumb_mode(regs)) { + /* 16-bit Thumb instruction */ + __le16 instr_le; + if (get_user(instr_le, (__le16 __user *)pc)) +- goto exit; ++ return -EFAULT; + instr = le16_to_cpu(instr_le); + if (aarch32_insn_is_wide(instr)) { + u32 instr2; + + if (get_user(instr_le, (__le16 __user *)(pc + 2))) +- goto exit; ++ return -EFAULT; + instr2 = le16_to_cpu(instr_le); + instr = (instr << 16) | instr2; + } +@@ -420,10 +417,20 @@ static int call_undef_hook(struct pt_reg + /* 32-bit ARM instruction */ + __le32 instr_le; + if (get_user(instr_le, (__le32 __user *)pc)) +- goto exit; ++ return -EFAULT; + instr = le32_to_cpu(instr_le); + } + ++ *insnp = instr; ++ return 0; ++} ++ ++static int call_undef_hook(struct pt_regs *regs, u32 instr) ++{ ++ struct undef_hook *hook; ++ unsigned long flags; ++ int (*fn)(struct pt_regs *regs, u32 instr) = NULL; ++ + raw_spin_lock_irqsave(&undef_lock, flags); + list_for_each_entry(hook, &undef_hook, node) + if ((instr & hook->instr_mask) == hook->instr_val && +@@ -431,7 +438,7 @@ static int call_undef_hook(struct pt_reg + fn = hook->fn; + + raw_spin_unlock_irqrestore(&undef_lock, flags); +-exit: ++ + return fn ? fn(regs, instr) : 1; + } + +@@ -483,13 +490,19 @@ void arm64_notify_segfault(unsigned long + + void do_el0_undef(struct pt_regs *regs, unsigned long esr) + { ++ u32 insn; ++ + /* check for AArch32 breakpoint instructions */ + if (!aarch32_break_handler(regs)) + return; + +- if (call_undef_hook(regs) == 0) ++ if (user_insn_read(regs, &insn)) ++ goto out_err; ++ ++ if (call_undef_hook(regs, insn) == 0) + return; + ++out_err: + force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); + } + diff --git a/queue-5.15/arm64-factor-out-el1-ssbs-emulation-hook.patch b/queue-5.15/arm64-factor-out-el1-ssbs-emulation-hook.patch new file mode 100644 index 00000000000..0a660c7303a --- /dev/null +++ b/queue-5.15/arm64-factor-out-el1-ssbs-emulation-hook.patch @@ -0,0 +1,167 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:46 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:48 +0000 +Subject: arm64: factor out EL1 SSBS emulation hook +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-9-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit bff8f413c71ffc3cb679dbd9a5632b33af563f9f upstream. + +Currently call_undef_hook() is used to handle UNDEFINED exceptions from +EL0 and EL1. As support for deprecated instructions may be enabled +independently, the handlers for individual instructions are organised as +a linked list of struct undef_hook which can be manipulated dynamically. +As this can be manipulated dynamically, the list is protected with a +raw_spinlock which must be acquired when handling UNDEFINED exceptions +or when manipulating the list of handlers. + +This locking is unfortunate as it serialises handling of UNDEFINED +exceptions, and requires RCU to be enabled for lockdep, requiring the +use of RCU_NONIDLE() in resume path of cpu_suspend() since commit: + + a2c42bbabbe260b7 ("arm64: spectre: Prevent lockdep splat on v4 mitigation enable path") + +The list of UNDEFINED handlers largely consist of handlers for +exceptions taken from EL0, and the only handler for exceptions taken +from EL1 handles `MSR SSBS, #imm` on CPUs which feature PSTATE.SSBS but +lack the corresponding MSR (Immediate) instruction. Other than this we +never expect to take an UNDEFINED exception from EL1 in normal +operation. + +This patch reworks do_el0_undef() to invoke the EL1 SSBS handler +directly, relegating call_undef_hook() to only handle EL0 UNDEFs. This +removes redundant work to iterate the list for EL1 UNDEFs, and removes +the need for locking, permitting EL1 UNDEFs to be handled in parallel +without contention. + +The RCU_NONIDLE() call in cpu_suspend() will be removed in a subsequent +patch, as there are other potential issues with the use of +instrumentable code and RCU in the CPU suspend code. + +I've tested this by forcing the detection of SSBS on a CPU that doesn't +have it, and verifying that the try_emulate_el1_ssbs() callback is +invoked. + +Signed-off-by: Mark Rutland +Cc: Catalin Marinas +Cc: James Morse +Cc: Joey Gouly +Cc: Peter Zijlstra +Cc: Will Deacon +Link: https://lore.kernel.org/r/20221019144123.612388-4-mark.rutland@arm.com +Signed-off-by: Will Deacon +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/spectre.h | 2 ++ + arch/arm64/kernel/proton-pack.c | 26 +++++++------------------- + arch/arm64/kernel/traps.c | 15 ++++++++------- + 3 files changed, 17 insertions(+), 26 deletions(-) + +--- a/arch/arm64/include/asm/spectre.h ++++ b/arch/arm64/include/asm/spectre.h +@@ -26,6 +26,7 @@ enum mitigation_state { + SPECTRE_VULNERABLE, + }; + ++struct pt_regs; + struct task_struct; + + /* +@@ -98,5 +99,6 @@ enum mitigation_state arm64_get_spectre_ + bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope); + u8 spectre_bhb_loop_affected(int scope); + void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused); ++bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr); + #endif /* __ASSEMBLY__ */ + #endif /* __ASM_SPECTRE_H */ +--- a/arch/arm64/kernel/proton-pack.c ++++ b/arch/arm64/kernel/proton-pack.c +@@ -521,10 +521,13 @@ bool has_spectre_v4(const struct arm64_c + return state != SPECTRE_UNAFFECTED; + } + +-static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr) ++bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr) + { +- if (user_mode(regs)) +- return 1; ++ const u32 instr_mask = ~(1U << PSTATE_Imm_shift); ++ const u32 instr_val = 0xd500401f | PSTATE_SSBS; ++ ++ if ((instr & instr_mask) != instr_val) ++ return false; + + if (instr & BIT(PSTATE_Imm_shift)) + regs->pstate |= PSR_SSBS_BIT; +@@ -532,19 +535,11 @@ static int ssbs_emulation_handler(struct + regs->pstate &= ~PSR_SSBS_BIT; + + arm64_skip_faulting_instruction(regs, 4); +- return 0; ++ return true; + } + +-static struct undef_hook ssbs_emulation_hook = { +- .instr_mask = ~(1U << PSTATE_Imm_shift), +- .instr_val = 0xd500401f | PSTATE_SSBS, +- .fn = ssbs_emulation_handler, +-}; +- + static enum mitigation_state spectre_v4_enable_hw_mitigation(void) + { +- static bool undef_hook_registered = false; +- static DEFINE_RAW_SPINLOCK(hook_lock); + enum mitigation_state state; + + /* +@@ -555,13 +550,6 @@ static enum mitigation_state spectre_v4_ + if (state != SPECTRE_MITIGATED || !this_cpu_has_cap(ARM64_SSBS)) + return state; + +- raw_spin_lock(&hook_lock); +- if (!undef_hook_registered) { +- register_undef_hook(&ssbs_emulation_hook); +- undef_hook_registered = true; +- } +- raw_spin_unlock(&hook_lock); +- + if (spectre_v4_mitigations_off()) { + sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS); + set_pstate_ssbs(1); +--- a/arch/arm64/kernel/traps.c ++++ b/arch/arm64/kernel/traps.c +@@ -402,12 +402,7 @@ static int call_undef_hook(struct pt_reg + int (*fn)(struct pt_regs *regs, u32 instr) = NULL; + void __user *pc = (void __user *)instruction_pointer(regs); + +- if (!user_mode(regs)) { +- __le32 instr_le; +- if (get_kernel_nofault(instr_le, (__force __le32 *)pc)) +- goto exit; +- instr = le32_to_cpu(instr_le); +- } else if (compat_thumb_mode(regs)) { ++ if (compat_thumb_mode(regs)) { + /* 16-bit Thumb instruction */ + __le16 instr_le; + if (get_user(instr_le, (__le16 __user *)pc)) +@@ -500,9 +495,15 @@ void do_el0_undef(struct pt_regs *regs, + + void do_el1_undef(struct pt_regs *regs, unsigned long esr) + { +- if (call_undef_hook(regs) == 0) ++ u32 insn; ++ ++ if (aarch64_insn_read((void *)regs->pc, &insn)) ++ goto out_err; ++ ++ if (try_emulate_el1_ssbs(regs, insn)) + return; + ++out_err: + die("Oops - Undefined instruction", regs, esr); + } + diff --git a/queue-5.15/arm64-report-el1-undefs-better.patch b/queue-5.15/arm64-report-el1-undefs-better.patch new file mode 100644 index 00000000000..69903323916 --- /dev/null +++ b/queue-5.15/arm64-report-el1-undefs-better.patch @@ -0,0 +1,113 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:39 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:41 +0000 +Subject: arm64: report EL1 UNDEFs better +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-2-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit b502c87d2a26c349acbc231ff2acd6f17147926b upstream. + +If an UNDEFINED exception is taken from EL1, and do_undefinstr() doesn't +find any suitable undef_hook, it will call: + + BUG_ON(!user_mode(regs)) + +... and the kernel will report a failure witin do_undefinstr() rather +than reporting the original context that the UNDEFINED exception was +taken from. The pt_regs and ESR value reported within the BUG() handler +will be from within do_undefinstr() and the code dump will be for the +BRK in BUG_ON(), which isn't sufficient to debug the cause of the +original exception. + +This patch makes the reporting better by having do_undefinstr() call +die() directly in this case to report the original context from which +the UNDEFINED exception was taken. + +Prior to this patch, an undefined instruction is reported as: + +| kernel BUG at arch/arm64/kernel/traps.c:497! +| Internal error: Oops - BUG: 0 [#1] PREEMPT SMP +| Modules linked in: +| CPU: 0 PID: 0 Comm: swapper Not tainted 5.19.0-rc3-00127-geff044f1b04e-dirty #3 +| Hardware name: linux,dummy-virt (DT) +| pstate: 000000c5 (nzcv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--) +| pc : do_undefinstr+0x28c/0x2ac +| lr : do_undefinstr+0x298/0x2ac +| sp : ffff800009f63bc0 +| x29: ffff800009f63bc0 x28: ffff800009f73c00 x27: ffff800009644a70 +| x26: ffff8000096778a8 x25: 0000000000000040 x24: 0000000000000000 +| x23: 00000000800000c5 x22: ffff800009894060 x21: ffff800009f63d90 +| x20: 0000000000000000 x19: ffff800009f63c40 x18: 0000000000000006 +| x17: 0000000000403000 x16: 00000000bfbfd000 x15: ffff800009f63830 +| x14: ffffffffffffffff x13: 0000000000000000 x12: 0000000000000019 +| x11: 0101010101010101 x10: 0000000000161b98 x9 : 0000000000000000 +| x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000000 +| x5 : ffff800009f761d0 x4 : 0000000000000000 x3 : ffff80000a2b80f8 +| x2 : 0000000000000000 x1 : ffff800009f73c00 x0 : 00000000800000c5 +| Call trace: +| do_undefinstr+0x28c/0x2ac +| el1_undef+0x2c/0x4c +| el1h_64_sync_handler+0x84/0xd0 +| el1h_64_sync+0x64/0x68 +| setup_arch+0x550/0x598 +| start_kernel+0x88/0x6ac +| __primary_switched+0xb8/0xc0 +| Code: 17ffff95 a9425bf5 17ffffb8 a9025bf5 (d4210000) + +With this patch applied, an undefined instruction is reported as: + +| Internal error: Oops - Undefined instruction: 0 [#1] PREEMPT SMP +| Modules linked in: +| CPU: 0 PID: 0 Comm: swapper Not tainted 5.19.0-rc3-00128-gf27cfcc80e52-dirty #5 +| Hardware name: linux,dummy-virt (DT) +| pstate: 800000c5 (Nzcv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--) +| pc : setup_arch+0x550/0x598 +| lr : setup_arch+0x50c/0x598 +| sp : ffff800009f63d90 +| x29: ffff800009f63d90 x28: 0000000081000200 x27: ffff800009644a70 +| x26: ffff8000096778c8 x25: 0000000000000040 x24: 0000000000000000 +| x23: 0000000000000100 x22: ffff800009f69a58 x21: ffff80000a2b80b8 +| x20: 0000000000000000 x19: 0000000000000000 x18: 0000000000000006 +| x17: 0000000000403000 x16: 00000000bfbfd000 x15: ffff800009f63830 +| x14: ffffffffffffffff x13: 0000000000000000 x12: 0000000000000019 +| x11: 0101010101010101 x10: 0000000000161b98 x9 : 0000000000000000 +| x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000000 +| x5 : 0000000000000008 x4 : 0000000000000010 x3 : 0000000000000000 +| x2 : 0000000000000000 x1 : 0000000000000000 x0 : 0000000000000000 +| Call trace: +| setup_arch+0x550/0x598 +| start_kernel+0x88/0x6ac +| __primary_switched+0xb8/0xc0 +| Code: b4000080 90ffed80 912ac000 97db745f (00000000) + +Signed-off-by: Mark Rutland +Reviewed-by: Mark Brown +Cc: Alexandru Elisei +Cc: Amit Daniel Kachhap +Cc: James Morse +Cc: Will Deacon +Reviewed-by: Anshuman Khandual +Link: https://lore.kernel.org/r/20220913101732.3925290-2-mark.rutland@arm.com +Signed-off-by: Catalin Marinas +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/kernel/traps.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +--- a/arch/arm64/kernel/traps.c ++++ b/arch/arm64/kernel/traps.c +@@ -495,7 +495,9 @@ void do_undefinstr(struct pt_regs *regs) + if (call_undef_hook(regs) == 0) + return; + +- BUG_ON(!user_mode(regs)); ++ if (!user_mode(regs)) ++ die("Oops - Undefined instruction", regs, 0); ++ + force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); + } + NOKPROBE_SYMBOL(do_undefinstr); diff --git a/queue-5.15/arm64-rework-bti-exception-handling.patch b/queue-5.15/arm64-rework-bti-exception-handling.patch new file mode 100644 index 00000000000..6af0b0a83df --- /dev/null +++ b/queue-5.15/arm64-rework-bti-exception-handling.patch @@ -0,0 +1,185 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:42 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:45 +0000 +Subject: arm64: rework BTI exception handling +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-6-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit 830a2a4d853f2c4a1e4606aa03341b7f273b0e9b upstream. + +If a BTI exception is taken from EL1, the entry code will treat this as +an unhandled exception and will panic() the kernel. This is inconsistent +with the way we handle FPAC exceptions, which have a dedicated handler +and only necessarily kill the thread from which the exception was taken +from, and we don't log all the information that could be relevant to +debug the issue. + +The code in do_bti() has: + + BUG_ON(!user_mode(regs)); + +... and it seems like the intent was to call this for EL1 BTI +exceptions, as with FPAC, but this was omitted due to an oversight. + +This patch adds separate EL0 and EL1 BTI exception handlers, with the +latter calling die() directly to report the original context the BTI +exception was taken from. This matches our handling of FPAC exceptions. + +Prior to this patch, a BTI failure is reported as: + +| Unhandled 64-bit el1h sync exception on CPU0, ESR 0x0000000034000002 -- BTI +| CPU: 0 PID: 1 Comm: swapper/0 Not tainted 5.19.0-rc3-00131-g7d937ff0221d-dirty #9 +| Hardware name: linux,dummy-virt (DT) +| pstate: 20400809 (nzCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=-c) +| pc : test_bti_callee+0x4/0x10 +| lr : test_bti_caller+0x1c/0x28 +| sp : ffff80000800bdf0 +| x29: ffff80000800bdf0 x28: 0000000000000000 x27: 0000000000000000 +| x26: 0000000000000000 x25: 0000000000000000 x24: 0000000000000000 +| x23: ffff80000a2b8000 x22: 0000000000000000 x21: 0000000000000000 +| x20: ffff8000099fa5b0 x19: ffff800009ff7000 x18: fffffbfffda37000 +| x17: 3120676e696d7573 x16: 7361202c6e6f6974 x15: 0000000041a90000 +| x14: 0040000000000041 x13: 0040000000000001 x12: ffff000001a90000 +| x11: fffffbfffda37480 x10: 0068000000000703 x9 : 0001000040000000 +| x8 : 0000000000090000 x7 : 0068000000000f03 x6 : 0060000000000f83 +| x5 : ffff80000a2b6000 x4 : ffff0000028d0000 x3 : ffff800009f78378 +| x2 : 0000000000000000 x1 : 0000000040210000 x0 : ffff8000080257e4 +| Kernel panic - not syncing: Unhandled exception +| CPU: 0 PID: 1 Comm: swapper/0 Not tainted 5.19.0-rc3-00131-g7d937ff0221d-dirty #9 +| Hardware name: linux,dummy-virt (DT) +| Call trace: +| dump_backtrace.part.0+0xcc/0xe0 +| show_stack+0x18/0x5c +| dump_stack_lvl+0x64/0x80 +| dump_stack+0x18/0x34 +| panic+0x170/0x360 +| arm64_exit_nmi.isra.0+0x0/0x80 +| el1h_64_sync_handler+0x64/0xd0 +| el1h_64_sync+0x64/0x68 +| test_bti_callee+0x4/0x10 +| smp_cpus_done+0xb0/0xbc +| smp_init+0x7c/0x8c +| kernel_init_freeable+0x128/0x28c +| kernel_init+0x28/0x13c +| ret_from_fork+0x10/0x20 + +With this patch applied, a BTI failure is reported as: + +| Internal error: Oops - BTI: 0000000034000002 [#1] PREEMPT SMP +| Modules linked in: +| CPU: 0 PID: 1 Comm: swapper/0 Not tainted 5.19.0-rc3-00132-g0ad98265d582-dirty #8 +| Hardware name: linux,dummy-virt (DT) +| pstate: 20400809 (nzCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=-c) +| pc : test_bti_callee+0x4/0x10 +| lr : test_bti_caller+0x1c/0x28 +| sp : ffff80000800bdf0 +| x29: ffff80000800bdf0 x28: 0000000000000000 x27: 0000000000000000 +| x26: 0000000000000000 x25: 0000000000000000 x24: 0000000000000000 +| x23: ffff80000a2b8000 x22: 0000000000000000 x21: 0000000000000000 +| x20: ffff8000099fa5b0 x19: ffff800009ff7000 x18: fffffbfffda37000 +| x17: 3120676e696d7573 x16: 7361202c6e6f6974 x15: 0000000041a90000 +| x14: 0040000000000041 x13: 0040000000000001 x12: ffff000001a90000 +| x11: fffffbfffda37480 x10: 0068000000000703 x9 : 0001000040000000 +| x8 : 0000000000090000 x7 : 0068000000000f03 x6 : 0060000000000f83 +| x5 : ffff80000a2b6000 x4 : ffff0000028d0000 x3 : ffff800009f78378 +| x2 : 0000000000000000 x1 : 0000000040210000 x0 : ffff800008025804 +| Call trace: +| test_bti_callee+0x4/0x10 +| smp_cpus_done+0xb0/0xbc +| smp_init+0x7c/0x8c +| kernel_init_freeable+0x128/0x28c +| kernel_init+0x28/0x13c +| ret_from_fork+0x10/0x20 +| Code: d50323bf d53cd040 d65f03c0 d503233f (d50323bf) + +Signed-off-by: Mark Rutland +Reviewed-by: Mark Brown +Reviewed-by: Anshuman Khandual +Cc: Alexandru Elisei +Cc: Amit Daniel Kachhap +Cc: James Morse +Cc: Will Deacon +Link: https://lore.kernel.org/r/20220913101732.3925290-6-mark.rutland@arm.com +Signed-off-by: Catalin Marinas +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/exception.h | 3 ++- + arch/arm64/kernel/entry-common.c | 14 +++++++++++++- + arch/arm64/kernel/traps.c | 10 +++++++--- + 3 files changed, 22 insertions(+), 5 deletions(-) + +--- a/arch/arm64/include/asm/exception.h ++++ b/arch/arm64/include/asm/exception.h +@@ -59,7 +59,8 @@ asmlinkage void asm_exit_to_user_mode(st + + void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs); + void do_undefinstr(struct pt_regs *regs, unsigned long esr); +-void do_bti(struct pt_regs *regs); ++void do_el0_bti(struct pt_regs *regs); ++void do_el1_bti(struct pt_regs *regs, unsigned long esr); + void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr, + struct pt_regs *regs); + void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs); +--- a/arch/arm64/kernel/entry-common.c ++++ b/arch/arm64/kernel/entry-common.c +@@ -380,6 +380,15 @@ static void noinstr el1_undef(struct pt_ + exit_to_kernel_mode(regs); + } + ++static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr) ++{ ++ enter_from_kernel_mode(regs); ++ local_daif_inherit(regs); ++ do_el1_bti(regs, esr); ++ local_daif_mask(); ++ exit_to_kernel_mode(regs); ++} ++ + static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr) + { + unsigned long far = read_sysreg(far_el1); +@@ -419,6 +428,9 @@ asmlinkage void noinstr el1h_64_sync_han + case ESR_ELx_EC_UNKNOWN: + el1_undef(regs, esr); + break; ++ case ESR_ELx_EC_BTI: ++ el1_bti(regs, esr); ++ break; + case ESR_ELx_EC_BREAKPT_CUR: + case ESR_ELx_EC_SOFTSTP_CUR: + case ESR_ELx_EC_WATCHPT_CUR: +@@ -566,7 +578,7 @@ static void noinstr el0_bti(struct pt_re + { + enter_from_user_mode(regs); + local_daif_restore(DAIF_PROCCTX); +- do_bti(regs); ++ do_el0_bti(regs); + exit_to_user_mode(regs); + } + +--- a/arch/arm64/kernel/traps.c ++++ b/arch/arm64/kernel/traps.c +@@ -502,12 +502,16 @@ void do_undefinstr(struct pt_regs *regs, + } + NOKPROBE_SYMBOL(do_undefinstr); + +-void do_bti(struct pt_regs *regs) ++void do_el0_bti(struct pt_regs *regs) + { +- BUG_ON(!user_mode(regs)); + force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); + } +-NOKPROBE_SYMBOL(do_bti); ++ ++void do_el1_bti(struct pt_regs *regs, unsigned long esr) ++{ ++ die("Oops - BTI", regs, esr); ++} ++NOKPROBE_SYMBOL(do_el1_bti); + + void do_el0_fpac(struct pt_regs *regs, unsigned long esr) + { diff --git a/queue-5.15/arm64-rework-el0-mrs-emulation.patch b/queue-5.15/arm64-rework-el0-mrs-emulation.patch new file mode 100644 index 00000000000..b2536b9d508 --- /dev/null +++ b/queue-5.15/arm64-rework-el0-mrs-emulation.patch @@ -0,0 +1,144 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:47 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:50 +0000 +Subject: arm64: rework EL0 MRS emulation +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-11-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit f5962add74b61f8ae31c6311f75ca35d7e1d2d8f upstream. + +On CPUs without FEAT_IDST, ID register emulation is slower than it needs +to be, as all threads contend for the same lock to perform the +emulation. This patch reworks the emulation to avoid this unnecessary +contention. + +On CPUs with FEAT_IDST (which is mandatory from ARMv8.4 onwards), EL0 +accesses to ID registers result in a SYS trap, and emulation of these is +handled with a sys64_hook. These hooks are statically allocated, and no +locking is required to iterate through the hooks and perform the +emulation, allowing emulation to occur in parallel with no contention. + +On CPUs without FEAT_IDST, EL0 accesses to ID registers result in an +UNDEFINED exception, and emulation of these accesses is handled with an +undef_hook. When an EL0 MRS instruction is trapped to EL1, the kernel +finds the relevant handler by iterating through all of the undef_hooks, +requiring undef_lock to be held during this lookup. + +This locking is only required to safely traverse the list of undef_hooks +(as it can be concurrently modified), and the actual emulation of the +MRS does not require any mutual exclusion. This locking is an +unfortunate bottleneck, especially given that MRS emulation is enabled +unconditionally and is never disabled. + +This patch reworks the non-FEAT_IDST MRS emulation logic so that it can +be invoked directly from do_el0_undef(). This removes the bottleneck, +allowing MRS traps to be handled entirely in parallel, and is a stepping +stone to making all of the undef_hooks lock-free. + +I've tested this in a 64-vCPU VM on a 64-CPU ThunderX2 host, with a +benchmark which spawns a number of threads which each try to read +ID_AA64ISAR0_EL1 1000000 times. This is vastly more contention than will +ever be seen in realistic usage, but clearly demonstrates the removal of +the bottleneck: + + | Threads || Time (seconds) | + | || Before || After | + | || Real | System || Real | System | + |---------++--------+---------++--------+---------| + | 1 || 0.29 | 0.20 || 0.24 | 0.12 | + | 2 || 0.35 | 0.51 || 0.23 | 0.27 | + | 4 || 1.08 | 3.87 || 0.24 | 0.56 | + | 8 || 4.31 | 33.60 || 0.24 | 1.11 | + | 16 || 9.47 | 149.39 || 0.23 | 2.15 | + | 32 || 19.07 | 605.27 || 0.24 | 4.38 | + | 64 || 65.40 | 3609.09 || 0.33 | 11.27 | + +Aside from the speedup, there should be no functional change as a result +of this patch. + +Signed-off-by: Mark Rutland +Cc: Catalin Marinas +Cc: James Morse +Cc: Joey Gouly +Cc: Peter Zijlstra +Cc: Will Deacon +Link: https://lore.kernel.org/r/20221019144123.612388-6-mark.rutland@arm.com +Signed-off-by: Will Deacon +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/cpufeature.h | 3 ++- + arch/arm64/kernel/cpufeature.c | 23 +++++------------------ + arch/arm64/kernel/traps.c | 3 +++ + 3 files changed, 10 insertions(+), 19 deletions(-) + +--- a/arch/arm64/include/asm/cpufeature.h ++++ b/arch/arm64/include/asm/cpufeature.h +@@ -808,7 +808,8 @@ static inline bool system_supports_tlb_r + cpus_have_const_cap(ARM64_HAS_TLB_RANGE); + } + +-extern int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt); ++int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt); ++bool try_emulate_mrs(struct pt_regs *regs, u32 isn); + + static inline u32 id_aa64mmfr0_parange_to_phys_shift(int parange) + { +--- a/arch/arm64/kernel/cpufeature.c ++++ b/arch/arm64/kernel/cpufeature.c +@@ -3106,35 +3106,22 @@ int do_emulate_mrs(struct pt_regs *regs, + return rc; + } + +-static int emulate_mrs(struct pt_regs *regs, u32 insn) ++bool try_emulate_mrs(struct pt_regs *regs, u32 insn) + { + u32 sys_reg, rt; + ++ if (compat_user_mode(regs) || !aarch64_insn_is_mrs(insn)) ++ return false; ++ + /* + * sys_reg values are defined as used in mrs/msr instruction. + * shift the imm value to get the encoding. + */ + sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5; + rt = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn); +- return do_emulate_mrs(regs, sys_reg, rt); +-} +- +-static struct undef_hook mrs_hook = { +- .instr_mask = 0xffff0000, +- .instr_val = 0xd5380000, +- .pstate_mask = PSR_AA32_MODE_MASK, +- .pstate_val = PSR_MODE_EL0t, +- .fn = emulate_mrs, +-}; +- +-static int __init enable_mrs_emulation(void) +-{ +- register_undef_hook(&mrs_hook); +- return 0; ++ return do_emulate_mrs(regs, sys_reg, rt) == 0; + } + +-core_initcall(enable_mrs_emulation); +- + enum mitigation_state arm64_get_meltdown_state(void) + { + if (__meltdown_safe) +--- a/arch/arm64/kernel/traps.c ++++ b/arch/arm64/kernel/traps.c +@@ -499,6 +499,9 @@ void do_el0_undef(struct pt_regs *regs, + if (user_insn_read(regs, &insn)) + goto out_err; + ++ if (try_emulate_mrs(regs, insn)) ++ return; ++ + if (call_undef_hook(regs, insn) == 0) + return; + diff --git a/queue-5.15/arm64-rework-fpac-exception-handling.patch b/queue-5.15/arm64-rework-fpac-exception-handling.patch new file mode 100644 index 00000000000..36a968d3220 --- /dev/null +++ b/queue-5.15/arm64-rework-fpac-exception-handling.patch @@ -0,0 +1,173 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:07:41 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:44 +0000 +Subject: arm64: rework FPAC exception handling +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-5-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit a1fafa3b24a70461bbf3e5c0770893feb0a49292 upstream. + +If an FPAC exception is taken from EL1, the entry code will call +do_ptrauth_fault(), where due to: + + BUG_ON(!user_mode(regs)) + +... the kernel will report a problem within do_ptrauth_fault() rather +than reporting the original context the FPAC exception was taken from. +The pt_regs and ESR value reported will be from within +do_ptrauth_fault() and the code dump will be for the BRK in BUG_ON(), +which isn't sufficient to debug the cause of the original exception. + +This patch makes the reporting better by having separate EL0 and EL1 +FPAC exception handlers, with the latter calling die() directly to +report the original context the FPAC exception was taken from. + +Note that we only need to prevent kprobes of the EL1 FPAC handler, since +the EL0 FPAC handler cannot be called recursively. + +For consistency with do_el0_svc*(), I've named the split functions +do_el{0,1}_fpac() rather than do_el{0,1}_ptrauth_fault(). I've also +clarified the comment to not imply there are casues other than FPAC +exceptions. + +Prior to this patch FPAC exceptions are reported as: + +| kernel BUG at arch/arm64/kernel/traps.c:517! +| Internal error: Oops - BUG: 00000000f2000800 [#1] PREEMPT SMP +| Modules linked in: +| CPU: 0 PID: 1 Comm: swapper/0 Not tainted 5.19.0-rc3-00130-g9c8a180a1cdf-dirty #12 +| Hardware name: FVP Base RevC (DT) +| pstate: 00400009 (nzcv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) +| pc : do_ptrauth_fault+0x3c/0x40 +| lr : el1_fpac+0x34/0x54 +| sp : ffff80000a3bbc80 +| x29: ffff80000a3bbc80 x28: ffff0008001d8000 x27: 0000000000000000 +| x26: 0000000000000000 x25: 0000000000000000 x24: 0000000000000000 +| x23: 0000000020400009 x22: ffff800008f70fa4 x21: ffff80000a3bbe00 +| x20: 0000000072000000 x19: ffff80000a3bbcb0 x18: fffffbfffda37000 +| x17: 3120676e696d7573 x16: 7361202c6e6f6974 x15: 0000000081a90000 +| x14: 0040000000000041 x13: 0040000000000001 x12: ffff000001a90000 +| x11: fffffbfffda37480 x10: 0068000000000703 x9 : 0001000080000000 +| x8 : 0000000000090000 x7 : 0068000000000f03 x6 : 0060000000000783 +| x5 : ffff80000a3bbcb0 x4 : ffff0008001d8000 x3 : 0000000072000000 +| x2 : 0000000000000000 x1 : 0000000020400009 x0 : ffff80000a3bbcb0 +| Call trace: +| do_ptrauth_fault+0x3c/0x40 +| el1h_64_sync_handler+0xc4/0xd0 +| el1h_64_sync+0x64/0x68 +| test_pac+0x8/0x10 +| smp_init+0x7c/0x8c +| kernel_init_freeable+0x128/0x28c +| kernel_init+0x28/0x13c +| ret_from_fork+0x10/0x20 +| Code: 97fffe5e a8c17bfd d50323bf d65f03c0 (d4210000) + +With this patch applied FPAC exceptions are reported as: + +| Internal error: Oops - FPAC: 0000000072000000 [#1] PREEMPT SMP +| Modules linked in: +| CPU: 0 PID: 1 Comm: swapper/0 Not tainted 5.19.0-rc3-00132-g78846e1c4757-dirty #11 +| Hardware name: FVP Base RevC (DT) +| pstate: 20400009 (nzCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) +| pc : test_pac+0x8/0x10 +| lr : 0x0 +| sp : ffff80000a3bbe00 +| x29: ffff80000a3bbe00 x28: 0000000000000000 x27: 0000000000000000 +| x26: 0000000000000000 x25: 0000000000000000 x24: 0000000000000000 +| x23: ffff80000a2c8000 x22: 0000000000000000 x21: 0000000000000000 +| x20: ffff8000099fa5b0 x19: ffff80000a007000 x18: fffffbfffda37000 +| x17: 3120676e696d7573 x16: 7361202c6e6f6974 x15: 0000000081a90000 +| x14: 0040000000000041 x13: 0040000000000001 x12: ffff000001a90000 +| x11: fffffbfffda37480 x10: 0068000000000703 x9 : 0001000080000000 +| x8 : 0000000000090000 x7 : 0068000000000f03 x6 : 0060000000000783 +| x5 : ffff80000a2c6000 x4 : ffff0008001d8000 x3 : ffff800009f88378 +| x2 : 0000000000000000 x1 : 0000000080210000 x0 : ffff000001a90000 +| Call trace: +| test_pac+0x8/0x10 +| smp_init+0x7c/0x8c +| kernel_init_freeable+0x128/0x28c +| kernel_init+0x28/0x13c +| ret_from_fork+0x10/0x20 +| Code: d50323bf d65f03c0 d503233f aa1f03fe (d50323bf) + +Signed-off-by: Mark Rutland +Reviewed-by: Mark Brown +Reviewed-by: Anshuman Khandual +Cc: Alexandru Elisei +Cc: Amit Daniel Kachhap +Cc: James Morse +Cc: Will Deacon +Link: https://lore.kernel.org/r/20220913101732.3925290-5-mark.rutland@arm.com +Signed-off-by: Catalin Marinas +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/exception.h | 3 ++- + arch/arm64/kernel/entry-common.c | 4 ++-- + arch/arm64/kernel/traps.c | 16 ++++++++++------ + 3 files changed, 14 insertions(+), 9 deletions(-) + +--- a/arch/arm64/include/asm/exception.h ++++ b/arch/arm64/include/asm/exception.h +@@ -71,7 +71,8 @@ void bad_el0_sync(struct pt_regs *regs, + void do_cp15instr(unsigned long esr, struct pt_regs *regs); + void do_el0_svc(struct pt_regs *regs); + void do_el0_svc_compat(struct pt_regs *regs); +-void do_ptrauth_fault(struct pt_regs *regs, unsigned long esr); ++void do_el0_fpac(struct pt_regs *regs, unsigned long esr); ++void do_el1_fpac(struct pt_regs *regs, unsigned long esr); + void do_serror(struct pt_regs *regs, unsigned long esr); + void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags); + +--- a/arch/arm64/kernel/entry-common.c ++++ b/arch/arm64/kernel/entry-common.c +@@ -394,7 +394,7 @@ static void noinstr el1_fpac(struct pt_r + { + enter_from_kernel_mode(regs); + local_daif_inherit(regs); +- do_ptrauth_fault(regs, esr); ++ do_el1_fpac(regs, esr); + local_daif_mask(); + exit_to_kernel_mode(regs); + } +@@ -601,7 +601,7 @@ static void noinstr el0_fpac(struct pt_r + { + enter_from_user_mode(regs); + local_daif_restore(DAIF_PROCCTX); +- do_ptrauth_fault(regs, esr); ++ do_el0_fpac(regs, esr); + exit_to_user_mode(regs); + } + +--- a/arch/arm64/kernel/traps.c ++++ b/arch/arm64/kernel/traps.c +@@ -509,16 +509,20 @@ void do_bti(struct pt_regs *regs) + } + NOKPROBE_SYMBOL(do_bti); + +-void do_ptrauth_fault(struct pt_regs *regs, unsigned long esr) ++void do_el0_fpac(struct pt_regs *regs, unsigned long esr) ++{ ++ force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr); ++} ++ ++void do_el1_fpac(struct pt_regs *regs, unsigned long esr) + { + /* +- * Unexpected FPAC exception or pointer authentication failure in +- * the kernel: kill the task before it does any more harm. ++ * Unexpected FPAC exception in the kernel: kill the task before it ++ * does any more harm. + */ +- BUG_ON(!user_mode(regs)); +- force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr); ++ die("Oops - FPAC", regs, esr); + } +-NOKPROBE_SYMBOL(do_ptrauth_fault); ++NOKPROBE_SYMBOL(do_el1_fpac); + + #define __user_cache_maint(insn, address, res) \ + if (address >= user_addr_max()) { \ diff --git a/queue-5.15/arm64-split-el0-el1-undef-handlers.patch b/queue-5.15/arm64-split-el0-el1-undef-handlers.patch new file mode 100644 index 00000000000..ea786e919e5 --- /dev/null +++ b/queue-5.15/arm64-split-el0-el1-undef-handlers.patch @@ -0,0 +1,177 @@ +From ruanjinjie@huawei.com Wed Oct 11 12:08:17 2023 +From: Jinjie Ruan +Date: Wed, 11 Oct 2023 10:06:47 +0000 +Subject: arm64: split EL0/EL1 UNDEF handlers +To: , , , , , , , , , , , , , , , , , +Cc: +Message-ID: <20231011100655.979626-8-ruanjinjie@huawei.com> + +From: Mark Rutland + +commit 61d64a376ea80f9097e7ea599bcd68671b836dc6 upstream. + +In general, exceptions taken from EL1 need to be handled separately from +exceptions taken from EL0, as the logic to handle the two cases can be +significantly divergent, and exceptions taken from EL1 typically have +more stringent requirements on locking and instrumentation. + +Subsequent patches will rework the way EL1 UNDEFs are handled in order +to address longstanding soundness issues with instrumentation and RCU. +In preparation for that rework, this patch splits the existing +do_undefinstr() handler into separate do_el0_undef() and do_el1_undef() +handlers. + +Prior to this patch, do_undefinstr() was marked with NOKPROBE_SYMBOL(), +preventing instrumentation via kprobes. However, do_undefinstr() invokes +other code which can be instrumented, and: + +* For UNDEFINED exceptions taken from EL0, there is no risk of recursion + within kprobes. Therefore it is safe for do_el0_undef to be + instrumented with kprobes, and it does not need to be marked with + NOKPROBE_SYMBOL(). + +* For UNDEFINED exceptions taken from EL1, either: + + (a) The exception is has been taken when manipulating SSBS; these cases + are limited and do not occur within code that can be invoked + recursively via kprobes. Hence, in these cases instrumentation + with kprobes is benign. + + (b) The exception has been taken for an unknown reason, as other than + manipulating SSBS we do not expect to take UNDEFINED exceptions + from EL1. Any handling of these exception is best-effort. + + ... and in either case, marking do_el1_undef() with NOKPROBE_SYMBOL() + isn't sufficient to prevent recursion via kprobes as functions it + calls (including die()) are instrumentable via kprobes. + + Hence, it's not worthwhile to mark do_el1_undef() with + NOKPROBE_SYMBOL(). The same applies to do_el1_bti() and do_el1_fpac(), + so their NOKPROBE_SYMBOL() annotations are also removed. + +Aside from the new instrumentability, there should be no functional +change as a result of this patch. + +Signed-off-by: Mark Rutland +Cc: Catalin Marinas +Cc: James Morse +Cc: Joey Gouly +Cc: Peter Zijlstra +Cc: Will Deacon +Link: https://lore.kernel.org/r/20221019144123.612388-3-mark.rutland@arm.com +Signed-off-by: Will Deacon +Signed-off-by: Jinjie Ruan +Signed-off-by: Greg Kroah-Hartman +--- + arch/arm64/include/asm/exception.h | 3 ++- + arch/arm64/kernel/entry-common.c | 4 ++-- + arch/arm64/kernel/traps.c | 22 ++++++++++++---------- + 3 files changed, 16 insertions(+), 13 deletions(-) + +--- a/arch/arm64/include/asm/exception.h ++++ b/arch/arm64/include/asm/exception.h +@@ -58,7 +58,8 @@ asmlinkage void call_on_irq_stack(struct + asmlinkage void asm_exit_to_user_mode(struct pt_regs *regs); + + void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs); +-void do_undefinstr(struct pt_regs *regs, unsigned long esr); ++void do_el0_undef(struct pt_regs *regs, unsigned long esr); ++void do_el1_undef(struct pt_regs *regs, unsigned long esr); + void do_el0_bti(struct pt_regs *regs); + void do_el1_bti(struct pt_regs *regs, unsigned long esr); + void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr, +--- a/arch/arm64/kernel/entry-common.c ++++ b/arch/arm64/kernel/entry-common.c +@@ -375,7 +375,7 @@ static void noinstr el1_undef(struct pt_ + { + enter_from_kernel_mode(regs); + local_daif_inherit(regs); +- do_undefinstr(regs, esr); ++ do_el1_undef(regs, esr); + local_daif_mask(); + exit_to_kernel_mode(regs); + } +@@ -570,7 +570,7 @@ static void noinstr el0_undef(struct pt_ + { + enter_from_user_mode(regs); + local_daif_restore(DAIF_PROCCTX); +- do_undefinstr(regs, esr); ++ do_el0_undef(regs, esr); + exit_to_user_mode(regs); + } + +--- a/arch/arm64/kernel/traps.c ++++ b/arch/arm64/kernel/traps.c +@@ -486,7 +486,7 @@ void arm64_notify_segfault(unsigned long + force_signal_inject(SIGSEGV, code, addr, 0); + } + +-void do_undefinstr(struct pt_regs *regs, unsigned long esr) ++void do_el0_undef(struct pt_regs *regs, unsigned long esr) + { + /* check for AArch32 breakpoint instructions */ + if (!aarch32_break_handler(regs)) +@@ -495,12 +495,16 @@ void do_undefinstr(struct pt_regs *regs, + if (call_undef_hook(regs) == 0) + return; + +- if (!user_mode(regs)) +- die("Oops - Undefined instruction", regs, esr); +- + force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); + } +-NOKPROBE_SYMBOL(do_undefinstr); ++ ++void do_el1_undef(struct pt_regs *regs, unsigned long esr) ++{ ++ if (call_undef_hook(regs) == 0) ++ return; ++ ++ die("Oops - Undefined instruction", regs, esr); ++} + + void do_el0_bti(struct pt_regs *regs) + { +@@ -511,7 +515,6 @@ void do_el1_bti(struct pt_regs *regs, un + { + die("Oops - BTI", regs, esr); + } +-NOKPROBE_SYMBOL(do_el1_bti); + + void do_el0_fpac(struct pt_regs *regs, unsigned long esr) + { +@@ -526,7 +529,6 @@ void do_el1_fpac(struct pt_regs *regs, u + */ + die("Oops - FPAC", regs, esr); + } +-NOKPROBE_SYMBOL(do_el1_fpac); + + #define __user_cache_maint(insn, address, res) \ + if (address >= user_addr_max()) { \ +@@ -763,7 +765,7 @@ void do_el0_cp15(unsigned long esr, stru + hook_base = cp15_64_hooks; + break; + default: +- do_undefinstr(regs, esr); ++ do_el0_undef(regs, esr); + return; + } + +@@ -778,7 +780,7 @@ void do_el0_cp15(unsigned long esr, stru + * EL0. Fall back to our usual undefined instruction handler + * so that we handle these consistently. + */ +- do_undefinstr(regs, esr); ++ do_el0_undef(regs, esr); + } + #endif + +@@ -797,7 +799,7 @@ void do_el0_sys(unsigned long esr, struc + * back to our usual undefined instruction handler so that we handle + * these consistently. + */ +- do_undefinstr(regs, esr); ++ do_el0_undef(regs, esr); + } + + static const char *esr_class_str[] = { diff --git a/queue-5.15/series b/queue-5.15/series index 5a1a78f82a1..e24a2fb9251 100644 --- a/queue-5.15/series +++ b/queue-5.15/series @@ -82,3 +82,18 @@ dmaengine-idxd-use-spin_lock_irqsave-before-wait_eve.patch dmaengine-mediatek-fix-deadlock-caused-by-synchroniz.patch powerpc-8xx-fix-pte_access_permitted-for-page_none.patch powerpc-64e-fix-wrong-test-in-__ptep_test_and_clear_.patch +arm64-report-el1-undefs-better.patch +arm64-die-pass-err-as-long.patch +arm64-consistently-pass-esr_elx-to-die.patch +arm64-rework-fpac-exception-handling.patch +arm64-rework-bti-exception-handling.patch +arm64-allow-kprobes-on-el0-handlers.patch +arm64-split-el0-el1-undef-handlers.patch +arm64-factor-out-el1-ssbs-emulation-hook.patch +arm64-factor-insn-read-out-of-call_undef_hook.patch +arm64-rework-el0-mrs-emulation.patch +arm64-armv8_deprecated-fold-ops-into-insn_emulation.patch +arm64-armv8_deprecated-move-emulation-functions.patch +arm64-armv8_deprecated-move-aarch32-helper-earlier.patch +arm64-armv8_deprecated-rework-deprected-instruction-handling.patch +arm64-armv8_deprecated-fix-unused-function-error.patch -- 2.47.3