From: Greg Kroah-Hartman Date: Fri, 5 Apr 2024 09:36:10 +0000 (+0200) Subject: 6.6-stable patches X-Git-Tag: v5.15.154~102 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=340d2c8a7e20d7213f0f4069163509667ee1f4eb;p=thirdparty%2Fkernel%2Fstable-queue.git 6.6-stable patches added patches: x86-bugs-fix-the-srso-mitigation-on-zen3-4.patch x86-nospec-refactor-untrain_ret.patch x86-srso-disentangle-rethunk-dependent-options.patch x86-srso-improve-i-cache-locality-for-alias-mitigation.patch --- diff --git a/queue-6.6/series b/queue-6.6/series index a3db8f40be3..a8e836c0ab2 100644 --- a/queue-6.6/series +++ b/queue-6.6/series @@ -87,3 +87,7 @@ net-rds-fix-possible-cp-null-dereference.patch net-usb-ax88179_178a-avoid-the-interface-always-configured-as-random-address.patch net-mana-fix-rx-dma-datasize-and-skb_over_panic.patch vsock-virtio-fix-packet-delivery-to-tap-device.patch +x86-srso-improve-i-cache-locality-for-alias-mitigation.patch +x86-srso-disentangle-rethunk-dependent-options.patch +x86-nospec-refactor-untrain_ret.patch +x86-bugs-fix-the-srso-mitigation-on-zen3-4.patch diff --git a/queue-6.6/x86-bugs-fix-the-srso-mitigation-on-zen3-4.patch b/queue-6.6/x86-bugs-fix-the-srso-mitigation-on-zen3-4.patch new file mode 100644 index 00000000000..49ecd612c5f --- /dev/null +++ b/queue-6.6/x86-bugs-fix-the-srso-mitigation-on-zen3-4.patch @@ -0,0 +1,117 @@ +From 5ec47ec6bd0952874bd60b3f8e7833eb2b9cdcbb Mon Sep 17 00:00:00 2001 +From: "Borislav Petkov (AMD)" +Date: Thu, 28 Mar 2024 13:59:05 +0100 +Subject: x86/bugs: Fix the SRSO mitigation on Zen3/4 + +From: "Borislav Petkov (AMD)" + +Commit 4535e1a4174c4111d92c5a9a21e542d232e0fcaa upstream. + +The original version of the mitigation would patch in the calls to the +untraining routines directly. That is, the alternative() in UNTRAIN_RET +will patch in the CALL to srso_alias_untrain_ret() directly. + +However, even if commit e7c25c441e9e ("x86/cpu: Cleanup the untrain +mess") meant well in trying to clean up the situation, due to micro- +architectural reasons, the untraining routine srso_alias_untrain_ret() +must be the target of a CALL instruction and not of a JMP instruction as +it is done now. + +Reshuffle the alternative macros to accomplish that. + +Fixes: e7c25c441e9e ("x86/cpu: Cleanup the untrain mess") +Signed-off-by: Borislav Petkov (AMD) +Reviewed-by: Ingo Molnar +Cc: stable@kernel.org +Signed-off-by: Linus Torvalds +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/include/asm/asm-prototypes.h | 1 + + arch/x86/include/asm/nospec-branch.h | 21 ++++++++++++++++----- + arch/x86/lib/retpoline.S | 10 +++++----- + 3 files changed, 22 insertions(+), 10 deletions(-) + +--- a/arch/x86/include/asm/asm-prototypes.h ++++ b/arch/x86/include/asm/asm-prototypes.h +@@ -13,6 +13,7 @@ + #include + #include + #include ++#include + + #ifndef CONFIG_X86_CMPXCHG64 + extern void cmpxchg8b_emu(void); +--- a/arch/x86/include/asm/nospec-branch.h ++++ b/arch/x86/include/asm/nospec-branch.h +@@ -271,11 +271,20 @@ + .Lskip_rsb_\@: + .endm + ++/* ++ * The CALL to srso_alias_untrain_ret() must be patched in directly at ++ * the spot where untraining must be done, ie., srso_alias_untrain_ret() ++ * must be the target of a CALL instruction instead of indirectly ++ * jumping to a wrapper which then calls it. Therefore, this macro is ++ * called outside of __UNTRAIN_RET below, for the time being, before the ++ * kernel can support nested alternatives with arbitrary nesting. ++ */ ++.macro CALL_UNTRAIN_RET + #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_SRSO) +-#define CALL_UNTRAIN_RET "call entry_untrain_ret" +-#else +-#define CALL_UNTRAIN_RET "" ++ ALTERNATIVE_2 "", "call entry_untrain_ret", X86_FEATURE_UNRET, \ ++ "call srso_alias_untrain_ret", X86_FEATURE_SRSO_ALIAS + #endif ++.endm + + /* + * Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the +@@ -291,8 +300,8 @@ + .macro __UNTRAIN_RET ibpb_feature, call_depth_insns + #if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY) + VALIDATE_UNRET_END +- ALTERNATIVE_3 "", \ +- CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ ++ CALL_UNTRAIN_RET ++ ALTERNATIVE_2 "", \ + "call entry_ibpb", \ibpb_feature, \ + __stringify(\call_depth_insns), X86_FEATURE_CALL_DEPTH + #endif +@@ -351,6 +360,8 @@ extern void retbleed_return_thunk(void); + static inline void retbleed_return_thunk(void) {} + #endif + ++extern void srso_alias_untrain_ret(void); ++ + #ifdef CONFIG_CPU_SRSO + extern void srso_return_thunk(void); + extern void srso_alias_return_thunk(void); +--- a/arch/x86/lib/retpoline.S ++++ b/arch/x86/lib/retpoline.S +@@ -218,10 +218,12 @@ SYM_CODE_START(srso_return_thunk) + SYM_CODE_END(srso_return_thunk) + + #define JMP_SRSO_UNTRAIN_RET "jmp srso_untrain_ret" +-#define JMP_SRSO_ALIAS_UNTRAIN_RET "jmp srso_alias_untrain_ret" + #else /* !CONFIG_CPU_SRSO */ + #define JMP_SRSO_UNTRAIN_RET "ud2" +-#define JMP_SRSO_ALIAS_UNTRAIN_RET "ud2" ++/* Dummy for the alternative in CALL_UNTRAIN_RET. */ ++SYM_CODE_START(srso_alias_untrain_ret) ++ RET ++SYM_FUNC_END(srso_alias_untrain_ret) + #endif /* CONFIG_CPU_SRSO */ + + #ifdef CONFIG_CPU_UNRET_ENTRY +@@ -314,9 +316,7 @@ __EXPORT_THUNK(retbleed_untrain_ret) + #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_SRSO) + + SYM_FUNC_START(entry_untrain_ret) +- ALTERNATIVE_2 JMP_RETBLEED_UNTRAIN_RET, \ +- JMP_SRSO_UNTRAIN_RET, X86_FEATURE_SRSO, \ +- JMP_SRSO_ALIAS_UNTRAIN_RET, X86_FEATURE_SRSO_ALIAS ++ ALTERNATIVE JMP_RETBLEED_UNTRAIN_RET, JMP_SRSO_UNTRAIN_RET, X86_FEATURE_SRSO + SYM_FUNC_END(entry_untrain_ret) + __EXPORT_THUNK(entry_untrain_ret) + diff --git a/queue-6.6/x86-nospec-refactor-untrain_ret.patch b/queue-6.6/x86-nospec-refactor-untrain_ret.patch new file mode 100644 index 00000000000..3914b6698ae --- /dev/null +++ b/queue-6.6/x86-nospec-refactor-untrain_ret.patch @@ -0,0 +1,69 @@ +From c280017682332ba6d59ec822d0eb56d30ad76f2c Mon Sep 17 00:00:00 2001 +From: Josh Poimboeuf +Date: Mon, 4 Sep 2023 22:05:03 -0700 +Subject: x86/nospec: Refactor UNTRAIN_RET[_*] + +From: Josh Poimboeuf + +Commit e8efc0800b8b5045ba8c0d1256bfbb47e92e192a upstream. + +Factor out the UNTRAIN_RET[_*] common bits into a helper macro. + +Signed-off-by: Josh Poimboeuf +Signed-off-by: Ingo Molnar +Signed-off-by: Borislav Petkov (AMD) +Acked-by: Borislav Petkov (AMD) +Link: https://lore.kernel.org/r/f06d45489778bd49623297af2a983eea09067a74.1693889988.git.jpoimboe@kernel.org +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/include/asm/nospec-branch.h | 31 ++++++++++--------------------- + 1 file changed, 10 insertions(+), 21 deletions(-) + +--- a/arch/x86/include/asm/nospec-branch.h ++++ b/arch/x86/include/asm/nospec-branch.h +@@ -288,35 +288,24 @@ + * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point + * where we have a stack but before any RET instruction. + */ +-.macro UNTRAIN_RET ++.macro __UNTRAIN_RET ibpb_feature, call_depth_insns + #if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY) + VALIDATE_UNRET_END + ALTERNATIVE_3 "", \ + CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ +- "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ +- __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH ++ "call entry_ibpb", \ibpb_feature, \ ++ __stringify(\call_depth_insns), X86_FEATURE_CALL_DEPTH + #endif + .endm + +-.macro UNTRAIN_RET_VM +-#if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY) +- VALIDATE_UNRET_END +- ALTERNATIVE_3 "", \ +- CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ +- "call entry_ibpb", X86_FEATURE_IBPB_ON_VMEXIT, \ +- __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH +-#endif +-.endm ++#define UNTRAIN_RET \ ++ __UNTRAIN_RET X86_FEATURE_ENTRY_IBPB, __stringify(RESET_CALL_DEPTH) + +-.macro UNTRAIN_RET_FROM_CALL +-#if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY) +- VALIDATE_UNRET_END +- ALTERNATIVE_3 "", \ +- CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ +- "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ +- __stringify(RESET_CALL_DEPTH_FROM_CALL), X86_FEATURE_CALL_DEPTH +-#endif +-.endm ++#define UNTRAIN_RET_VM \ ++ __UNTRAIN_RET X86_FEATURE_IBPB_ON_VMEXIT, __stringify(RESET_CALL_DEPTH) ++ ++#define UNTRAIN_RET_FROM_CALL \ ++ __UNTRAIN_RET X86_FEATURE_ENTRY_IBPB, __stringify(RESET_CALL_DEPTH_FROM_CALL) + + + .macro CALL_DEPTH_ACCOUNT diff --git a/queue-6.6/x86-srso-disentangle-rethunk-dependent-options.patch b/queue-6.6/x86-srso-disentangle-rethunk-dependent-options.patch new file mode 100644 index 00000000000..a494cce6643 --- /dev/null +++ b/queue-6.6/x86-srso-disentangle-rethunk-dependent-options.patch @@ -0,0 +1,344 @@ +From bda2adb3cbbeb20561a803bde681dc0f4015d29e Mon Sep 17 00:00:00 2001 +From: Josh Poimboeuf +Date: Mon, 4 Sep 2023 22:05:00 -0700 +Subject: x86/srso: Disentangle rethunk-dependent options + +From: Josh Poimboeuf + +Commit 34a3cae7474c6e6f4a85aad4a7b8191b8b35cdcd upstream. + +CONFIG_RETHUNK, CONFIG_CPU_UNRET_ENTRY and CONFIG_CPU_SRSO are all +tangled up. De-spaghettify the code a bit. + +Some of the rethunk-related code has been shuffled around within the +'.text..__x86.return_thunk' section, but otherwise there are no +functional changes. srso_alias_untrain_ret() and srso_alias_safe_ret() +((which are very address-sensitive) haven't moved. + +Signed-off-by: Josh Poimboeuf +Signed-off-by: Ingo Molnar +Signed-off-by: Borislav Petkov (AMD) +Acked-by: Borislav Petkov (AMD) +Link: https://lore.kernel.org/r/2845084ed303d8384905db3b87b77693945302b4.1693889988.git.jpoimboe@kernel.org +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/include/asm/nospec-branch.h | 25 +++-- + arch/x86/kernel/cpu/bugs.c | 5 - + arch/x86/kernel/vmlinux.lds.S | 7 - + arch/x86/lib/retpoline.S | 158 +++++++++++++++++++---------------- + 4 files changed, 109 insertions(+), 86 deletions(-) + +--- a/arch/x86/include/asm/nospec-branch.h ++++ b/arch/x86/include/asm/nospec-branch.h +@@ -289,19 +289,17 @@ + * where we have a stack but before any RET instruction. + */ + .macro UNTRAIN_RET +-#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \ +- defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO) ++#if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY) + VALIDATE_UNRET_END + ALTERNATIVE_3 "", \ + CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ + "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ +- __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH ++ __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH + #endif + .endm + + .macro UNTRAIN_RET_VM +-#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \ +- defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO) ++#if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY) + VALIDATE_UNRET_END + ALTERNATIVE_3 "", \ + CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ +@@ -311,8 +309,7 @@ + .endm + + .macro UNTRAIN_RET_FROM_CALL +-#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \ +- defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO) ++#if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY) + VALIDATE_UNRET_END + ALTERNATIVE_3 "", \ + CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ +@@ -359,6 +356,20 @@ extern void __x86_return_thunk(void); + static inline void __x86_return_thunk(void) {} + #endif + ++#ifdef CONFIG_CPU_UNRET_ENTRY ++extern void retbleed_return_thunk(void); ++#else ++static inline void retbleed_return_thunk(void) {} ++#endif ++ ++#ifdef CONFIG_CPU_SRSO ++extern void srso_return_thunk(void); ++extern void srso_alias_return_thunk(void); ++#else ++static inline void srso_return_thunk(void) {} ++static inline void srso_alias_return_thunk(void) {} ++#endif ++ + extern void retbleed_return_thunk(void); + extern void srso_return_thunk(void); + extern void srso_alias_return_thunk(void); +--- a/arch/x86/kernel/cpu/bugs.c ++++ b/arch/x86/kernel/cpu/bugs.c +@@ -63,7 +63,7 @@ EXPORT_SYMBOL_GPL(x86_pred_cmd); + + static DEFINE_MUTEX(spec_ctrl_mutex); + +-void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk; ++void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk; + + /* Update SPEC_CTRL MSR and its cached copy unconditionally */ + static void update_spec_ctrl(u64 val) +@@ -1108,8 +1108,7 @@ do_cmd_auto: + setup_force_cpu_cap(X86_FEATURE_RETHUNK); + setup_force_cpu_cap(X86_FEATURE_UNRET); + +- if (IS_ENABLED(CONFIG_RETHUNK)) +- x86_return_thunk = retbleed_return_thunk; ++ x86_return_thunk = retbleed_return_thunk; + + if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && + boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) +--- a/arch/x86/kernel/vmlinux.lds.S ++++ b/arch/x86/kernel/vmlinux.lds.S +@@ -139,10 +139,7 @@ SECTIONS + STATIC_CALL_TEXT + + ALIGN_ENTRY_TEXT_BEGIN +-#ifdef CONFIG_CPU_SRSO + *(.text..__x86.rethunk_untrain) +-#endif +- + ENTRY_TEXT + + #ifdef CONFIG_CPU_SRSO +@@ -520,12 +517,12 @@ INIT_PER_CPU(irq_stack_backing_store); + "fixed_percpu_data is not at start of per-cpu area"); + #endif + +-#ifdef CONFIG_RETHUNK ++#ifdef CONFIG_CPU_UNRET_ENTRY + . = ASSERT((retbleed_return_thunk & 0x3f) == 0, "retbleed_return_thunk not cacheline-aligned"); +-. = ASSERT((srso_safe_ret & 0x3f) == 0, "srso_safe_ret not cacheline-aligned"); + #endif + + #ifdef CONFIG_CPU_SRSO ++. = ASSERT((srso_safe_ret & 0x3f) == 0, "srso_safe_ret not cacheline-aligned"); + /* + * GNU ld cannot do XOR until 2.41. + * https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=f6f78318fca803c4907fb8d7f6ded8295f1947b1 +--- a/arch/x86/lib/retpoline.S ++++ b/arch/x86/lib/retpoline.S +@@ -126,12 +126,13 @@ SYM_CODE_END(__x86_indirect_jump_thunk_a + #include + #undef GEN + #endif +-/* +- * This function name is magical and is used by -mfunction-return=thunk-extern +- * for the compiler to generate JMPs to it. +- */ ++ + #ifdef CONFIG_RETHUNK + ++ .section .text..__x86.return_thunk ++ ++#ifdef CONFIG_CPU_SRSO ++ + /* + * srso_alias_untrain_ret() and srso_alias_safe_ret() are placed at + * special addresses: +@@ -147,9 +148,7 @@ SYM_CODE_END(__x86_indirect_jump_thunk_a + * + * As a result, srso_alias_safe_ret() becomes a safe return. + */ +-#ifdef CONFIG_CPU_SRSO +- .section .text..__x86.rethunk_untrain +- ++ .pushsection .text..__x86.rethunk_untrain + SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) + UNWIND_HINT_FUNC + ANNOTATE_NOENDBR +@@ -158,17 +157,9 @@ SYM_START(srso_alias_untrain_ret, SYM_L_ + jmp srso_alias_return_thunk + SYM_FUNC_END(srso_alias_untrain_ret) + __EXPORT_THUNK(srso_alias_untrain_ret) ++ .popsection + +- .section .text..__x86.rethunk_safe +-#else +-/* dummy definition for alternatives */ +-SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) +- ANNOTATE_UNRET_SAFE +- ret +- int3 +-SYM_FUNC_END(srso_alias_untrain_ret) +-#endif +- ++ .pushsection .text..__x86.rethunk_safe + SYM_START(srso_alias_safe_ret, SYM_L_GLOBAL, SYM_A_NONE) + lea 8(%_ASM_SP), %_ASM_SP + UNWIND_HINT_FUNC +@@ -183,8 +174,58 @@ SYM_CODE_START_NOALIGN(srso_alias_return + call srso_alias_safe_ret + ud2 + SYM_CODE_END(srso_alias_return_thunk) ++ .popsection ++ ++/* ++ * SRSO untraining sequence for Zen1/2, similar to retbleed_untrain_ret() ++ * above. On kernel entry, srso_untrain_ret() is executed which is a ++ * ++ * movabs $0xccccc30824648d48,%rax ++ * ++ * and when the return thunk executes the inner label srso_safe_ret() ++ * later, it is a stack manipulation and a RET which is mispredicted and ++ * thus a "safe" one to use. ++ */ ++ .align 64 ++ .skip 64 - (srso_safe_ret - srso_untrain_ret), 0xcc ++SYM_START(srso_untrain_ret, SYM_L_LOCAL, SYM_A_NONE) ++ ANNOTATE_NOENDBR ++ .byte 0x48, 0xb8 ++ ++/* ++ * This forces the function return instruction to speculate into a trap ++ * (UD2 in srso_return_thunk() below). This RET will then mispredict ++ * and execution will continue at the return site read from the top of ++ * the stack. ++ */ ++SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL) ++ lea 8(%_ASM_SP), %_ASM_SP ++ ret ++ int3 ++ int3 ++ /* end of movabs */ ++ lfence ++ call srso_safe_ret ++ ud2 ++SYM_CODE_END(srso_safe_ret) ++SYM_FUNC_END(srso_untrain_ret) ++ ++SYM_CODE_START(srso_return_thunk) ++ UNWIND_HINT_FUNC ++ ANNOTATE_NOENDBR ++ call srso_safe_ret ++ ud2 ++SYM_CODE_END(srso_return_thunk) ++ ++#define JMP_SRSO_UNTRAIN_RET "jmp srso_untrain_ret" ++#define JMP_SRSO_ALIAS_UNTRAIN_RET "jmp srso_alias_untrain_ret" ++#else /* !CONFIG_CPU_SRSO */ ++#define JMP_SRSO_UNTRAIN_RET "ud2" ++#define JMP_SRSO_ALIAS_UNTRAIN_RET "ud2" ++#endif /* CONFIG_CPU_SRSO */ ++ ++#ifdef CONFIG_CPU_UNRET_ENTRY + +- .section .text..__x86.return_thunk + /* + * Some generic notes on the untraining sequences: + * +@@ -265,65 +306,21 @@ SYM_CODE_END(retbleed_return_thunk) + SYM_FUNC_END(retbleed_untrain_ret) + __EXPORT_THUNK(retbleed_untrain_ret) + +-/* +- * SRSO untraining sequence for Zen1/2, similar to retbleed_untrain_ret() +- * above. On kernel entry, srso_untrain_ret() is executed which is a +- * +- * movabs $0xccccc30824648d48,%rax +- * +- * and when the return thunk executes the inner label srso_safe_ret() +- * later, it is a stack manipulation and a RET which is mispredicted and +- * thus a "safe" one to use. +- */ +- .align 64 +- .skip 64 - (srso_safe_ret - srso_untrain_ret), 0xcc +-SYM_START(srso_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) +- ANNOTATE_NOENDBR +- .byte 0x48, 0xb8 ++#define JMP_RETBLEED_UNTRAIN_RET "jmp retbleed_untrain_ret" ++#else /* !CONFIG_CPU_UNRET_ENTRY */ ++#define JMP_RETBLEED_UNTRAIN_RET "ud2" ++#endif /* CONFIG_CPU_UNRET_ENTRY */ + +-/* +- * This forces the function return instruction to speculate into a trap +- * (UD2 in srso_return_thunk() below). This RET will then mispredict +- * and execution will continue at the return site read from the top of +- * the stack. +- */ +-SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL) +- lea 8(%_ASM_SP), %_ASM_SP +- ret +- int3 +- int3 +- /* end of movabs */ +- lfence +- call srso_safe_ret +- ud2 +-SYM_CODE_END(srso_safe_ret) +-SYM_FUNC_END(srso_untrain_ret) +-__EXPORT_THUNK(srso_untrain_ret) +- +-SYM_CODE_START(srso_return_thunk) +- UNWIND_HINT_FUNC +- ANNOTATE_NOENDBR +- call srso_safe_ret +- ud2 +-SYM_CODE_END(srso_return_thunk) ++#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_SRSO) + + SYM_FUNC_START(entry_untrain_ret) +- ALTERNATIVE_2 "jmp retbleed_untrain_ret", \ +- "jmp srso_untrain_ret", X86_FEATURE_SRSO, \ +- "jmp srso_alias_untrain_ret", X86_FEATURE_SRSO_ALIAS ++ ALTERNATIVE_2 JMP_RETBLEED_UNTRAIN_RET, \ ++ JMP_SRSO_UNTRAIN_RET, X86_FEATURE_SRSO, \ ++ JMP_SRSO_ALIAS_UNTRAIN_RET, X86_FEATURE_SRSO_ALIAS + SYM_FUNC_END(entry_untrain_ret) + __EXPORT_THUNK(entry_untrain_ret) + +-SYM_CODE_START(__x86_return_thunk) +- UNWIND_HINT_FUNC +- ANNOTATE_NOENDBR +- ANNOTATE_UNRET_SAFE +- ret +- int3 +-SYM_CODE_END(__x86_return_thunk) +-EXPORT_SYMBOL(__x86_return_thunk) +- +-#endif /* CONFIG_RETHUNK */ ++#endif /* CONFIG_CPU_UNRET_ENTRY || CONFIG_CPU_SRSO */ + + #ifdef CONFIG_CALL_DEPTH_TRACKING + +@@ -358,3 +355,22 @@ SYM_FUNC_START(__x86_return_skl) + SYM_FUNC_END(__x86_return_skl) + + #endif /* CONFIG_CALL_DEPTH_TRACKING */ ++ ++/* ++ * This function name is magical and is used by -mfunction-return=thunk-extern ++ * for the compiler to generate JMPs to it. ++ * ++ * This code is only used during kernel boot or module init. All ++ * 'JMP __x86_return_thunk' sites are changed to something else by ++ * apply_returns(). ++ */ ++SYM_CODE_START(__x86_return_thunk) ++ UNWIND_HINT_FUNC ++ ANNOTATE_NOENDBR ++ ANNOTATE_UNRET_SAFE ++ ret ++ int3 ++SYM_CODE_END(__x86_return_thunk) ++EXPORT_SYMBOL(__x86_return_thunk) ++ ++#endif /* CONFIG_RETHUNK */ diff --git a/queue-6.6/x86-srso-improve-i-cache-locality-for-alias-mitigation.patch b/queue-6.6/x86-srso-improve-i-cache-locality-for-alias-mitigation.patch new file mode 100644 index 00000000000..d075f3246a5 --- /dev/null +++ b/queue-6.6/x86-srso-improve-i-cache-locality-for-alias-mitigation.patch @@ -0,0 +1,42 @@ +From 0a784a06a7d1f06a74962616bf7222463a145b2b Mon Sep 17 00:00:00 2001 +From: Josh Poimboeuf +Date: Mon, 4 Sep 2023 22:04:55 -0700 +Subject: x86/srso: Improve i-cache locality for alias mitigation + +From: Josh Poimboeuf + +Commit aa730cff0c26244e88066b5b461a9f5fbac13823 upstream. + +Move srso_alias_return_thunk() to the same section as +srso_alias_safe_ret() so they can share a cache line. + +Signed-off-by: Josh Poimboeuf +Signed-off-by: Ingo Molnar +Signed-off-by: Borislav Petkov (AMD) +Acked-by: Borislav Petkov (AMD) +Link: https://lore.kernel.org/r/eadaf5530b46a7ae8b936522da45ae555d2b3393.1693889988.git.jpoimboe@kernel.org +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/lib/retpoline.S | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +--- a/arch/x86/lib/retpoline.S ++++ b/arch/x86/lib/retpoline.S +@@ -177,15 +177,14 @@ SYM_START(srso_alias_safe_ret, SYM_L_GLO + int3 + SYM_FUNC_END(srso_alias_safe_ret) + +- .section .text..__x86.return_thunk +- +-SYM_CODE_START(srso_alias_return_thunk) ++SYM_CODE_START_NOALIGN(srso_alias_return_thunk) + UNWIND_HINT_FUNC + ANNOTATE_NOENDBR + call srso_alias_safe_ret + ud2 + SYM_CODE_END(srso_alias_return_thunk) + ++ .section .text..__x86.return_thunk + /* + * Some generic notes on the untraining sequences: + *