1 From 5ec47ec6bd0952874bd60b3f8e7833eb2b9cdcbb Mon Sep 17 00:00:00 2001
2 From: "Borislav Petkov (AMD)" <bp@alien8.de>
3 Date: Thu, 28 Mar 2024 13:59:05 +0100
4 Subject: x86/bugs: Fix the SRSO mitigation on Zen3/4
6 From: "Borislav Petkov (AMD)" <bp@alien8.de>
8 Commit 4535e1a4174c4111d92c5a9a21e542d232e0fcaa upstream.
10 The original version of the mitigation would patch in the calls to the
11 untraining routines directly. That is, the alternative() in UNTRAIN_RET
12 will patch in the CALL to srso_alias_untrain_ret() directly.
14 However, even if commit e7c25c441e9e ("x86/cpu: Cleanup the untrain
15 mess") meant well in trying to clean up the situation, due to micro-
16 architectural reasons, the untraining routine srso_alias_untrain_ret()
17 must be the target of a CALL instruction and not of a JMP instruction as
20 Reshuffle the alternative macros to accomplish that.
22 Fixes: e7c25c441e9e ("x86/cpu: Cleanup the untrain mess")
23 Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
24 Reviewed-by: Ingo Molnar <mingo@kernel.org>
26 Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
27 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
29 arch/x86/include/asm/asm-prototypes.h | 1 +
30 arch/x86/include/asm/nospec-branch.h | 21 ++++++++++++++++-----
31 arch/x86/lib/retpoline.S | 10 +++++-----
32 3 files changed, 22 insertions(+), 10 deletions(-)
34 --- a/arch/x86/include/asm/asm-prototypes.h
35 +++ b/arch/x86/include/asm/asm-prototypes.h
37 #include <asm/preempt.h>
39 #include <asm/gsseg.h>
40 +#include <asm/nospec-branch.h>
42 #ifndef CONFIG_X86_CMPXCHG64
43 extern void cmpxchg8b_emu(void);
44 --- a/arch/x86/include/asm/nospec-branch.h
45 +++ b/arch/x86/include/asm/nospec-branch.h
51 + * The CALL to srso_alias_untrain_ret() must be patched in directly at
52 + * the spot where untraining must be done, ie., srso_alias_untrain_ret()
53 + * must be the target of a CALL instruction instead of indirectly
54 + * jumping to a wrapper which then calls it. Therefore, this macro is
55 + * called outside of __UNTRAIN_RET below, for the time being, before the
56 + * kernel can support nested alternatives with arbitrary nesting.
58 +.macro CALL_UNTRAIN_RET
59 #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_SRSO)
60 -#define CALL_UNTRAIN_RET "call entry_untrain_ret"
62 -#define CALL_UNTRAIN_RET ""
63 + ALTERNATIVE_2 "", "call entry_untrain_ret", X86_FEATURE_UNRET, \
64 + "call srso_alias_untrain_ret", X86_FEATURE_SRSO_ALIAS
69 * Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the
71 .macro __UNTRAIN_RET ibpb_feature, call_depth_insns
72 #if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY)
75 - CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \
78 "call entry_ibpb", \ibpb_feature, \
79 __stringify(\call_depth_insns), X86_FEATURE_CALL_DEPTH
81 @@ -351,6 +360,8 @@ extern void retbleed_return_thunk(void);
82 static inline void retbleed_return_thunk(void) {}
85 +extern void srso_alias_untrain_ret(void);
87 #ifdef CONFIG_CPU_SRSO
88 extern void srso_return_thunk(void);
89 extern void srso_alias_return_thunk(void);
90 --- a/arch/x86/lib/retpoline.S
91 +++ b/arch/x86/lib/retpoline.S
92 @@ -218,10 +218,12 @@ SYM_CODE_START(srso_return_thunk)
93 SYM_CODE_END(srso_return_thunk)
95 #define JMP_SRSO_UNTRAIN_RET "jmp srso_untrain_ret"
96 -#define JMP_SRSO_ALIAS_UNTRAIN_RET "jmp srso_alias_untrain_ret"
97 #else /* !CONFIG_CPU_SRSO */
98 #define JMP_SRSO_UNTRAIN_RET "ud2"
99 -#define JMP_SRSO_ALIAS_UNTRAIN_RET "ud2"
100 +/* Dummy for the alternative in CALL_UNTRAIN_RET. */
101 +SYM_CODE_START(srso_alias_untrain_ret)
103 +SYM_FUNC_END(srso_alias_untrain_ret)
104 #endif /* CONFIG_CPU_SRSO */
106 #ifdef CONFIG_CPU_UNRET_ENTRY
107 @@ -314,9 +316,7 @@ __EXPORT_THUNK(retbleed_untrain_ret)
108 #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_SRSO)
110 SYM_FUNC_START(entry_untrain_ret)
111 - ALTERNATIVE_2 JMP_RETBLEED_UNTRAIN_RET, \
112 - JMP_SRSO_UNTRAIN_RET, X86_FEATURE_SRSO, \
113 - JMP_SRSO_ALIAS_UNTRAIN_RET, X86_FEATURE_SRSO_ALIAS
114 + ALTERNATIVE JMP_RETBLEED_UNTRAIN_RET, JMP_SRSO_UNTRAIN_RET, X86_FEATURE_SRSO
115 SYM_FUNC_END(entry_untrain_ret)
116 __EXPORT_THUNK(entry_untrain_ret)