x86/cpu: Clean up SRSO return thunk mess

commit d43490d0ab824023e11d0b57d0aeec17a6e0ca13 upstream.

Use the existing configurable return thunk. There is absolute no
justification for having created this __x86_return_thunk alternative.

To clarify, the whole thing looks like:

Zen3/4 does:

  srso_alias_untrain_ret:
	  nop2
	  lfence
	  jmp srso_alias_return_thunk
	  int3

  srso_alias_safe_ret: // aliasses srso_alias_untrain_ret just so
	  add $8, %rsp
	  ret
	  int3

  srso_alias_return_thunk:
	  call srso_alias_safe_ret
	  ud2

While Zen1/2 does:

  srso_untrain_ret:
	  movabs $foo, %rax
	  lfence
	  call srso_safe_ret           (jmp srso_return_thunk ?)
	  int3

  srso_safe_ret: // embedded in movabs instruction
	  add $8,%rsp
          ret
          int3

  srso_return_thunk:
	  call srso_safe_ret
	  ud2

While retbleed does:

  zen_untrain_ret:
	  test $0xcc, %bl
	  lfence
	  jmp zen_return_thunk
          int3

  zen_return_thunk: // embedded in the test instruction
	  ret
          int3

Where Zen1/2 flush the BTB entry using the instruction decoder trick
(test,movabs) Zen3/4 use BTB aliasing. SRSO adds a return sequence
(srso_safe_ret()) which forces the function return instruction to
speculate into a trap (UD2).  This RET will then mispredict and
execution will continue at the return site read from the top of the
stack.

Pick one of three options at boot (evey function can only ever return
once).

  [ bp: Fixup commit message uarch details and add them in a comment in
    the code too. Add a comment about the srso_select_mitigation()
    dependency on retbleed_select_mitigation(). Add moar ifdeffery for
    32-bit builds. Add a dummy srso_untrain_ret_alias() definition for
    32-bit alternatives needing the symbol. ]

Fixes: fb3bd914b3ec ("x86/srso: Add a Speculative RAS Overflow mitigation")
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Link: https://lore.kernel.org/r/20230814121148.842775684@infradead.org
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 86632468f02cfca2166f837a42d4d9f3900eed40..e99f7382fea782fe5195c5754ad94e114042ea91 100644 (file)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -201,9 +201,14 @@ extern void __x86_return_thunk(void);
 static inline void __x86_return_thunk(void) {}
 #endif
 
+extern void zen_return_thunk(void);
+extern void srso_return_thunk(void);
+extern void srso_alias_return_thunk(void);
+
 extern void zen_untrain_ret(void);
 extern void srso_untrain_ret(void);
 extern void srso_untrain_ret_alias(void);
+
 extern void entry_ibpb(void);
 
 #ifdef CONFIG_RETPOLINE

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 9c04690ead1728b94825c7eebb04dcaf0d075883..fcff55621367d6fc9499715a9eacc86c22d45251 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -157,8 +157,13 @@ void __init cpu_select_mitigations(void)
        l1tf_select_mitigation();
        md_clear_select_mitigation();
        srbds_select_mitigation();
-       gds_select_mitigation();
+
+       /*
+        * srso_select_mitigation() depends and must run after
+        * retbleed_select_mitigation().
+        */
        srso_select_mitigation();
+       gds_select_mitigation();
 }
 
 /*
@@ -978,6 +983,9 @@ do_cmd_auto:
                setup_force_cpu_cap(X86_FEATURE_RETHUNK);
                setup_force_cpu_cap(X86_FEATURE_UNRET);
 
+               if (IS_ENABLED(CONFIG_RETHUNK))
+                       x86_return_thunk = zen_return_thunk;
+
                if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
                    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
                        pr_err(RETBLEED_UNTRAIN_MSG);
@@ -2352,10 +2360,13 @@ static void __init srso_select_mitigation(void)
                         */
                        setup_force_cpu_cap(X86_FEATURE_RETHUNK);
 
-                       if (boot_cpu_data.x86 == 0x19)
+                       if (boot_cpu_data.x86 == 0x19) {
                                setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS);
-                       else
+                               x86_return_thunk = srso_alias_return_thunk;
+                       } else {
                                setup_force_cpu_cap(X86_FEATURE_SRSO);
+                               x86_return_thunk = srso_return_thunk;
+                       }
                        srso_mitigation = SRSO_MITIGATION_SAFE_RET;
                } else {
                        pr_err("WARNING: kernel not compiled with CPU_SRSO.\n");

diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 72ba175cb9d4cec32cb1ebe98095b0f6398054a1..560fa84161a2386c803bbfed33596bebc30b2e30 100644 (file)
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -518,7 +518,7 @@ INIT_PER_CPU(irq_stack_backing_store);
 #endif
 
 #ifdef CONFIG_RETHUNK
-. = ASSERT((__ret & 0x3f) == 0, "__ret not cacheline-aligned");
+. = ASSERT((zen_return_thunk & 0x3f) == 0, "zen_return_thunk not cacheline-aligned");
 . = ASSERT((srso_safe_ret & 0x3f) == 0, "srso_safe_ret not cacheline-aligned");
 #endif
 

diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
index f6b65faa266a3f19d8192846a3abdbb0cdb304b6..0d0de750b87ea697f8de86856f86c099f0667ec7 100644 (file)
--- a/arch/x86/lib/retpoline.S
+++ b/arch/x86/lib/retpoline.S
@@ -93,21 +93,26 @@ SYM_CODE_END(__x86_indirect_thunk_array)
        .section .text.__x86.rethunk_untrain
 
 SYM_START(srso_untrain_ret_alias, SYM_L_GLOBAL, SYM_A_NONE)
+       UNWIND_HINT_FUNC
        ASM_NOP2
        lfence
-       jmp __x86_return_thunk
+       jmp srso_alias_return_thunk
 SYM_FUNC_END(srso_untrain_ret_alias)
 __EXPORT_THUNK(srso_untrain_ret_alias)
 
        .section .text.__x86.rethunk_safe
+#else
+/* dummy definition for alternatives */
+SYM_START(srso_untrain_ret_alias, SYM_L_GLOBAL, SYM_A_NONE)
+       ANNOTATE_UNRET_SAFE
+       ret
+       int3
+SYM_FUNC_END(srso_untrain_ret_alias)
 #endif
 
-/* Needs a definition for the __x86_return_thunk alternative below. */
 SYM_START(srso_safe_ret_alias, SYM_L_GLOBAL, SYM_A_NONE)
-#ifdef CONFIG_CPU_SRSO
        add $8, %_ASM_SP
        UNWIND_HINT_FUNC
-#endif
        ANNOTATE_UNRET_SAFE
        ret
        int3
@@ -115,9 +120,16 @@ SYM_FUNC_END(srso_safe_ret_alias)
 
        .section .text.__x86.return_thunk
 
+SYM_CODE_START(srso_alias_return_thunk)
+       UNWIND_HINT_FUNC
+       ANNOTATE_NOENDBR
+       call srso_safe_ret_alias
+       ud2
+SYM_CODE_END(srso_alias_return_thunk)
+
 /*
  * Safety details here pertain to the AMD Zen{1,2} microarchitecture:
- * 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for
+ * 1) The RET at zen_return_thunk must be on a 64 byte boundary, for
  *    alignment within the BTB.
  * 2) The instruction at zen_untrain_ret must contain, and not
  *    end with, the 0xc3 byte of the RET.
@@ -125,7 +137,7 @@ SYM_FUNC_END(srso_safe_ret_alias)
  *    from re-poisioning the BTB prediction.
  */
        .align 64
-       .skip 64 - (__ret - zen_untrain_ret), 0xcc
+       .skip 64 - (zen_return_thunk - zen_untrain_ret), 0xcc
 SYM_FUNC_START_NOALIGN(zen_untrain_ret);
 
        /*
@@ -133,16 +145,16 @@ SYM_FUNC_START_NOALIGN(zen_untrain_ret);
         *
         *   TEST $0xcc, %bl
         *   LFENCE
-        *   JMP __x86_return_thunk
+        *   JMP zen_return_thunk
         *
         * Executing the TEST instruction has a side effect of evicting any BTB
         * prediction (potentially attacker controlled) attached to the RET, as
-        * __x86_return_thunk + 1 isn't an instruction boundary at the moment.
+        * zen_return_thunk + 1 isn't an instruction boundary at the moment.
         */
        .byte   0xf6
 
        /*
-        * As executed from __x86_return_thunk, this is a plain RET.
+        * As executed from zen_return_thunk, this is a plain RET.
         *
         * As part of the TEST above, RET is the ModRM byte, and INT3 the imm8.
         *
@@ -154,13 +166,13 @@ SYM_FUNC_START_NOALIGN(zen_untrain_ret);
         * With SMT enabled and STIBP active, a sibling thread cannot poison
         * RET's prediction to a type of its choice, but can evict the
         * prediction due to competitive sharing. If the prediction is
-        * evicted, __x86_return_thunk will suffer Straight Line Speculation
+        * evicted, zen_return_thunk will suffer Straight Line Speculation
         * which will be contained safely by the INT3.
         */
-SYM_INNER_LABEL(__ret, SYM_L_GLOBAL)
+SYM_INNER_LABEL(zen_return_thunk, SYM_L_GLOBAL)
        ret
        int3
-SYM_CODE_END(__ret)
+SYM_CODE_END(zen_return_thunk)
 
        /*
         * Ensure the TEST decoding / BTB invalidation is complete.
@@ -171,7 +183,7 @@ SYM_CODE_END(__ret)
         * Jump back and execute the RET in the middle of the TEST instruction.
         * INT3 is for SLS protection.
         */
-       jmp __ret
+       jmp zen_return_thunk
        int3
 SYM_FUNC_END(zen_untrain_ret)
 __EXPORT_THUNK(zen_untrain_ret)
@@ -191,12 +203,19 @@ __EXPORT_THUNK(zen_untrain_ret)
 SYM_START(srso_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE)
        .byte 0x48, 0xb8
 
+/*
+ * This forces the function return instruction to speculate into a trap
+ * (UD2 in srso_return_thunk() below).  This RET will then mispredict
+ * and execution will continue at the return site read from the top of
+ * the stack.
+ */
 SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL)
        add $8, %_ASM_SP
        ret
        int3
        int3
        int3
+       /* end of movabs */
        lfence
        call srso_safe_ret
        ud2
@@ -204,12 +223,19 @@ SYM_CODE_END(srso_safe_ret)
 SYM_FUNC_END(srso_untrain_ret)
 __EXPORT_THUNK(srso_untrain_ret)
 
-SYM_CODE_START(__x86_return_thunk)
+SYM_CODE_START(srso_return_thunk)
        UNWIND_HINT_FUNC
        ANNOTATE_NOENDBR
-       ALTERNATIVE_2 "jmp __ret", "call srso_safe_ret", X86_FEATURE_SRSO, \
-                       "call srso_safe_ret_alias", X86_FEATURE_SRSO_ALIAS
+       call srso_safe_ret
        ud2
+SYM_CODE_END(srso_return_thunk)
+
+SYM_CODE_START(__x86_return_thunk)
+       UNWIND_HINT_FUNC
+       ANNOTATE_NOENDBR
+       ANNOTATE_UNRET_SAFE
+       ret
+       int3
 SYM_CODE_END(__x86_return_thunk)
 EXPORT_SYMBOL(__x86_return_thunk)
 

diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c
index 5b915ebb61163a50a82be5ae06e45feb8d640afb..b34b2726b724dda6be1bcacc40094358968e1aeb 100644 (file)
--- a/tools/objtool/arch/x86/decode.c
+++ b/tools/objtool/arch/x86/decode.c
@@ -655,5 +655,5 @@ bool arch_is_rethunk(struct symbol *sym)
        return !strcmp(sym->name, "__x86_return_thunk") ||
               !strcmp(sym->name, "srso_untrain_ret") ||
               !strcmp(sym->name, "srso_safe_ret") ||
-              !strcmp(sym->name, "__ret");
+              !strcmp(sym->name, "zen_return_thunk");
 }