x86/bugs: Restructure retbleed mitigation

Restructure retbleed mitigation to use select/update/apply functions to create
consistent vulnerability handling.  The retbleed_update_mitigation()
simplifies the dependency between spectre_v2 and retbleed.

The command line options now directly select a preferred mitigation
which simplifies the logic.

Signed-off-by: David Kaplan <david.kaplan@amd.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>
Link: https://lore.kernel.org/20250418161721.1855190-11-david.kaplan@amd.com

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 7edf42912cdf83a8846d7fdc37314f74356f6c91..207a472d1a6e6ec0def05bf2fe54fc6e43e073d2 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -57,6 +57,8 @@ static void __init spectre_v1_select_mitigation(void);
 static void __init spectre_v1_apply_mitigation(void);
 static void __init spectre_v2_select_mitigation(void);
 static void __init retbleed_select_mitigation(void);
+static void __init retbleed_update_mitigation(void);
+static void __init retbleed_apply_mitigation(void);
 static void __init spectre_v2_user_select_mitigation(void);
 static void __init ssb_select_mitigation(void);
 static void __init l1tf_select_mitigation(void);
@@ -187,11 +189,6 @@ void __init cpu_select_mitigations(void)
        /* Select the proper CPU mitigations before patching alternatives: */
        spectre_v1_select_mitigation();
        spectre_v2_select_mitigation();
-       /*
-        * retbleed_select_mitigation() relies on the state set by
-        * spectre_v2_select_mitigation(); specifically it wants to know about
-        * spectre_v2=ibrs.
-        */
        retbleed_select_mitigation();
        /*
         * spectre_v2_user_select_mitigation() relies on the state set by
@@ -219,12 +216,14 @@ void __init cpu_select_mitigations(void)
         * After mitigations are selected, some may need to update their
         * choices.
         */
+       retbleed_update_mitigation();
        mds_update_mitigation();
        taa_update_mitigation();
        mmio_update_mitigation();
        rfds_update_mitigation();
 
        spectre_v1_apply_mitigation();
+       retbleed_apply_mitigation();
        mds_apply_mitigation();
        taa_apply_mitigation();
        mmio_apply_mitigation();
@@ -1085,6 +1084,7 @@ enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE;
 
 enum retbleed_mitigation {
        RETBLEED_MITIGATION_NONE,
+       RETBLEED_MITIGATION_AUTO,
        RETBLEED_MITIGATION_UNRET,
        RETBLEED_MITIGATION_IBPB,
        RETBLEED_MITIGATION_IBRS,
@@ -1092,14 +1092,6 @@ enum retbleed_mitigation {
        RETBLEED_MITIGATION_STUFF,
 };
 
-enum retbleed_mitigation_cmd {
-       RETBLEED_CMD_OFF,
-       RETBLEED_CMD_AUTO,
-       RETBLEED_CMD_UNRET,
-       RETBLEED_CMD_IBPB,
-       RETBLEED_CMD_STUFF,
-};
-
 static const char * const retbleed_strings[] = {
        [RETBLEED_MITIGATION_NONE]      = "Vulnerable",
        [RETBLEED_MITIGATION_UNRET]     = "Mitigation: untrained return thunk",
@@ -1110,9 +1102,7 @@ static const char * const retbleed_strings[] = {
 };
 
 static enum retbleed_mitigation retbleed_mitigation __ro_after_init =
-       RETBLEED_MITIGATION_NONE;
-static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init =
-       IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_CMD_AUTO : RETBLEED_CMD_OFF;
+       IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_MITIGATION_AUTO : RETBLEED_MITIGATION_NONE;
 
 static int __ro_after_init retbleed_nosmt = false;
 
@@ -1129,15 +1119,15 @@ static int __init retbleed_parse_cmdline(char *str)
                }
 
                if (!strcmp(str, "off")) {
-                       retbleed_cmd = RETBLEED_CMD_OFF;
+                       retbleed_mitigation = RETBLEED_MITIGATION_NONE;
                } else if (!strcmp(str, "auto")) {
-                       retbleed_cmd = RETBLEED_CMD_AUTO;
+                       retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
                } else if (!strcmp(str, "unret")) {
-                       retbleed_cmd = RETBLEED_CMD_UNRET;
+                       retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
                } else if (!strcmp(str, "ibpb")) {
-                       retbleed_cmd = RETBLEED_CMD_IBPB;
+                       retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
                } else if (!strcmp(str, "stuff")) {
-                       retbleed_cmd = RETBLEED_CMD_STUFF;
+                       retbleed_mitigation = RETBLEED_MITIGATION_STUFF;
                } else if (!strcmp(str, "nosmt")) {
                        retbleed_nosmt = true;
                } else if (!strcmp(str, "force")) {
@@ -1158,76 +1148,109 @@ early_param("retbleed", retbleed_parse_cmdline);
 
 static void __init retbleed_select_mitigation(void)
 {
-       bool mitigate_smt = false;
-
-       if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off())
-               return;
-
-       switch (retbleed_cmd) {
-       case RETBLEED_CMD_OFF:
+       if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) {
+               retbleed_mitigation = RETBLEED_MITIGATION_NONE;
                return;
+       }
 
-       case RETBLEED_CMD_UNRET:
-               if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) {
-                       retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
-               } else {
+       switch (retbleed_mitigation) {
+       case RETBLEED_MITIGATION_UNRET:
+               if (!IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) {
+                       retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
                        pr_err("WARNING: kernel not compiled with MITIGATION_UNRET_ENTRY.\n");
-                       goto do_cmd_auto;
                }
                break;
-
-       case RETBLEED_CMD_IBPB:
+       case RETBLEED_MITIGATION_IBPB:
                if (!boot_cpu_has(X86_FEATURE_IBPB)) {
                        pr_err("WARNING: CPU does not support IBPB.\n");
-                       goto do_cmd_auto;
-               } else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
-                       retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
-               } else {
+                       retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
+               } else if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
                        pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");
-                       goto do_cmd_auto;
+                       retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
+               }
+               break;
+       case RETBLEED_MITIGATION_STUFF:
+               if (!IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) {
+                       pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n");
+                       retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
+               } else if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
+                       pr_err("WARNING: retbleed=stuff only supported for Intel CPUs.\n");
+                       retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
                }
                break;
+       default:
+               break;
+       }
 
-       case RETBLEED_CMD_STUFF:
-               if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) &&
-                   spectre_v2_enabled == SPECTRE_V2_RETPOLINE) {
-                       if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
-                               pr_err("WARNING: retbleed=stuff only supported for Intel CPUs.\n");
-                               goto do_cmd_auto;
-                       }
-                       retbleed_mitigation = RETBLEED_MITIGATION_STUFF;
+       if (retbleed_mitigation != RETBLEED_MITIGATION_AUTO)
+               return;
 
-               } else {
-                       if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING))
-                               pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n");
-                       else
-                               pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n");
+       /* Intel mitigation selected in retbleed_update_mitigation() */
+       if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
+           boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
+               if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY))
+                       retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
+               else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) &&
+                        boot_cpu_has(X86_FEATURE_IBPB))
+                       retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
+               else
+                       retbleed_mitigation = RETBLEED_MITIGATION_NONE;
+       }
+}
 
-                       goto do_cmd_auto;
-               }
-               break;
+static void __init retbleed_update_mitigation(void)
+{
+       if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off())
+               return;
+
+       if (retbleed_mitigation == RETBLEED_MITIGATION_NONE)
+               goto out;
 
-do_cmd_auto:
-       case RETBLEED_CMD_AUTO:
-               if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
-                   boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
-                       if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY))
-                               retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
-                       else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) &&
-                                boot_cpu_has(X86_FEATURE_IBPB))
-                               retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
+       /*
+        * retbleed=stuff is only allowed on Intel.  If stuffing can't be used
+        * then a different mitigation will be selected below.
+        */
+       if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF) {
+               if (spectre_v2_enabled != SPECTRE_V2_RETPOLINE) {
+                       pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n");
+                       retbleed_mitigation = RETBLEED_MITIGATION_AUTO;
                }
+       }
+       /*
+        * Let IBRS trump all on Intel without affecting the effects of the
+        * retbleed= cmdline option except for call depth based stuffing
+        */
+       if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
+               switch (spectre_v2_enabled) {
+               case SPECTRE_V2_IBRS:
+                       retbleed_mitigation = RETBLEED_MITIGATION_IBRS;
+                       break;
+               case SPECTRE_V2_EIBRS:
+               case SPECTRE_V2_EIBRS_RETPOLINE:
+               case SPECTRE_V2_EIBRS_LFENCE:
+                       retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;
+                       break;
+               default:
+                       if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF)
+                               pr_err(RETBLEED_INTEL_MSG);
+               }
+               /* If nothing has set the mitigation yet, default to NONE. */
+               if (retbleed_mitigation == RETBLEED_MITIGATION_AUTO)
+                       retbleed_mitigation = RETBLEED_MITIGATION_NONE;
+       }
+out:
+       pr_info("%s\n", retbleed_strings[retbleed_mitigation]);
+}
 
-               /*
-                * The Intel mitigation (IBRS or eIBRS) was already selected in
-                * spectre_v2_select_mitigation().  'retbleed_mitigation' will
-                * be set accordingly below.
-                */
 
-               break;
-       }
+static void __init retbleed_apply_mitigation(void)
+{
+       bool mitigate_smt = false;
 
        switch (retbleed_mitigation) {
+       case RETBLEED_MITIGATION_NONE:
+               return;
+
        case RETBLEED_MITIGATION_UNRET:
                setup_force_cpu_cap(X86_FEATURE_RETHUNK);
                setup_force_cpu_cap(X86_FEATURE_UNRET);
@@ -1277,28 +1300,6 @@ do_cmd_auto:
        if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) &&
            (retbleed_nosmt || cpu_mitigations_auto_nosmt()))
                cpu_smt_disable(false);
-
-       /*
-        * Let IBRS trump all on Intel without affecting the effects of the
-        * retbleed= cmdline option except for call depth based stuffing
-        */
-       if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
-               switch (spectre_v2_enabled) {
-               case SPECTRE_V2_IBRS:
-                       retbleed_mitigation = RETBLEED_MITIGATION_IBRS;
-                       break;
-               case SPECTRE_V2_EIBRS:
-               case SPECTRE_V2_EIBRS_RETPOLINE:
-               case SPECTRE_V2_EIBRS_LFENCE:
-                       retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;
-                       break;
-               default:
-                       if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF)
-                               pr_err(RETBLEED_INTEL_MSG);
-               }
-       }
-
-       pr_info("%s\n", retbleed_strings[retbleed_mitigation]);
 }
 
 #undef pr_fmt
@@ -1855,8 +1856,8 @@ static void __init spectre_v2_select_mitigation(void)
 
                if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) &&
                    boot_cpu_has_bug(X86_BUG_RETBLEED) &&
-                   retbleed_cmd != RETBLEED_CMD_OFF &&
-                   retbleed_cmd != RETBLEED_CMD_STUFF &&
+                   retbleed_mitigation != RETBLEED_MITIGATION_NONE &&
+                   retbleed_mitigation != RETBLEED_MITIGATION_STUFF &&
                    boot_cpu_has(X86_FEATURE_IBRS) &&
                    boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
                        mode = SPECTRE_V2_IBRS;
@@ -1964,7 +1965,7 @@ static void __init spectre_v2_select_mitigation(void)
            (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
             boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) {
 
-               if (retbleed_cmd != RETBLEED_CMD_IBPB) {
+               if (retbleed_mitigation != RETBLEED_MITIGATION_IBPB) {
                        setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW);
                        pr_info("Enabling Speculation Barrier for firmware calls\n");
                }