x86/bugs: Restructure spectre_v2_user mitigation

Restructure spectre_v2_user to use select/update/apply functions to
create consistent vulnerability handling.

The IBPB/STIBP choices are first decided based on the spectre_v2_user
command line but can be modified by the spectre_v2 command line option
as well.

Signed-off-by: David Kaplan <david.kaplan@amd.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>
Link: https://lore.kernel.org/20250418161721.1855190-12-david.kaplan@amd.com

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 207a472d1a6e6ec0def05bf2fe54fc6e43e073d2..dc75195760ca533d222f5e48c3836a7c53aba84d 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -60,6 +60,8 @@ static void __init retbleed_select_mitigation(void);
 static void __init retbleed_update_mitigation(void);
 static void __init retbleed_apply_mitigation(void);
 static void __init spectre_v2_user_select_mitigation(void);
+static void __init spectre_v2_user_update_mitigation(void);
+static void __init spectre_v2_user_apply_mitigation(void);
 static void __init ssb_select_mitigation(void);
 static void __init l1tf_select_mitigation(void);
 static void __init mds_select_mitigation(void);
@@ -190,11 +192,6 @@ void __init cpu_select_mitigations(void)
        spectre_v1_select_mitigation();
        spectre_v2_select_mitigation();
        retbleed_select_mitigation();
-       /*
-        * spectre_v2_user_select_mitigation() relies on the state set by
-        * retbleed_select_mitigation(); specifically the STIBP selection is
-        * forced for UNRET or IBPB.
-        */
        spectre_v2_user_select_mitigation();
        ssb_select_mitigation();
        l1tf_select_mitigation();
@@ -217,6 +214,13 @@ void __init cpu_select_mitigations(void)
         * choices.
         */
        retbleed_update_mitigation();
+
+       /*
+        * spectre_v2_user_update_mitigation() depends on
+        * retbleed_update_mitigation(), specifically the STIBP
+        * selection is forced for UNRET or IBPB.
+        */
+       spectre_v2_user_update_mitigation();
        mds_update_mitigation();
        taa_update_mitigation();
        mmio_update_mitigation();
@@ -224,6 +228,7 @@ void __init cpu_select_mitigations(void)
 
        spectre_v1_apply_mitigation();
        retbleed_apply_mitigation();
+       spectre_v2_user_apply_mitigation();
        mds_apply_mitigation();
        taa_apply_mitigation();
        mmio_apply_mitigation();
@@ -1379,6 +1384,8 @@ enum spectre_v2_mitigation_cmd {
        SPECTRE_V2_CMD_IBRS,
 };
 
+static enum spectre_v2_mitigation_cmd spectre_v2_cmd __ro_after_init = SPECTRE_V2_CMD_AUTO;
+
 enum spectre_v2_user_cmd {
        SPECTRE_V2_USER_CMD_NONE,
        SPECTRE_V2_USER_CMD_AUTO,
@@ -1417,31 +1424,18 @@ static void __init spec_v2_user_print_cond(const char *reason, bool secure)
                pr_info("spectre_v2_user=%s forced on command line.\n", reason);
 }
 
-static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd;
-
-static enum spectre_v2_user_cmd __init
-spectre_v2_parse_user_cmdline(void)
+static enum spectre_v2_user_cmd __init spectre_v2_parse_user_cmdline(void)
 {
-       enum spectre_v2_user_cmd mode;
        char arg[20];
        int ret, i;
 
-       mode = IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ?
-               SPECTRE_V2_USER_CMD_AUTO : SPECTRE_V2_USER_CMD_NONE;
-
-       switch (spectre_v2_cmd) {
-       case SPECTRE_V2_CMD_NONE:
+       if (cpu_mitigations_off() || !IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2))
                return SPECTRE_V2_USER_CMD_NONE;
-       case SPECTRE_V2_CMD_FORCE:
-               return SPECTRE_V2_USER_CMD_FORCE;
-       default:
-               break;
-       }
 
        ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
                                  arg, sizeof(arg));
        if (ret < 0)
-               return mode;
+               return SPECTRE_V2_USER_CMD_AUTO;
 
        for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
                if (match_option(arg, ret, v2_user_options[i].option)) {
@@ -1452,7 +1446,7 @@ spectre_v2_parse_user_cmdline(void)
        }
 
        pr_err("Unknown user space protection option (%s). Switching to default\n", arg);
-       return mode;
+       return SPECTRE_V2_USER_CMD_AUTO;
 }
 
 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode)
@@ -1460,60 +1454,72 @@ static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode)
        return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS;
 }
 
-static void __init
-spectre_v2_user_select_mitigation(void)
+static void __init spectre_v2_user_select_mitigation(void)
 {
-       enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
-       enum spectre_v2_user_cmd cmd;
-
        if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
                return;
 
-       cmd = spectre_v2_parse_user_cmdline();
-       switch (cmd) {
+       switch (spectre_v2_parse_user_cmdline()) {
        case SPECTRE_V2_USER_CMD_NONE:
-               goto set_mode;
+               return;
        case SPECTRE_V2_USER_CMD_FORCE:
-               mode = SPECTRE_V2_USER_STRICT;
+               spectre_v2_user_ibpb  = SPECTRE_V2_USER_STRICT;
+               spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT;
                break;
        case SPECTRE_V2_USER_CMD_AUTO:
        case SPECTRE_V2_USER_CMD_PRCTL:
+               spectre_v2_user_ibpb  = SPECTRE_V2_USER_PRCTL;
+               spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL;
+               break;
        case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
-               mode = SPECTRE_V2_USER_PRCTL;
+               spectre_v2_user_ibpb  = SPECTRE_V2_USER_STRICT;
+               spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL;
                break;
        case SPECTRE_V2_USER_CMD_SECCOMP:
+               if (IS_ENABLED(CONFIG_SECCOMP))
+                       spectre_v2_user_ibpb = SPECTRE_V2_USER_SECCOMP;
+               else
+                       spectre_v2_user_ibpb = SPECTRE_V2_USER_PRCTL;
+               spectre_v2_user_stibp = spectre_v2_user_ibpb;
+               break;
        case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
+               spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
                if (IS_ENABLED(CONFIG_SECCOMP))
-                       mode = SPECTRE_V2_USER_SECCOMP;
+                       spectre_v2_user_stibp = SPECTRE_V2_USER_SECCOMP;
                else
-                       mode = SPECTRE_V2_USER_PRCTL;
+                       spectre_v2_user_stibp = SPECTRE_V2_USER_PRCTL;
                break;
        }
 
-       /* Initialize Indirect Branch Prediction Barrier */
-       if (boot_cpu_has(X86_FEATURE_IBPB)) {
-               static_branch_enable(&switch_vcpu_ibpb);
+       /*
+        * At this point, an STIBP mode other than "off" has been set.
+        * If STIBP support is not being forced, check if STIBP always-on
+        * is preferred.
+        */
+       if ((spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL ||
+            spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) &&
+           boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
+               spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED;
 
-               spectre_v2_user_ibpb = mode;
-               switch (cmd) {
-               case SPECTRE_V2_USER_CMD_NONE:
-                       break;
-               case SPECTRE_V2_USER_CMD_FORCE:
-               case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
-               case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
-                       static_branch_enable(&switch_mm_always_ibpb);
-                       spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
-                       break;
-               case SPECTRE_V2_USER_CMD_PRCTL:
-               case SPECTRE_V2_USER_CMD_AUTO:
-               case SPECTRE_V2_USER_CMD_SECCOMP:
-                       static_branch_enable(&switch_mm_cond_ibpb);
-                       break;
-               }
+       if (!boot_cpu_has(X86_FEATURE_IBPB))
+               spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE;
 
-               pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
-                       static_key_enabled(&switch_mm_always_ibpb) ?
-                       "always-on" : "conditional");
+       if (!boot_cpu_has(X86_FEATURE_STIBP))
+               spectre_v2_user_stibp = SPECTRE_V2_USER_NONE;
+}
+
+static void __init spectre_v2_user_update_mitigation(void)
+{
+       if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
+               return;
+
+       /* The spectre_v2 cmd line can override spectre_v2_user options */
+       if (spectre_v2_cmd == SPECTRE_V2_CMD_NONE) {
+               spectre_v2_user_ibpb = SPECTRE_V2_USER_NONE;
+               spectre_v2_user_stibp = SPECTRE_V2_USER_NONE;
+       } else if (spectre_v2_cmd == SPECTRE_V2_CMD_FORCE) {
+               spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
+               spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT;
        }
 
        /*
@@ -1531,30 +1537,44 @@ spectre_v2_user_select_mitigation(void)
        if (!boot_cpu_has(X86_FEATURE_STIBP) ||
            !cpu_smt_possible() ||
            (spectre_v2_in_eibrs_mode(spectre_v2_enabled) &&
-            !boot_cpu_has(X86_FEATURE_AUTOIBRS)))
+            !boot_cpu_has(X86_FEATURE_AUTOIBRS))) {
+               spectre_v2_user_stibp = SPECTRE_V2_USER_NONE;
                return;
+       }
 
-       /*
-        * At this point, an STIBP mode other than "off" has been set.
-        * If STIBP support is not being forced, check if STIBP always-on
-        * is preferred.
-        */
-       if (mode != SPECTRE_V2_USER_STRICT &&
-           boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
-               mode = SPECTRE_V2_USER_STRICT_PREFERRED;
-
-       if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET ||
-           retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
-               if (mode != SPECTRE_V2_USER_STRICT &&
-                   mode != SPECTRE_V2_USER_STRICT_PREFERRED)
+       if (spectre_v2_user_stibp != SPECTRE_V2_USER_NONE &&
+           (retbleed_mitigation == RETBLEED_MITIGATION_UNRET ||
+            retbleed_mitigation == RETBLEED_MITIGATION_IBPB)) {
+               if (spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT &&
+                   spectre_v2_user_stibp != SPECTRE_V2_USER_STRICT_PREFERRED)
                        pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n");
-               mode = SPECTRE_V2_USER_STRICT_PREFERRED;
+               spectre_v2_user_stibp = SPECTRE_V2_USER_STRICT_PREFERRED;
        }
+       pr_info("%s\n", spectre_v2_user_strings[spectre_v2_user_stibp]);
+}
 
-       spectre_v2_user_stibp = mode;
+static void __init spectre_v2_user_apply_mitigation(void)
+{
+       /* Initialize Indirect Branch Prediction Barrier */
+       if (spectre_v2_user_ibpb != SPECTRE_V2_USER_NONE) {
+               static_branch_enable(&switch_vcpu_ibpb);
+
+               switch (spectre_v2_user_ibpb) {
+               case SPECTRE_V2_USER_STRICT:
+                       static_branch_enable(&switch_mm_always_ibpb);
+                       break;
+               case SPECTRE_V2_USER_PRCTL:
+               case SPECTRE_V2_USER_SECCOMP:
+                       static_branch_enable(&switch_mm_cond_ibpb);
+                       break;
+               default:
+                       break;
+               }
 
-set_mode:
-       pr_info("%s\n", spectre_v2_user_strings[mode]);
+               pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
+                       static_key_enabled(&switch_mm_always_ibpb) ?
+                       "always-on" : "conditional");
+       }
 }
 
 static const char * const spectre_v2_strings[] = {