x86/bugs: Restructure spectre_v2 mitigation

Restructure spectre_v2 to use select/update/apply functions to create
consistent vulnerability handling.

The spectre_v2 mitigation may be updated based on the selected retbleed
mitigation.

Signed-off-by: David Kaplan <david.kaplan@amd.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>
Link: https://lore.kernel.org/20250418161721.1855190-14-david.kaplan@amd.com

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 100a320443856311b84d7f5d625f9ff9a4c22888..93d07438eea78e06fffd32bf4a7cb2e19191b6d2 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -56,6 +56,8 @@
 static void __init spectre_v1_select_mitigation(void);
 static void __init spectre_v1_apply_mitigation(void);
 static void __init spectre_v2_select_mitigation(void);
+static void __init spectre_v2_update_mitigation(void);
+static void __init spectre_v2_apply_mitigation(void);
 static void __init retbleed_select_mitigation(void);
 static void __init retbleed_update_mitigation(void);
 static void __init retbleed_apply_mitigation(void);
@@ -217,6 +219,12 @@ void __init cpu_select_mitigations(void)
         * After mitigations are selected, some may need to update their
         * choices.
         */
+       spectre_v2_update_mitigation();
+       /*
+        * retbleed_update_mitigation() relies on the state set by
+        * spectre_v2_update_mitigation(); specifically it wants to know about
+        * spectre_v2=ibrs.
+        */
        retbleed_update_mitigation();
 
        /*
@@ -232,6 +240,7 @@ void __init cpu_select_mitigations(void)
        bhi_update_mitigation();
 
        spectre_v1_apply_mitigation();
+       spectre_v2_apply_mitigation();
        retbleed_apply_mitigation();
        spectre_v2_user_apply_mitigation();
        mds_apply_mitigation();
@@ -1878,75 +1887,80 @@ static void __init bhi_apply_mitigation(void)
 
 static void __init spectre_v2_select_mitigation(void)
 {
-       enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
-       enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
+       spectre_v2_cmd = spectre_v2_parse_cmdline();
 
-       /*
-        * If the CPU is not affected and the command line mode is NONE or AUTO
-        * then nothing to do.
-        */
        if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
-           (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
+           (spectre_v2_cmd == SPECTRE_V2_CMD_NONE || spectre_v2_cmd == SPECTRE_V2_CMD_AUTO))
                return;
 
-       switch (cmd) {
+       switch (spectre_v2_cmd) {
        case SPECTRE_V2_CMD_NONE:
                return;
 
        case SPECTRE_V2_CMD_FORCE:
        case SPECTRE_V2_CMD_AUTO:
                if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
-                       mode = SPECTRE_V2_EIBRS;
-                       break;
-               }
-
-               if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) &&
-                   boot_cpu_has_bug(X86_BUG_RETBLEED) &&
-                   retbleed_mitigation != RETBLEED_MITIGATION_NONE &&
-                   retbleed_mitigation != RETBLEED_MITIGATION_STUFF &&
-                   boot_cpu_has(X86_FEATURE_IBRS) &&
-                   boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
-                       mode = SPECTRE_V2_IBRS;
+                       spectre_v2_enabled = SPECTRE_V2_EIBRS;
                        break;
                }
 
-               mode = spectre_v2_select_retpoline();
+               spectre_v2_enabled = spectre_v2_select_retpoline();
                break;
 
        case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
                pr_err(SPECTRE_V2_LFENCE_MSG);
-               mode = SPECTRE_V2_LFENCE;
+               spectre_v2_enabled = SPECTRE_V2_LFENCE;
                break;
 
        case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
-               mode = SPECTRE_V2_RETPOLINE;
+               spectre_v2_enabled = SPECTRE_V2_RETPOLINE;
                break;
 
        case SPECTRE_V2_CMD_RETPOLINE:
-               mode = spectre_v2_select_retpoline();
+               spectre_v2_enabled = spectre_v2_select_retpoline();
                break;
 
        case SPECTRE_V2_CMD_IBRS:
-               mode = SPECTRE_V2_IBRS;
+               spectre_v2_enabled = SPECTRE_V2_IBRS;
                break;
 
        case SPECTRE_V2_CMD_EIBRS:
-               mode = SPECTRE_V2_EIBRS;
+               spectre_v2_enabled = SPECTRE_V2_EIBRS;
                break;
 
        case SPECTRE_V2_CMD_EIBRS_LFENCE:
-               mode = SPECTRE_V2_EIBRS_LFENCE;
+               spectre_v2_enabled = SPECTRE_V2_EIBRS_LFENCE;
                break;
 
        case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
-               mode = SPECTRE_V2_EIBRS_RETPOLINE;
+               spectre_v2_enabled = SPECTRE_V2_EIBRS_RETPOLINE;
                break;
        }
+}
 
-       if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
+static void __init spectre_v2_update_mitigation(void)
+{
+       if (spectre_v2_cmd == SPECTRE_V2_CMD_AUTO) {
+               if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) &&
+                   boot_cpu_has_bug(X86_BUG_RETBLEED) &&
+                   retbleed_mitigation != RETBLEED_MITIGATION_NONE &&
+                   retbleed_mitigation != RETBLEED_MITIGATION_STUFF &&
+                   boot_cpu_has(X86_FEATURE_IBRS) &&
+                   boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
+                       spectre_v2_enabled = SPECTRE_V2_IBRS;
+               }
+       }
+
+       if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && !cpu_mitigations_off())
+               pr_info("%s\n", spectre_v2_strings[spectre_v2_enabled]);
+}
+
+static void __init spectre_v2_apply_mitigation(void)
+{
+       if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
                pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
 
-       if (spectre_v2_in_ibrs_mode(mode)) {
+       if (spectre_v2_in_ibrs_mode(spectre_v2_enabled)) {
                if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) {
                        msr_set_bit(MSR_EFER, _EFER_AUTOIBRS);
                } else {
@@ -1955,8 +1969,10 @@ static void __init spectre_v2_select_mitigation(void)
                }
        }
 
-       switch (mode) {
+       switch (spectre_v2_enabled) {
        case SPECTRE_V2_NONE:
+               return;
+
        case SPECTRE_V2_EIBRS:
                break;
 
@@ -1982,15 +1998,12 @@ static void __init spectre_v2_select_mitigation(void)
         * JMPs gets protection against BHI and Intramode-BTI, but RET
         * prediction from a non-RSB predictor is still a risk.
         */
-       if (mode == SPECTRE_V2_EIBRS_LFENCE ||
-           mode == SPECTRE_V2_EIBRS_RETPOLINE ||
-           mode == SPECTRE_V2_RETPOLINE)
+       if (spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE ||
+           spectre_v2_enabled == SPECTRE_V2_EIBRS_RETPOLINE ||
+           spectre_v2_enabled == SPECTRE_V2_RETPOLINE)
                spec_ctrl_disable_kernel_rrsba();
 
-       spectre_v2_enabled = mode;
-       pr_info("%s\n", spectre_v2_strings[mode]);
-
-       spectre_v2_select_rsb_mitigation(mode);
+       spectre_v2_select_rsb_mitigation(spectre_v2_enabled);
 
        /*
         * Retpoline protects the kernel, but doesn't protect firmware.  IBRS
@@ -1998,10 +2011,10 @@ static void __init spectre_v2_select_mitigation(void)
         * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't
         * otherwise enabled.
         *
-        * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
-        * the user might select retpoline on the kernel command line and if
-        * the CPU supports Enhanced IBRS, kernel might un-intentionally not
-        * enable IBRS around firmware calls.
+        * Use "spectre_v2_enabled" to check Enhanced IBRS instead of
+        * boot_cpu_has(), because the user might select retpoline on the kernel
+        * command line and if the CPU supports Enhanced IBRS, kernel might
+        * un-intentionally not enable IBRS around firmware calls.
         */
        if (boot_cpu_has_bug(X86_BUG_RETBLEED) &&
            boot_cpu_has(X86_FEATURE_IBPB) &&
@@ -2013,13 +2026,11 @@ static void __init spectre_v2_select_mitigation(void)
                        pr_info("Enabling Speculation Barrier for firmware calls\n");
                }
 
-       } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) {
+       } else if (boot_cpu_has(X86_FEATURE_IBRS) &&
+                  !spectre_v2_in_ibrs_mode(spectre_v2_enabled)) {
                setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
                pr_info("Enabling Restricted Speculation for firmware calls\n");
        }
-
-       /* Set up IBPB and STIBP depending on the general spectre V2 command */
-       spectre_v2_cmd = cmd;
 }
 
 static void update_stibp_msr(void * __unused)