x86/bugs: Restructure MDS mitigation

Restructure MDS mitigation selection to use select/update/apply
functions to create consistent vulnerability handling.

  [ bp: rename and beef up comment over VERW mitigation selected var for
    maximum clarity. ]

Signed-off-by: David Kaplan <david.kaplan@amd.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>
Link: https://lore.kernel.org/20250418161721.1855190-2-david.kaplan@amd.com

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 9131e612de170a6b2bf27c35651f8e824f3d016c..f697e6b41afd3496b3f201a4815872cc0413e337 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -34,6 +34,25 @@
 
 #include "cpu.h"
 
+/*
+ * Speculation Vulnerability Handling
+ *
+ * Each vulnerability is handled with the following functions:
+ *   <vuln>_select_mitigation() -- Selects a mitigation to use.  This should
+ *                                take into account all relevant command line
+ *                                options.
+ *   <vuln>_update_mitigation() -- This is called after all vulnerabilities have
+ *                                selected a mitigation, in case the selection
+ *                                may want to change based on other choices
+ *                                made.  This function is optional.
+ *   <vuln>_apply_mitigation() -- Enable the selected mitigation.
+ *
+ * The compile-time mitigation in all cases should be AUTO.  An explicit
+ * command-line option can override AUTO.  If no such option is
+ * provided, <vuln>_select_mitigation() will override AUTO to the best
+ * mitigation option.
+ */
+
 static void __init spectre_v1_select_mitigation(void);
 static void __init spectre_v2_select_mitigation(void);
 static void __init retbleed_select_mitigation(void);
@@ -41,6 +60,8 @@ static void __init spectre_v2_user_select_mitigation(void);
 static void __init ssb_select_mitigation(void);
 static void __init l1tf_select_mitigation(void);
 static void __init mds_select_mitigation(void);
+static void __init mds_update_mitigation(void);
+static void __init mds_apply_mitigation(void);
 static void __init md_clear_update_mitigation(void);
 static void __init md_clear_select_mitigation(void);
 static void __init taa_select_mitigation(void);
@@ -172,6 +193,7 @@ void __init cpu_select_mitigations(void)
        spectre_v2_user_select_mitigation();
        ssb_select_mitigation();
        l1tf_select_mitigation();
+       mds_select_mitigation();
        md_clear_select_mitigation();
        srbds_select_mitigation();
        l1d_flush_select_mitigation();
@@ -182,6 +204,14 @@ void __init cpu_select_mitigations(void)
         */
        srso_select_mitigation();
        gds_select_mitigation();
+
+       /*
+        * After mitigations are selected, some may need to update their
+        * choices.
+        */
+       mds_update_mitigation();
+
+       mds_apply_mitigation();
 }
 
 /*
@@ -284,6 +314,12 @@ enum rfds_mitigations {
 static enum rfds_mitigations rfds_mitigation __ro_after_init =
        IS_ENABLED(CONFIG_MITIGATION_RFDS) ? RFDS_MITIGATION_AUTO : RFDS_MITIGATION_OFF;
 
+/*
+ * Set if any of MDS/TAA/MMIO/RFDS are going to enable VERW clearing
+ * through X86_FEATURE_CLEAR_CPU_BUF on kernel and guest entry.
+ */
+static bool verw_clear_cpu_buf_mitigation_selected __ro_after_init;
+
 static void __init mds_select_mitigation(void)
 {
        if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) {
@@ -294,12 +330,34 @@ static void __init mds_select_mitigation(void)
        if (mds_mitigation == MDS_MITIGATION_AUTO)
                mds_mitigation = MDS_MITIGATION_FULL;
 
+       if (mds_mitigation == MDS_MITIGATION_OFF)
+               return;
+
+       verw_clear_cpu_buf_mitigation_selected = true;
+}
+
+static void __init mds_update_mitigation(void)
+{
+       if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off())
+               return;
+
+       /* If TAA, MMIO, or RFDS are being mitigated, MDS gets mitigated too. */
+       if (verw_clear_cpu_buf_mitigation_selected)
+               mds_mitigation = MDS_MITIGATION_FULL;
+
        if (mds_mitigation == MDS_MITIGATION_FULL) {
                if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
                        mds_mitigation = MDS_MITIGATION_VMWERV;
+       }
 
-               setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
+       pr_info("%s\n", mds_strings[mds_mitigation]);
+}
 
+static void __init mds_apply_mitigation(void)
+{
+       if (mds_mitigation == MDS_MITIGATION_FULL ||
+           mds_mitigation == MDS_MITIGATION_VMWERV) {
+               setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
                if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
                    (mds_nosmt || cpu_mitigations_auto_nosmt()))
                        cpu_smt_disable(false);
@@ -602,7 +660,6 @@ out:
 
 static void __init md_clear_select_mitigation(void)
 {
-       mds_select_mitigation();
        taa_select_mitigation();
        mmio_select_mitigation();
        rfds_select_mitigation();