x86/bugs: Restructure MMIO mitigation

Restructure MMIO mitigation to use select/update/apply functions to
create consistent vulnerability handling.

Signed-off-by: David Kaplan <david.kaplan@amd.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>
Link: https://lore.kernel.org/20250418161721.1855190-4-david.kaplan@amd.com

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 5db21d2ee87dd175f1f45ec5b68ae89ab4418a35..bc74c22e927c59315d0bbba77a7f6aaccc20b12b 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -68,6 +68,8 @@ static void __init taa_select_mitigation(void);
 static void __init taa_update_mitigation(void);
 static void __init taa_apply_mitigation(void);
 static void __init mmio_select_mitigation(void);
+static void __init mmio_update_mitigation(void);
+static void __init mmio_apply_mitigation(void);
 static void __init srbds_select_mitigation(void);
 static void __init l1d_flush_select_mitigation(void);
 static void __init srso_select_mitigation(void);
@@ -197,6 +199,7 @@ void __init cpu_select_mitigations(void)
        l1tf_select_mitigation();
        mds_select_mitigation();
        taa_select_mitigation();
+       mmio_select_mitigation();
        md_clear_select_mitigation();
        srbds_select_mitigation();
        l1d_flush_select_mitigation();
@@ -214,9 +217,11 @@ void __init cpu_select_mitigations(void)
         */
        mds_update_mitigation();
        taa_update_mitigation();
+       mmio_update_mitigation();
 
        mds_apply_mitigation();
        taa_apply_mitigation();
+       mmio_apply_mitigation();
 }
 
 /*
@@ -520,25 +525,62 @@ static void __init mmio_select_mitigation(void)
                return;
        }
 
+       /* Microcode will be checked in mmio_update_mitigation(). */
+       if (mmio_mitigation == MMIO_MITIGATION_AUTO)
+               mmio_mitigation = MMIO_MITIGATION_VERW;
+
        if (mmio_mitigation == MMIO_MITIGATION_OFF)
                return;
 
        /*
         * Enable CPU buffer clear mitigation for host and VMM, if also affected
-        * by MDS or TAA. Otherwise, enable mitigation for VMM only.
+        * by MDS or TAA.
         */
-       if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) &&
-                                             boot_cpu_has(X86_FEATURE_RTM)))
-               setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
+       if (boot_cpu_has_bug(X86_BUG_MDS) || taa_vulnerable())
+               verw_clear_cpu_buf_mitigation_selected = true;
+}
+
+static void __init mmio_update_mitigation(void)
+{
+       if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || cpu_mitigations_off())
+               return;
+
+       if (verw_clear_cpu_buf_mitigation_selected)
+               mmio_mitigation = MMIO_MITIGATION_VERW;
+
+       if (mmio_mitigation == MMIO_MITIGATION_VERW) {
+               /*
+                * Check if the system has the right microcode.
+                *
+                * CPU Fill buffer clear mitigation is enumerated by either an explicit
+                * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS
+                * affected systems.
+                */
+               if (!((x86_arch_cap_msr & ARCH_CAP_FB_CLEAR) ||
+                     (boot_cpu_has(X86_FEATURE_MD_CLEAR) &&
+                      boot_cpu_has(X86_FEATURE_FLUSH_L1D) &&
+                    !(x86_arch_cap_msr & ARCH_CAP_MDS_NO))))
+                       mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED;
+       }
+
+       pr_info("%s\n", mmio_strings[mmio_mitigation]);
+}
+
+static void __init mmio_apply_mitigation(void)
+{
+       if (mmio_mitigation == MMIO_MITIGATION_OFF)
+               return;
 
        /*
-        * X86_FEATURE_CLEAR_CPU_BUF could be enabled by other VERW based
-        * mitigations, disable KVM-only mitigation in that case.
+        * Only enable the VMM mitigation if the CPU buffer clear mitigation is
+        * not being used.
         */
-       if (boot_cpu_has(X86_FEATURE_CLEAR_CPU_BUF))
+       if (verw_clear_cpu_buf_mitigation_selected) {
+               setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
                static_branch_disable(&cpu_buf_vm_clear);
-       else
+       } else {
                static_branch_enable(&cpu_buf_vm_clear);
+       }
 
        /*
         * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can
@@ -548,21 +590,6 @@ static void __init mmio_select_mitigation(void)
        if (!(x86_arch_cap_msr & ARCH_CAP_FBSDP_NO))
                static_branch_enable(&mds_idle_clear);
 
-       /*
-        * Check if the system has the right microcode.
-        *
-        * CPU Fill buffer clear mitigation is enumerated by either an explicit
-        * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS
-        * affected systems.
-        */
-       if ((x86_arch_cap_msr & ARCH_CAP_FB_CLEAR) ||
-           (boot_cpu_has(X86_FEATURE_MD_CLEAR) &&
-            boot_cpu_has(X86_FEATURE_FLUSH_L1D) &&
-            !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)))
-               mmio_mitigation = MMIO_MITIGATION_VERW;
-       else
-               mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED;
-
        if (mmio_nosmt || cpu_mitigations_auto_nosmt())
                cpu_smt_disable(false);
 }
@@ -685,7 +712,6 @@ out:
 
 static void __init md_clear_select_mitigation(void)
 {
-       mmio_select_mitigation();
        rfds_select_mitigation();
 
        /*