x86/cpu: Support AMD Automatic IBRS

commit e7862eda309ecfccc36bb5558d937ed3ace07f3f upstream.

The AMD Zen4 core supports a new feature called Automatic IBRS.

It is a "set-and-forget" feature that means that, like Intel's Enhanced IBRS,
h/w manages its IBRS mitigation resources automatically across CPL transitions.

The feature is advertised by CPUID_Fn80000021_EAX bit 8 and is enabled by
setting MSR C000_0080 (EFER) bit 21.

Enable Automatic IBRS by default if the CPU feature is present.  It typically
provides greater performance over the incumbent generic retpolines mitigation.

Reuse the SPECTRE_V2_EIBRS spectre_v2_mitigation enum.  AMD Automatic IBRS and
Intel Enhanced IBRS have similar enablement.  Add NO_EIBRS_PBRSB to
cpu_vuln_whitelist, since AMD Automatic IBRS isn't affected by PBRSB-eIBRS.

The kernel command line option spectre_v2=eibrs is used to select AMD Automatic
IBRS, if available.

Signed-off-by: Kim Phillips <kim.phillips@amd.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Acked-by: Sean Christopherson <seanjc@google.com>
Acked-by: Dave Hansen <dave.hansen@linux.intel.com>
Link: https://lore.kernel.org/r/20230124163319.2277355-8-kim.phillips@amd.com
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/Documentation/admin-guide/hw-vuln/spectre.rst b/Documentation/admin-guide/hw-vuln/spectre.rst
index 3c95970ccfeb01e92e9951fe8339571408f37b0c..5420a9f21b7dae17ddedb5f78c9b4a1412df06e5 100644 (file)
--- a/Documentation/admin-guide/hw-vuln/spectre.rst
+++ b/Documentation/admin-guide/hw-vuln/spectre.rst
@@ -622,9 +622,9 @@ kernel command line.
                 retpoline,generic       Retpolines
                 retpoline,lfence        LFENCE; indirect branch
                 retpoline,amd           alias for retpoline,lfence
-                eibrs                   enhanced IBRS
-                eibrs,retpoline         enhanced IBRS + Retpolines
-                eibrs,lfence            enhanced IBRS + LFENCE
+                eibrs                   Enhanced/Auto IBRS
+                eibrs,retpoline         Enhanced/Auto IBRS + Retpolines
+                eibrs,lfence            Enhanced/Auto IBRS + LFENCE
                 ibrs                    use IBRS to protect kernel
 
                Not specifying this option is equivalent to

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index f1f7c068cf65b33566789d32a67418756b1c0c45..baba1b6d3efad1068fb0cea469418b4fc2f05c0a 100644 (file)
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -5091,9 +5091,9 @@
                        retpoline,generic - Retpolines
                        retpoline,lfence  - LFENCE; indirect branch
                        retpoline,amd     - alias for retpoline,lfence
-                       eibrs             - enhanced IBRS
-                       eibrs,retpoline   - enhanced IBRS + Retpolines
-                       eibrs,lfence      - enhanced IBRS + LFENCE
+                       eibrs             - Enhanced/Auto IBRS
+                       eibrs,retpoline   - Enhanced/Auto IBRS + Retpolines
+                       eibrs,lfence      - Enhanced/Auto IBRS + LFENCE
                        ibrs              - use IBRS to protect kernel
 
                        Not specifying this option is equivalent to

diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 5a54c3685a0660d560751d48d0899df1f2281587..d0db474ddf9a2774087837ffbc97aedda83b1cbb 100644 (file)
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -403,6 +403,7 @@
 #define X86_FEATURE_SEV_ES             (19*32+ 3) /* AMD Secure Encrypted Virtualization - Encrypted State */
 #define X86_FEATURE_SME_COHERENT       (19*32+10) /* "" AMD hardware-enforced cache coherency */
 
+#define X86_FEATURE_AUTOIBRS           (20*32+ 8) /* "" Automatic IBRS */
 #define X86_FEATURE_SBPB               (20*32+27) /* "" Selective Branch Prediction Barrier */
 #define X86_FEATURE_IBPB_BRTYPE                (20*32+28) /* "" MSR_PRED_CMD[IBPB] flushes all branch type predictions */
 #define X86_FEATURE_SRSO_NO            (20*32+29) /* "" CPU is not affected by SRSO */

diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 7d7a3cbb8e0178751cadd8e3807d05c9f18a8b2c..7559b5fbc1114730aa0ca96fdece50ae2a74973b 100644 (file)
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -30,6 +30,7 @@
 #define _EFER_SVME             12 /* Enable virtualization */
 #define _EFER_LMSLE            13 /* Long Mode Segment Limit Enable */
 #define _EFER_FFXSR            14 /* Enable Fast FXSAVE/FXRSTOR */
+#define _EFER_AUTOIBRS         21 /* Enable Automatic IBRS */
 
 #define EFER_SCE               (1<<_EFER_SCE)
 #define EFER_LME               (1<<_EFER_LME)
@@ -38,6 +39,7 @@
 #define EFER_SVME              (1<<_EFER_SVME)
 #define EFER_LMSLE             (1<<_EFER_LMSLE)
 #define EFER_FFXSR             (1<<_EFER_FFXSR)
+#define EFER_AUTOIBRS          (1<<_EFER_AUTOIBRS)
 
 /* Intel MSRs. Some also available on other CPUs */
 

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index d9fda0b6eb19e2920c0c06c956b1884fae3ccc5b..e9a79bbec980ae2ee4351dfaf24e496343addec4 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -1293,9 +1293,9 @@ static const char * const spectre_v2_strings[] = {
        [SPECTRE_V2_NONE]                       = "Vulnerable",
        [SPECTRE_V2_RETPOLINE]                  = "Mitigation: Retpolines",
        [SPECTRE_V2_LFENCE]                     = "Mitigation: LFENCE",
-       [SPECTRE_V2_EIBRS]                      = "Mitigation: Enhanced IBRS",
-       [SPECTRE_V2_EIBRS_LFENCE]               = "Mitigation: Enhanced IBRS + LFENCE",
-       [SPECTRE_V2_EIBRS_RETPOLINE]            = "Mitigation: Enhanced IBRS + Retpolines",
+       [SPECTRE_V2_EIBRS]                      = "Mitigation: Enhanced / Automatic IBRS",
+       [SPECTRE_V2_EIBRS_LFENCE]               = "Mitigation: Enhanced / Automatic IBRS + LFENCE",
+       [SPECTRE_V2_EIBRS_RETPOLINE]            = "Mitigation: Enhanced / Automatic IBRS + Retpolines",
        [SPECTRE_V2_IBRS]                       = "Mitigation: IBRS",
 };
 
@@ -1364,7 +1364,7 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
             cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
             cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
            !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
-               pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
+               pr_err("%s selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n",
                       mitigation_options[i].option);
                return SPECTRE_V2_CMD_AUTO;
        }
@@ -1549,8 +1549,12 @@ static void __init spectre_v2_select_mitigation(void)
                pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
 
        if (spectre_v2_in_ibrs_mode(mode)) {
-               x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
-               update_spec_ctrl(x86_spec_ctrl_base);
+               if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) {
+                       msr_set_bit(MSR_EFER, _EFER_AUTOIBRS);
+               } else {
+                       x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
+                       update_spec_ctrl(x86_spec_ctrl_base);
+               }
        }
 
        switch (mode) {
@@ -1634,8 +1638,8 @@ static void __init spectre_v2_select_mitigation(void)
        /*
         * Retpoline protects the kernel, but doesn't protect firmware.  IBRS
         * and Enhanced IBRS protect firmware too, so enable IBRS around
-        * firmware calls only when IBRS / Enhanced IBRS aren't otherwise
-        * enabled.
+        * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't
+        * otherwise enabled.
         *
         * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
         * the user might select retpoline on the kernel command line and if

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 4ecc6072e9a48d18860e0eba9993423f337ad0cb..0c72ff732aa085528f00a405b43220a9e5f2c1d9 100644 (file)
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1098,8 +1098,8 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
        VULNWL_AMD(0x12,        NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
 
        /* FAMILY_ANY must be last, otherwise 0x0f - 0x12 matches won't work */
-       VULNWL_AMD(X86_FAMILY_ANY,      NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
-       VULNWL_HYGON(X86_FAMILY_ANY,    NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
+       VULNWL_AMD(X86_FAMILY_ANY,      NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
+       VULNWL_HYGON(X86_FAMILY_ANY,    NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
 
        /* Zhaoxin Family 7 */
        VULNWL(CENTAUR, 7, X86_MODEL_ANY,       NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO),
@@ -1219,8 +1219,16 @@ static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
           !cpu_has(c, X86_FEATURE_AMD_SSB_NO))
                setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
 
-       if (ia32_cap & ARCH_CAP_IBRS_ALL)
+       /*
+        * AMD's AutoIBRS is equivalent to Intel's eIBRS - use the Intel feature
+        * flag and protect from vendor-specific bugs via the whitelist.
+        */
+       if ((ia32_cap & ARCH_CAP_IBRS_ALL) || cpu_has(c, X86_FEATURE_AUTOIBRS)) {
                setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);
+               if (!cpu_matches(cpu_vuln_whitelist, NO_EIBRS_PBRSB) &&
+                   !(ia32_cap & ARCH_CAP_PBRSB_NO))
+                       setup_force_cpu_bug(X86_BUG_EIBRS_PBRSB);
+       }
 
        if (!cpu_matches(cpu_vuln_whitelist, NO_MDS) &&
            !(ia32_cap & ARCH_CAP_MDS_NO)) {
@@ -1282,11 +1290,6 @@ static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
                        setup_force_cpu_bug(X86_BUG_RETBLEED);
        }
 
-       if (cpu_has(c, X86_FEATURE_IBRS_ENHANCED) &&
-           !cpu_matches(cpu_vuln_whitelist, NO_EIBRS_PBRSB) &&
-           !(ia32_cap & ARCH_CAP_PBRSB_NO))
-               setup_force_cpu_bug(X86_BUG_EIBRS_PBRSB);
-
        /*
         * Check if CPU is vulnerable to GDS. If running in a virtual machine on
         * an affected processor, the VMM may have disabled the use of GATHER by