x86/its: Enable Indirect Target Selection mitigation

commit f4818881c47fd91fcb6d62373c57c7844e3de1c0 upstream.

Indirect Target Selection (ITS) is a bug in some pre-ADL Intel CPUs with
eIBRS. It affects prediction of indirect branch and RETs in the
lower half of cacheline. Due to ITS such branches may get wrongly predicted
to a target of (direct or indirect) branch that is located in the upper
half of the cacheline.

Scope of impact
===============

Guest/host isolation
--------------------
When eIBRS is used for guest/host isolation, the indirect branches in the
VMM may still be predicted with targets corresponding to branches in the
guest.

Intra-mode
----------
cBPF or other native gadgets can be used for intra-mode training and
disclosure using ITS.

User/kernel isolation
---------------------
When eIBRS is enabled user/kernel isolation is not impacted.

Indirect Branch Prediction Barrier (IBPB)
-----------------------------------------
After an IBPB, indirect branches may be predicted with targets
corresponding to direct branches which were executed prior to IBPB. This is
mitigated by a microcode update.

Add cmdline parameter indirect_target_selection=off|on|force to control the
mitigation to relocate the affected branches to an ITS-safe thunk i.e.
located in the upper half of cacheline. Also add the sysfs reporting.

When retpoline mitigation is deployed, ITS safe-thunks are not needed,
because retpoline sequence is already ITS-safe. Similarly, when call depth
tracking (CDT) mitigation is deployed (retbleed=stuff), ITS safe return
thunk is not used, as CDT prevents RSB-underflow.

To not overcomplicate things, ITS mitigation is not supported with
spectre-v2 lfence;jmp mitigation. Moreover, it is less practical to deploy
lfence;jmp mitigation on ITS affected parts anyways.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu
index 657bdee28d845a4710b71f05c8f3c39aacfb99a5..0426ec112155ec79143862806bbf8357ff3a2729 100644 (file)
--- a/Documentation/ABI/testing/sysfs-devices-system-cpu
+++ b/Documentation/ABI/testing/sysfs-devices-system-cpu
@@ -514,6 +514,7 @@ Description:        information about CPUs heterogeneity.
 
 What:          /sys/devices/system/cpu/vulnerabilities
                /sys/devices/system/cpu/vulnerabilities/gather_data_sampling
+               /sys/devices/system/cpu/vulnerabilities/indirect_target_selection
                /sys/devices/system/cpu/vulnerabilities/itlb_multihit
                /sys/devices/system/cpu/vulnerabilities/l1tf
                /sys/devices/system/cpu/vulnerabilities/mds

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 184f2f96f6a547f88aadc71f6ccd63a9fe8bf611..848b367ca95b5e5a998f05d74f3aa5144f0ff60f 100644 (file)
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2060,6 +2060,18 @@
                        different crypto accelerators. This option can be used
                        to achieve best performance for particular HW.
 
+       indirect_target_selection= [X86,Intel] Mitigation control for Indirect
+                       Target Selection(ITS) bug in Intel CPUs. Updated
+                       microcode is also required for a fix in IBPB.
+
+                       on:     Enable mitigation (default).
+                       off:    Disable mitigation.
+                       force:  Force the ITS bug and deploy default
+                               mitigation.
+
+                       For details see:
+                       Documentation/admin-guide/hw-vuln/indirect-target-selection.rst
+
        init=           [KNL]
                        Format: <full_path>
                        Run specified binary instead of /sbin/init as init
@@ -3331,6 +3343,7 @@
                                expose users to several CPU vulnerabilities.
                                Equivalent to: if nokaslr then kpti=0 [ARM64]
                                               gather_data_sampling=off [X86]
+                                              indirect_target_selection=off [X86]
                                               kvm.nx_huge_pages=off [X86]
                                               l1tf=off [X86]
                                               mds=off [X86]

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 1668ae8b71bbf58f7883168e529326b0a63f3184..bc4fa6d09d29d9b00ab91360930b51230ecb4540 100644 (file)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -421,11 +421,6 @@ extern void (*x86_return_thunk)(void);
 #ifdef CONFIG_CALL_DEPTH_TRACKING
 extern void __x86_return_skl(void);
 
-static inline void x86_set_skl_return_thunk(void)
-{
-       x86_return_thunk = &__x86_return_skl;
-}
-
 #define CALL_DEPTH_ACCOUNT                                     \
        ALTERNATIVE("",                                         \
                    __stringify(INCREMENT_CALL_DEPTH),          \
@@ -438,7 +433,6 @@ DECLARE_PER_CPU(u64, __x86_stuffs_count);
 DECLARE_PER_CPU(u64, __x86_ctxsw_count);
 #endif
 #else
-static inline void x86_set_skl_return_thunk(void) {}
 
 #define CALL_DEPTH_ACCOUNT ""
 

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 51f58776765bc048c26d881099c84fee708dd849..df5edadf8bbe24ffe40ca4c0a8648aad0f87a581 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -49,6 +49,7 @@ static void __init srbds_select_mitigation(void);
 static void __init l1d_flush_select_mitigation(void);
 static void __init srso_select_mitigation(void);
 static void __init gds_select_mitigation(void);
+static void __init its_select_mitigation(void);
 
 /* The base value of the SPEC_CTRL MSR without task-specific bits set */
 u64 x86_spec_ctrl_base;
@@ -67,6 +68,14 @@ static DEFINE_MUTEX(spec_ctrl_mutex);
 
 void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk;
 
+static void __init set_return_thunk(void *thunk)
+{
+       if (x86_return_thunk != __x86_return_thunk)
+               pr_warn("x86/bugs: return thunk changed\n");
+
+       x86_return_thunk = thunk;
+}
+
 /* Update SPEC_CTRL MSR and its cached copy unconditionally */
 static void update_spec_ctrl(u64 val)
 {
@@ -175,6 +184,7 @@ void __init cpu_select_mitigations(void)
         */
        srso_select_mitigation();
        gds_select_mitigation();
+       its_select_mitigation();
 }
 
 /*
@@ -1102,7 +1112,7 @@ do_cmd_auto:
                setup_force_cpu_cap(X86_FEATURE_RETHUNK);
                setup_force_cpu_cap(X86_FEATURE_UNRET);
 
-               x86_return_thunk = retbleed_return_thunk;
+               set_return_thunk(retbleed_return_thunk);
 
                if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
                    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
@@ -1136,7 +1146,9 @@ do_cmd_auto:
        case RETBLEED_MITIGATION_STUFF:
                setup_force_cpu_cap(X86_FEATURE_RETHUNK);
                setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH);
-               x86_set_skl_return_thunk();
+#ifdef CONFIG_CALL_DEPTH_TRACKING
+               set_return_thunk(&__x86_return_skl);
+#endif
                break;
 
        default:
@@ -1170,6 +1182,114 @@ do_cmd_auto:
        pr_info("%s\n", retbleed_strings[retbleed_mitigation]);
 }
 
+#undef pr_fmt
+#define pr_fmt(fmt)     "ITS: " fmt
+
+enum its_mitigation_cmd {
+       ITS_CMD_OFF,
+       ITS_CMD_ON,
+};
+
+enum its_mitigation {
+       ITS_MITIGATION_OFF,
+       ITS_MITIGATION_ALIGNED_THUNKS,
+       ITS_MITIGATION_RETPOLINE_STUFF,
+};
+
+static const char * const its_strings[] = {
+       [ITS_MITIGATION_OFF]                    = "Vulnerable",
+       [ITS_MITIGATION_ALIGNED_THUNKS]         = "Mitigation: Aligned branch/return thunks",
+       [ITS_MITIGATION_RETPOLINE_STUFF]        = "Mitigation: Retpolines, Stuffing RSB",
+};
+
+static enum its_mitigation its_mitigation __ro_after_init = ITS_MITIGATION_ALIGNED_THUNKS;
+
+static enum its_mitigation_cmd its_cmd __ro_after_init =
+       IS_ENABLED(CONFIG_MITIGATION_ITS) ? ITS_CMD_ON : ITS_CMD_OFF;
+
+static int __init its_parse_cmdline(char *str)
+{
+       if (!str)
+               return -EINVAL;
+
+       if (!IS_ENABLED(CONFIG_MITIGATION_ITS)) {
+               pr_err("Mitigation disabled at compile time, ignoring option (%s)", str);
+               return 0;
+       }
+
+       if (!strcmp(str, "off")) {
+               its_cmd = ITS_CMD_OFF;
+       } else if (!strcmp(str, "on")) {
+               its_cmd = ITS_CMD_ON;
+       } else if (!strcmp(str, "force")) {
+               its_cmd = ITS_CMD_ON;
+               setup_force_cpu_bug(X86_BUG_ITS);
+       } else {
+               pr_err("Ignoring unknown indirect_target_selection option (%s).", str);
+       }
+
+       return 0;
+}
+early_param("indirect_target_selection", its_parse_cmdline);
+
+static void __init its_select_mitigation(void)
+{
+       enum its_mitigation_cmd cmd = its_cmd;
+
+       if (!boot_cpu_has_bug(X86_BUG_ITS) || cpu_mitigations_off()) {
+               its_mitigation = ITS_MITIGATION_OFF;
+               return;
+       }
+
+       /* Retpoline+CDT mitigates ITS, bail out */
+       if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
+           boot_cpu_has(X86_FEATURE_CALL_DEPTH)) {
+               its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF;
+               goto out;
+       }
+
+       /* Exit early to avoid irrelevant warnings */
+       if (cmd == ITS_CMD_OFF) {
+               its_mitigation = ITS_MITIGATION_OFF;
+               goto out;
+       }
+       if (spectre_v2_enabled == SPECTRE_V2_NONE) {
+               pr_err("WARNING: Spectre-v2 mitigation is off, disabling ITS\n");
+               its_mitigation = ITS_MITIGATION_OFF;
+               goto out;
+       }
+       if (!IS_ENABLED(CONFIG_RETPOLINE) || !IS_ENABLED(CONFIG_RETHUNK)) {
+               pr_err("WARNING: ITS mitigation depends on retpoline and rethunk support\n");
+               its_mitigation = ITS_MITIGATION_OFF;
+               goto out;
+       }
+       if (IS_ENABLED(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)) {
+               pr_err("WARNING: ITS mitigation is not compatible with CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B\n");
+               its_mitigation = ITS_MITIGATION_OFF;
+               goto out;
+       }
+       if (boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) {
+               pr_err("WARNING: ITS mitigation is not compatible with lfence mitigation\n");
+               its_mitigation = ITS_MITIGATION_OFF;
+               goto out;
+       }
+
+       switch (cmd) {
+       case ITS_CMD_OFF:
+               its_mitigation = ITS_MITIGATION_OFF;
+               break;
+       case ITS_CMD_ON:
+               its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS;
+               if (!boot_cpu_has(X86_FEATURE_RETPOLINE))
+                       setup_force_cpu_cap(X86_FEATURE_INDIRECT_THUNK_ITS);
+               setup_force_cpu_cap(X86_FEATURE_RETHUNK);
+               set_return_thunk(its_return_thunk);
+               break;
+       }
+out:
+       pr_info("%s\n", its_strings[its_mitigation]);
+}
+
 #undef pr_fmt
 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 
@@ -2607,10 +2727,10 @@ static void __init srso_select_mitigation(void)
 
                        if (boot_cpu_data.x86 == 0x19) {
                                setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS);
-                               x86_return_thunk = srso_alias_return_thunk;
+                               set_return_thunk(srso_alias_return_thunk);
                        } else {
                                setup_force_cpu_cap(X86_FEATURE_SRSO);
-                               x86_return_thunk = srso_return_thunk;
+                               set_return_thunk(srso_return_thunk);
                        }
                        if (has_microcode)
                                srso_mitigation = SRSO_MITIGATION_SAFE_RET;
@@ -2794,6 +2914,11 @@ static ssize_t rfds_show_state(char *buf)
        return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]);
 }
 
+static ssize_t its_show_state(char *buf)
+{
+       return sysfs_emit(buf, "%s\n", its_strings[its_mitigation]);
+}
+
 static char *stibp_state(void)
 {
        if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) &&
@@ -2976,6 +3101,9 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr
        case X86_BUG_RFDS:
                return rfds_show_state(buf);
 
+       case X86_BUG_ITS:
+               return its_show_state(buf);
+
        default:
                break;
        }
@@ -3055,4 +3183,9 @@ ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attrib
 {
        return cpu_show_common(dev, attr, buf, X86_BUG_RFDS);
 }
+
+ssize_t cpu_show_indirect_target_selection(struct device *dev, struct device_attribute *attr, char *buf)
+{
+       return cpu_show_common(dev, attr, buf, X86_BUG_ITS);
+}
 #endif

diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
index ef427ee787a99bce2a66e7cb93d1d13046374fcf..a5cfc1bfad51fb770e0d667255e5a3649e3e3cd7 100644 (file)
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -566,6 +566,7 @@ CPU_SHOW_VULN_FALLBACK(retbleed);
 CPU_SHOW_VULN_FALLBACK(spec_rstack_overflow);
 CPU_SHOW_VULN_FALLBACK(gds);
 CPU_SHOW_VULN_FALLBACK(reg_file_data_sampling);
+CPU_SHOW_VULN_FALLBACK(indirect_target_selection);
 
 static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL);
 static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL);
@@ -581,6 +582,7 @@ static DEVICE_ATTR(retbleed, 0444, cpu_show_retbleed, NULL);
 static DEVICE_ATTR(spec_rstack_overflow, 0444, cpu_show_spec_rstack_overflow, NULL);
 static DEVICE_ATTR(gather_data_sampling, 0444, cpu_show_gds, NULL);
 static DEVICE_ATTR(reg_file_data_sampling, 0444, cpu_show_reg_file_data_sampling, NULL);
+static DEVICE_ATTR(indirect_target_selection, 0444, cpu_show_indirect_target_selection, NULL);
 
 static struct attribute *cpu_root_vulnerabilities_attrs[] = {
        &dev_attr_meltdown.attr,
@@ -597,6 +599,7 @@ static struct attribute *cpu_root_vulnerabilities_attrs[] = {
        &dev_attr_spec_rstack_overflow.attr,
        &dev_attr_gather_data_sampling.attr,
        &dev_attr_reg_file_data_sampling.attr,
+       &dev_attr_indirect_target_selection.attr,
        NULL
 };
 

diff --git a/include/linux/cpu.h b/include/linux/cpu.h
index a7d91a167a8b6452534b2509872cecfc3b708de4..20db7fc0651f3ca1ff3b5e89b98efbe47947ff18 100644 (file)
--- a/include/linux/cpu.h
+++ b/include/linux/cpu.h
@@ -77,6 +77,8 @@ extern ssize_t cpu_show_gds(struct device *dev,
                            struct device_attribute *attr, char *buf);
 extern ssize_t cpu_show_reg_file_data_sampling(struct device *dev,
                                               struct device_attribute *attr, char *buf);
+extern ssize_t cpu_show_indirect_target_selection(struct device *dev,
+                                                 struct device_attribute *attr, char *buf);
 
 extern __printf(4, 5)
 struct device *cpu_device_create(struct device *parent, void *drvdata,