x86/bugs: Enable IBPB flush on BPF JIT allocation

Enable hardening against JIT spraying when Spectre-v2 mitigations are in
use. Specifically, issue an IBPB flush on BPF JIT memory reuse. Skip
enabling the IBPB flush if the BPF dispatcher is already using a retpoline
sequence.

This hardening applies only when BPF-JIT is in use. Guard the enabling
under CONFIG_BPF_JIT so that bugs.c still builds with CONFIG_BPF_JIT=n.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Dave Hansen <dave.hansen@linux.intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 4f4b5e8a157430a35e42d7bb205222ffe8fbecc7..b68892e6d58c471ad6a2541bef0264d8acc7303e 100644 (file)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -388,6 +388,10 @@ extern void srso_alias_return_thunk(void);
 extern void entry_untrain_ret(void);
 extern void write_ibpb(void);
 
+#ifdef CONFIG_BPF_JIT
+extern void bpf_arch_ibpb(void);
+#endif
+
 #ifdef CONFIG_X86_64
 extern void clear_bhb_loop(void);
 #endif

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 83f51cab0b1e393e79123002869bb3da6f00e3d2..d9af230c0512596da833bdec7e4c327a96be3efc 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -16,6 +16,7 @@
 #include <linux/sched/smt.h>
 #include <linux/pgtable.h>
 #include <linux/bpf.h>
+#include <linux/filter.h>
 #include <linux/kvm_types.h>
 
 #include <asm/spec-ctrl.h>
@@ -1651,8 +1652,21 @@ static inline const char *spectre_v2_module_string(void)
 {
        return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 }
+
+/*
+ * The "retpoline sequence" is the "call;mov;ret" sequence that
+ * replaces normal indirect branch instructions. Differentiate
+ * *the* retpoline sequence from the LFENCE-prefixed indirect
+ * branches that simply use the retpoline infrastructure.
+ */
+static inline bool retpoline_seq_enabled(void)
+{
+       return boot_cpu_has(X86_FEATURE_RETPOLINE) && !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE);
+}
+
 #else
 static inline const char *spectre_v2_module_string(void) { return ""; }
+static inline bool retpoline_seq_enabled(void) { return false; }
 #endif
 
 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
@@ -2095,8 +2109,7 @@ static void __init bhi_apply_mitigation(void)
                return;
 
        /* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */
-       if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
-           !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) {
+       if (retpoline_seq_enabled()) {
                spec_ctrl_disable_kernel_rrsba();
                if (rrsba_disabled)
                        return;
@@ -2238,6 +2251,27 @@ static void __init spectre_v2_update_mitigation(void)
                pr_info("%s\n", spectre_v2_strings[spectre_v2_enabled]);
 }
 
+#ifdef CONFIG_BPF_JIT
+static void __bpf_arch_ibpb(void *unused)
+{
+       write_ibpb();
+}
+
+void bpf_arch_ibpb(void)
+{
+       on_each_cpu(__bpf_arch_ibpb, NULL, 1);
+}
+
+static bool __init cpu_wants_ibpb_bpf(void)
+{
+       /* A genuine retpoline already neutralizes ring0 indirect predictions */
+       if (retpoline_seq_enabled())
+               return false;
+
+       return boot_cpu_has(X86_FEATURE_IBPB);
+}
+#endif
+
 static void __init spectre_v2_apply_mitigation(void)
 {
        if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
@@ -2314,6 +2348,14 @@ static void __init spectre_v2_apply_mitigation(void)
                setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
                pr_info("Enabling Restricted Speculation for firmware calls\n");
        }
+
+#ifdef CONFIG_BPF_JIT
+       if (cpu_wants_ibpb_bpf()) {
+               static_call_update(bpf_arch_pred_flush, bpf_arch_ibpb);
+               static_branch_enable(&bpf_pred_flush_enabled);
+               pr_info("Enabling IBPB for BPF\n");
+       }
+#endif
 }
 
 static void update_stibp_msr(void * __unused)
@@ -3490,9 +3532,7 @@ static const char *spectre_bhi_state(void)
                return "; BHI: BHI_DIS_S";
        else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP))
                return "; BHI: SW loop, KVM: SW loop";
-       else if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
-                !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) &&
-                rrsba_disabled)
+       else if (retpoline_seq_enabled() && rrsba_disabled)
                return "; BHI: Retpoline";
        else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_VMEXIT))
                return "; BHI: Vulnerable, KVM: SW loop";