arm64: bpf: Add BHB mitigation to the epilogue for cBPF programs

[ Upstream commit 0dfefc2ea2f29ced2416017d7e5b1253a54c2735 ]

A malicious BPF program may manipulate the branch history to influence
what the hardware speculates will happen next.

On exit from a BPF program, emit the BHB mititgation sequence.

This is only applied for 'classic' cBPF programs that are loaded by
seccomp.

Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Pu Lehui <pulehui@huawei.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/arm64/include/asm/spectre.h b/arch/arm64/include/asm/spectre.h
index 3c211c04b8d9c96b594751ad95ce920114b48f73..09073bb6712a02168195b5dc58e7647b79384aca 100644 (file)
--- a/arch/arm64/include/asm/spectre.h
+++ b/arch/arm64/include/asm/spectre.h
@@ -32,6 +32,7 @@ void spectre_v4_enable_task_mitigation(struct task_struct *tsk);
 
 enum mitigation_state arm64_get_spectre_bhb_state(void);
 bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope);
+extern bool __nospectre_bhb;
 u8 get_spectre_bhb_loop_value(void);
 bool is_spectre_bhb_fw_mitigated(void);
 void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused);

diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c
index 7728026d98f75b7f7c693ef8284957fa727dd9fe..c218a9ef23d06098bf708fcc8db98dc7012d6f32 100644 (file)
--- a/arch/arm64/kernel/proton-pack.c
+++ b/arch/arm64/kernel/proton-pack.c
@@ -1088,7 +1088,7 @@ static void kvm_setup_bhb_slot(const char *hyp_vecs_start) { }
 #endif /* CONFIG_KVM */
 
 static bool spectre_bhb_fw_mitigated;
-static bool __read_mostly __nospectre_bhb;
+bool __read_mostly __nospectre_bhb;
 static int __init parse_spectre_bhb_param(char *str)
 {
        __nospectre_bhb = true;

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 18627cbd6da4ef45a90a549fd4f06a0fbce8dc6f..5c3f82c168a2774c36d6fa411d5c110204585b25 100644 (file)
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -7,14 +7,17 @@
 
 #define pr_fmt(fmt) "bpf_jit: " fmt
 
+#include <linux/arm-smccc.h>
 #include <linux/bitfield.h>
 #include <linux/bpf.h>
+#include <linux/cpu.h>
 #include <linux/filter.h>
 #include <linux/printk.h>
 #include <linux/slab.h>
 
 #include <asm/byteorder.h>
 #include <asm/cacheflush.h>
+#include <asm/cpufeature.h>
 #include <asm/debug-monitors.h>
 #include <asm/set_memory.h>
 
@@ -328,7 +331,48 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
 #undef jmp_offset
 }
 
-static void build_epilogue(struct jit_ctx *ctx)
+/* Clobbers BPF registers 1-4, aka x0-x3 */
+static void __maybe_unused build_bhb_mitigation(struct jit_ctx *ctx)
+{
+       const u8 r1 = bpf2a64[BPF_REG_1]; /* aka x0 */
+       u8 k = get_spectre_bhb_loop_value();
+
+       if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) ||
+           cpu_mitigations_off() || __nospectre_bhb ||
+           arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE)
+               return;
+
+       if (supports_clearbhb(SCOPE_SYSTEM)) {
+               emit(aarch64_insn_gen_hint(AARCH64_INSN_HINT_CLEARBHB), ctx);
+               return;
+       }
+
+       if (k) {
+               emit_a64_mov_i64(r1, k, ctx);
+               emit(A64_B(1), ctx);
+               emit(A64_SUBS_I(true, r1, r1, 1), ctx);
+               emit(A64_B_(A64_COND_NE, -2), ctx);
+               emit(aarch64_insn_gen_dsb(AARCH64_INSN_MB_ISH), ctx);
+               emit(aarch64_insn_get_isb_value(), ctx);
+       }
+
+       if (is_spectre_bhb_fw_mitigated()) {
+               emit(A64_ORR_I(false, r1, AARCH64_INSN_REG_ZR,
+                              ARM_SMCCC_ARCH_WORKAROUND_3), ctx);
+               switch (arm_smccc_1_1_get_conduit()) {
+               case SMCCC_CONDUIT_HVC:
+                       emit(aarch64_insn_get_hvc_value(), ctx);
+                       break;
+               case SMCCC_CONDUIT_SMC:
+                       emit(aarch64_insn_get_smc_value(), ctx);
+                       break;
+               default:
+                       pr_err_once("Firmware mitigation enabled with unknown conduit\n");
+               }
+       }
+}
+
+static void build_epilogue(struct jit_ctx *ctx, bool was_classic)
 {
        const u8 r0 = bpf2a64[BPF_REG_0];
        const u8 r6 = bpf2a64[BPF_REG_6];
@@ -347,10 +391,13 @@ static void build_epilogue(struct jit_ctx *ctx)
        emit(A64_POP(r8, r9, A64_SP), ctx);
        emit(A64_POP(r6, r7, A64_SP), ctx);
 
+       if (was_classic)
+               build_bhb_mitigation(ctx);
+
        /* Restore FP/LR registers */
        emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
 
-       /* Set return value */
+       /* Move the return value from bpf:r0 (aka x7) to x0 */
        emit(A64_MOV(1, A64_R(0), r0), ctx);
 
        emit(A64_RET(A64_LR), ctx);
@@ -1057,7 +1104,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
        }
 
        ctx.epilogue_offset = ctx.idx;
-       build_epilogue(&ctx);
+       build_epilogue(&ctx, was_classic);
 
        extable_size = prog->aux->num_exentries *
                sizeof(struct exception_table_entry);
@@ -1089,7 +1136,7 @@ skip_init_ctx:
                goto out_off;
        }
 
-       build_epilogue(&ctx);
+       build_epilogue(&ctx, was_classic);
 
        /* 3. Extra pass to validate JITed code. */
        if (validate_code(&ctx)) {