x86/bpf: Call branch history clearing sequence on exit

commit d4e89d212d401672e9cdfe825d947ee3a9fbe3f5 upstream.

Classic BPF programs have been identified as potential vectors for
intra-mode Branch Target Injection (BTI) attacks. Classic BPF programs can
be run by unprivileged users. They allow unprivileged code to execute
inside the kernel. Attackers can use unprivileged cBPF to craft branch
history in kernel mode that can influence the target of indirect branches.

Introduce a branch history buffer (BHB) clearing sequence during the JIT
compilation of classic BPF programs. The clearing sequence is the same as
is used in previous mitigations to protect syscalls. Since eBPF programs
already have their own mitigations in place, only insert the call on
classic programs that aren't run by privileged users.

Signed-off-by: Daniel Sneddon <daniel.sneddon@linux.intel.com>
Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index a50c99e9b5c01f155a06a1b51cc3cc06dbfe970e..4cdc301952b9b7fdafa28b163e8bb543a8085711 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1072,6 +1072,30 @@ static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op)
 #define RESTORE_TAIL_CALL_CNT(stack)                           \
        EMIT3_off32(0x48, 0x8B, 0x85, -round_up(stack, 8) - 8)
 
+static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip,
+                                   struct bpf_prog *bpf_prog)
+{
+       u8 *prog = *pprog;
+       u8 *func;
+
+       if (cpu_feature_enabled(X86_FEATURE_CLEAR_BHB_LOOP)) {
+               /* The clearing sequence clobbers eax and ecx. */
+               EMIT1(0x50); /* push rax */
+               EMIT1(0x51); /* push rcx */
+               ip += 2;
+
+               func = (u8 *)clear_bhb_loop;
+               ip += x86_call_depth_emit_accounting(&prog, func);
+
+               if (emit_call(&prog, func, ip))
+                       return -EINVAL;
+               EMIT1(0x59); /* pop rcx */
+               EMIT1(0x58); /* pop rax */
+       }
+       *pprog = prog;
+       return 0;
+}
+
 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image,
                  int oldproglen, struct jit_context *ctx, bool jmp_padding)
 {
@@ -1945,6 +1969,15 @@ emit_jmp:
                        seen_exit = true;
                        /* Update cleanup_addr */
                        ctx->cleanup_addr = proglen;
+
+                       if (bpf_prog_was_classic(bpf_prog) &&
+                           !capable(CAP_SYS_ADMIN)) {
+                               u8 *ip = image + addrs[i - 1];
+
+                               if (emit_spectre_bhb_barrier(&prog, ip, bpf_prog))
+                                       return -EINVAL;
+                       }
+
                        pop_callee_regs(&prog, callee_regs_used);
                        EMIT1(0xC9);         /* leave */
                        emit_return(&prog, image + addrs[i - 1] + (prog - temp));