bpf: Restrict JIT predictor flush to cBPF

Currently predictor flush on memory reuse is done for all BPF JIT
allocations, but only cBPF programs can be loaded by an unprivileged user.
eBPF is privileged by default, and flushing predictors for all CPUs on
every eBPF reuse penalizes the common case for no security benefit.

eBPF allocations can be frequent on busy systems, only flush predictors
for cBPF programs. Trampoline and dispatcher allocations also skip the
flush as they are eBPF-only.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index f6bcc0e1a950dff1edc4f90cf8a45fce1e3d7152..b0075ece4a6e7f977f23487cac70402dd2f47f6e 100644 (file)
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -2177,7 +2177,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
        image_size = extable_offset + extable_size;
        ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr,
                                              sizeof(u64), &header, &image_ptr,
-                                             jit_fill_hole);
+                                             jit_fill_hole, was_classic);
        if (!ro_header)
                goto out_off;
 
@@ -2870,7 +2870,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
 
 void *arch_alloc_bpf_trampoline(unsigned int size)
 {
-       return bpf_prog_pack_alloc(size, jit_fill_hole);
+       return bpf_prog_pack_alloc(size, jit_fill_hole, false);
 }
 
 void arch_free_bpf_trampoline(void *image, unsigned int size)

diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index 058ffbbaad856fde67251338a6c1e0eb1047f4ee..3f3f0335d63c85485106f2ed8ec2ec530ee00323 100644 (file)
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -1762,7 +1762,7 @@ static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn,
 
 void *arch_alloc_bpf_trampoline(unsigned int size)
 {
-       return bpf_prog_pack_alloc(size, jit_fill_hole);
+       return bpf_prog_pack_alloc(size, jit_fill_hole, false);
 }
 
 void arch_free_bpf_trampoline(void *image, unsigned int size)
@@ -2228,7 +2228,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
        image_size = prog_size + extable_size;
        /* Now we know the size of the structure to make */
        ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32),
-                                             &header, &image_ptr, jit_fill_hole);
+                                             &header, &image_ptr, jit_fill_hole,
+                                             bpf_prog_was_classic(prog));
        if (!ro_header)
                goto out_offset;
 

diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index d4a17e18c9fba968dcc3e07356c2b055296e2a9f..7b07b43575f111a04949c51a0635ed873115c5a1 100644 (file)
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -295,7 +295,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
        alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
 
        fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
-                                             bpf_jit_fill_ill_insns);
+                                        bpf_jit_fill_ill_insns, bpf_prog_was_classic(fp));
        if (!fhdr)
                goto out_err;
 
@@ -588,7 +588,7 @@ bool bpf_jit_inlines_helper_call(s32 imm)
 
 void *arch_alloc_bpf_trampoline(unsigned int size)
 {
-       return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns);
+       return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns, false);
 }
 
 void arch_free_bpf_trampoline(void *image, unsigned int size)

diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
index c03c1de16b79ace0e612dd3697a11773efa6a08c..f9d5347ba966b7e2c096e024b403ff02c07750e0 100644 (file)
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c
@@ -1321,7 +1321,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
 
 void *arch_alloc_bpf_trampoline(unsigned int size)
 {
-       return bpf_prog_pack_alloc(size, bpf_fill_ill_insns);
+       return bpf_prog_pack_alloc(size, bpf_fill_ill_insns, false);
 }
 
 void arch_free_bpf_trampoline(void *image, unsigned int size)

diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c
index 4365d07aaf547c40d1a254cf46b0c6440109b824..ce3bd3762e08cccceb9814835ee33af1c1a15d0f 100644 (file)
--- a/arch/riscv/net/bpf_jit_core.c
+++ b/arch/riscv/net/bpf_jit_core.c
@@ -109,7 +109,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
                                bpf_jit_binary_pack_alloc(prog_size + extable_size,
                                                          &jit_data->ro_image, sizeof(u32),
                                                          &jit_data->header, &jit_data->image,
-                                                         bpf_fill_ill_insns);
+                                                         bpf_fill_ill_insns,
+                                                         bpf_prog_was_classic(prog));
                        if (!jit_data->ro_header)
                                goto out_offset;
 

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 054e043ffcd22d1507b4b4a571af777368dada88..de7515ea1beae8dd11e999adf72ef0c81104786b 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -3653,7 +3653,7 @@ cleanup:
 
 void *arch_alloc_bpf_trampoline(unsigned int size)
 {
-       return bpf_prog_pack_alloc(size, jit_fill_hole);
+       return bpf_prog_pack_alloc(size, jit_fill_hole, false);
 }
 
 void arch_free_bpf_trampoline(void *image, unsigned int size)
@@ -3965,7 +3965,8 @@ out_image:
                        /* allocate module memory for x86 insns and extable */
                        header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size,
                                                           &image, align, &rw_header, &rw_image,
-                                                          jit_fill_hole);
+                                                          jit_fill_hole,
+                                                          bpf_prog_was_classic(prog));
                        if (!header)
                                goto out_addrs;
                        prog->aux->extable = (void *) image + roundup(proglen, align);

diff --git a/include/linux/filter.h b/include/linux/filter.h
index f68694f94ee7853c86b6b63ef5a25ac3e6401118..14acb2455746fc89c3c403ec65de1fc4092ac366 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1338,7 +1338,7 @@ void bpf_jit_free(struct bpf_prog *fp);
 struct bpf_binary_header *
 bpf_jit_binary_pack_hdr(const struct bpf_prog *fp);
 
-void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns);
+void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic);
 void bpf_prog_pack_free(void *ptr, u32 size);
 
 static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)
@@ -1352,7 +1352,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image,
                          unsigned int alignment,
                          struct bpf_binary_header **rw_hdr,
                          u8 **rw_image,
-                         bpf_jit_fill_hole_t bpf_fill_ill_insns);
+                         bpf_jit_fill_hole_t bpf_fill_ill_insns,
+                         bool was_classic);
 int bpf_jit_binary_pack_finalize(struct bpf_binary_header *ro_header,
                                 struct bpf_binary_header *rw_header);
 void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header,

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 7f0a17f128d4ec4b0a1d028042a0f98d9e8571d5..1614ccc3f111163dbeb58eadf08e7dedfdd165df 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -942,7 +942,7 @@ out:
        return NULL;
 }
 
-void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)
+void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic)
 {
        unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size);
        struct bpf_prog_pack *pack;
@@ -957,7 +957,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)
                 * safe because cBPF programs (the unprivileged attack surface)
                 * are bounded well below a pack size.
                 */
-               if (static_branch_unlikely(&bpf_pred_flush_enabled))
+               if (was_classic && static_branch_unlikely(&bpf_pred_flush_enabled))
                        pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n");
                size = round_up(size, PAGE_SIZE);
                ptr = bpf_jit_alloc_exec(size);
@@ -989,7 +989,9 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)
        pos = 0;
 
 found_free_area:
-       static_call_cond(bpf_arch_pred_flush)();
+       /* Flush only for cBPF as it may contain a crafted gadget */
+       if (static_branch_unlikely(&bpf_pred_flush_enabled) && was_classic)
+               static_call_cond(bpf_arch_pred_flush)();
        bitmap_set(pack->bitmap, pos, nbits);
        ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT);
 
@@ -1149,7 +1151,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr,
                          unsigned int alignment,
                          struct bpf_binary_header **rw_header,
                          u8 **rw_image,
-                         bpf_jit_fill_hole_t bpf_fill_ill_insns)
+                         bpf_jit_fill_hole_t bpf_fill_ill_insns,
+                         bool was_classic)
 {
        struct bpf_binary_header *ro_header;
        u32 size, hole, start;
@@ -1162,7 +1165,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr,
 
        if (bpf_jit_charge_modmem(size))
                return NULL;
-       ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns);
+       ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns, was_classic);
        if (!ro_header) {
                bpf_jit_uncharge_modmem(size);
                return NULL;

diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c
index b77db7413f8c70a20246dfb5fa2a1bc49344c724..ea2d60dc1feeb778cddd2e2c7ec4bfaa7bd2b04d 100644 (file)
--- a/kernel/bpf/dispatcher.c
+++ b/kernel/bpf/dispatcher.c
@@ -145,7 +145,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
 
        mutex_lock(&d->mutex);
        if (!d->image) {
-               d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero);
+               d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero, false);
                if (!d->image)
                        goto out;
                d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE);