bpf: Skip redundant IBPB in pack allocator

bpf_prog_pack_alloc() issues IBPB on all CPUs on every cBPF allocation,
even when reusing chunks from an existing pack where no new memory was
touched since the last IBPB.

Since IBPB on all CPUs is heavy, Dave Hansen suggested to track allocation
since last IBPB, and only issue IBPB at reuse for the chunks that have not
seen an IBPB since they were last freed.

Track per-pack whether an IBPB is needed via arch_flush_needed. Set it when
allocating a chunk, reset on IBPB flush. On reuse, conditionally issue the
flush. Since IBPB invalidates all BTB entries, clear the flag on all packs
after flushing.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 1614ccc3f111163dbeb58eadf08e7dedfdd165df..50aba113ef9dcc1e8b437ed0ba7bcacd00358146 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -876,6 +876,7 @@ int bpf_jit_add_poke_descriptor(struct bpf_prog *prog,
 struct bpf_prog_pack {
        struct list_head list;
        void *ptr;
+       bool arch_flush_needed;
        unsigned long bitmap[];
 };
 
@@ -928,6 +929,8 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins
        bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE);
        bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE);
 
+       if (static_branch_unlikely(&bpf_pred_flush_enabled))
+               pack->arch_flush_needed = true;
        set_vm_flush_reset_perms(pack->ptr);
        err = set_memory_rox((unsigned long)pack->ptr,
                             BPF_PROG_PACK_SIZE / PAGE_SIZE);
@@ -990,8 +993,15 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool
 
 found_free_area:
        /* Flush only for cBPF as it may contain a crafted gadget */
-       if (static_branch_unlikely(&bpf_pred_flush_enabled) && was_classic)
+       if (static_branch_unlikely(&bpf_pred_flush_enabled) &&
+           pack->arch_flush_needed &&
+           was_classic) {
+               struct bpf_prog_pack *p;
+
                static_call_cond(bpf_arch_pred_flush)();
+               list_for_each_entry(p, &pack_list, list)
+                       p->arch_flush_needed = false;
+       }
        bitmap_set(pack->bitmap, pos, nbits);
        ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT);
 
@@ -1029,6 +1039,9 @@ void bpf_prog_pack_free(void *ptr, u32 size)
                  "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n");
 
        bitmap_clear(pack->bitmap, pos, nbits);
+
+       if (static_branch_unlikely(&bpf_pred_flush_enabled))
+               pack->arch_flush_needed = true;
        if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
                                       BPF_PROG_CHUNK_COUNT, 0) == 0) {
                list_del(&pack->list);