bpf: Add bpf_has_frame_pointer()

Introduce a bpf_has_frame_pointer() helper that unwinders can call to
determine whether a given instruction pointer is within the valid frame
pointer region of a BPF JIT program or trampoline (i.e., after the
prologue, before the epilogue).

This will enable livepatch (with the ORC unwinder) to reliably unwind
through BPF JIT frames.

Acked-by: Song Liu <song@kernel.org>
Acked-and-tested-by: Andrey Grodzovsky <andrey.grodzovsky@crowdstrike.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@kernel.org>
Link: https://lore.kernel.org/r/fd2bc5b4e261a680774b28f6100509fd5ebad2f0.1764818927.git.jpoimboe@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Reviewed-by: Jiri Olsa <jolsa@kernel.org>

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index b69dc7194e2c072a80137726dc613f6d874e51dc..b0bac2a66eff3c471d6f4e1af1936ea8a2a2d763 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1678,6 +1678,9 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image
        emit_prologue(&prog, image, stack_depth,
                      bpf_prog_was_classic(bpf_prog), tail_call_reachable,
                      bpf_is_subprog(bpf_prog), bpf_prog->aux->exception_cb);
+
+       bpf_prog->aux->ksym.fp_start = prog - temp;
+
        /* Exception callback will clobber callee regs for its own use, and
         * restore the original callee regs from main prog's stack frame.
         */
@@ -2736,6 +2739,8 @@ emit_jmp:
                                        pop_r12(&prog);
                        }
                        EMIT1(0xC9);         /* leave */
+                       bpf_prog->aux->ksym.fp_end = prog - temp;
+
                        emit_return(&prog, image + addrs[i - 1] + (prog - temp));
                        break;
 
@@ -3325,6 +3330,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
        }
        EMIT1(0x55);             /* push rbp */
        EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
+       if (im)
+               im->ksym.fp_start = prog - (u8 *)rw_image;
+
        if (!is_imm8(stack_size)) {
                /* sub rsp, stack_size */
                EMIT3_off32(0x48, 0x81, 0xEC, stack_size);
@@ -3462,7 +3470,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
                emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8);
 
        emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, -rbx_off);
+
        EMIT1(0xC9); /* leave */
+       if (im)
+               im->ksym.fp_end = prog - (u8 *)rw_image;
+
        if (flags & BPF_TRAMP_F_SKIP_FRAME) {
                /* skip our return address and return to parent */
                EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 6498be4c44f8c27534bad235c035a170d1fb40d4..e5be698256d15a2838914851abb96c5f9b393c83 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1283,6 +1283,8 @@ struct bpf_ksym {
        struct list_head         lnode;
        struct latch_tree_node   tnode;
        bool                     prog;
+       u32                      fp_start;
+       u32                      fp_end;
 };
 
 enum bpf_tramp_prog_type {
@@ -1511,6 +1513,7 @@ void bpf_image_ksym_add(struct bpf_ksym *ksym);
 void bpf_image_ksym_del(struct bpf_ksym *ksym);
 void bpf_ksym_add(struct bpf_ksym *ksym);
 void bpf_ksym_del(struct bpf_ksym *ksym);
+bool bpf_has_frame_pointer(unsigned long ip);
 int bpf_jit_charge_modmem(u32 size);
 void bpf_jit_uncharge_modmem(u32 size);
 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index c8ae6ab316510035f8fa7db05b83e34202e0ffb8..1b9b18e5b03cb0f35b0d3b4c5ed6458c85d70da9 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -760,6 +760,22 @@ struct bpf_prog *bpf_prog_ksym_find(unsigned long addr)
               NULL;
 }
 
+bool bpf_has_frame_pointer(unsigned long ip)
+{
+       struct bpf_ksym *ksym;
+       unsigned long offset;
+
+       guard(rcu)();
+
+       ksym = bpf_ksym_find(ip);
+       if (!ksym || !ksym->fp_start || !ksym->fp_end)
+               return false;
+
+       offset = ip - ksym->start;
+
+       return offset >= ksym->fp_start && offset < ksym->fp_end;
+}
+
 const struct exception_table_entry *search_bpf_extables(unsigned long addr)
 {
        const struct exception_table_entry *e = NULL;