bpf,x86: Fix exception unwinding with outgoing stack arguments

When a main program with exception_boundary has outgoing stack
arguments (e.g. from calling subprogs with >5 args), bpf_throw() fails
to correctly restore callee-saved registers, causing a kernel crash.

The x86 JIT allocates the outgoing stack arg area below the
callee-saved registers via 'sub rsp, outgoing_rsp' in the prologue.
When bpf_throw() unwinds, it captures the main program's sp (which
includes this outgoing area) and passes it to the exception callback.
The callback gets rsp and rbp, followed by pop_callee_regs, but rsp
points into the outgoing arg area rather than the callee-saved
registers, so the pops restore garbage values. Returning to the
kernel with corrupted callee-saved registers causes a crash.

Fix this by adjusting the sp (adding stack_arg_sp_adjust) passed to
the exception callback, so it points to the bottom of the callee-saved
registers instead of the outgoing arg area. When stack_arg_sp_adjust
is 0 (the common case), this is a no-op.

Fixes: 324c3ca6eed6 ("bpf,x86: Implement JIT support for stack arguments")
Acked-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
Link: https://lore.kernel.org/r/20260517150702.288031-1-yonghong.song@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index ceefefb4da2175dc821eee12a701f95272e1aba9..a0c541a441cfaa2f26d93b36e9c5fb7b76219268 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1789,6 +1789,8 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
         * Arg 6 goes into r9 register, not on stack.
         */
        outgoing_rsp = out_stack_arg_cnt > 1 ? (out_stack_arg_cnt - 1) * 8 : 0;
+       if (bpf_prog->aux->exception_boundary)
+               bpf_prog->aux->stack_arg_sp_adjust = outgoing_rsp;
        emit_sub_rsp(&prog, outgoing_rsp);
 
        if (arena_vm_start)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 242f9597d9ab2e4334f8e3638f959d211e68bd77..1b28cacc30756820e60e101d73018754cf5a854b 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1736,6 +1736,7 @@ struct bpf_prog_aux {
        struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
        char name[BPF_OBJ_NAME_LEN];
        u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);
+       u16 stack_arg_sp_adjust;
 #ifdef CONFIG_SECURITY
        void *security;
 #endif

diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 2cec4e8cd4a0e1c75f46df565d6c189cc7d6fd01..52535671cb9a2ad3e8a7f430b40b6a0c0d5adab4 100644 (file)
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -1265,6 +1265,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
        prog->aux->real_func_cnt = env->subprog_cnt;
        prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;
        prog->aux->exception_boundary = func[0]->aux->exception_boundary;
+       prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust;
        bpf_prog_jit_attempt_done(prog);
        return 0;
 out_free:

diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index baa12b24bb64ac489eafa19347d0088c2ee98674..07de26e7314cf12eac771cda9eb9393014589c7e 100644 (file)
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -3301,7 +3301,7 @@ __bpf_kfunc void bpf_throw(u64 cookie)
         * which skips compiler generated instrumentation to do the same.
         */
        kasan_unpoison_task_stack_below((void *)(long)ctx.sp);
-       ctx.aux->bpf_exception_cb(cookie, ctx.sp, ctx.bp, 0, 0);
+       ctx.aux->bpf_exception_cb(cookie, ctx.sp + ctx.aux->stack_arg_sp_adjust, ctx.bp, 0, 0);
        WARN(1, "A call to BPF exception callback should never return\n");
 }