bpf: properly verify tail call behavior

[ Upstream commit e3245f8990431950d20631c72236d4e8cb2dcde8 ]

A successful ebpf tail call does not return to the caller, but to the
caller-of-the-caller, often just finishing the ebpf program altogether.

Any restrictions that the verifier needs to take into account - notably
the fact that the tail call might have modified packet pointers - are to
be checked on the caller-of-the-caller. Checking it on the caller made
the verifier refuse perfectly fine programs that would use the packet
pointers after a tail call, which is no problem as this code is only
executed if the tail call was unsuccessful, i.e. nothing happened.

This patch simulates the behavior of a tail call in the verifier. A
conditional jump to the code after the tail call is added for the case
of an unsucessful tail call, and a return to the caller is simulated for
a successful tail call.

For the successful case we assume that the tail call returns an int,
as tail calls are currently only allowed in functions that return and
int. We always assume that the tail call modified the packet pointers,
as we do not know what the tail call did.

For the unsuccessful case we know nothing happened, so we do not need to
add new constraints.

This approach also allows to check other problems that may occur with
tail calls, namely we are now able to check that precision is properly
propagated into subprograms using tail calls, as well as checking the
live slots in such a subprogram.

Fixes: 1a4607ffba35 ("bpf: consider that tail calls invalidate packet pointers")
Link: https://lore.kernel.org/bpf/20251029105828.1488347-1-martin.teichmann@xfel.eu/
Signed-off-by: Martin Teichmann <martin.teichmann@xfel.eu>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20251119160355.1160932-2-martin.teichmann@xfel.eu
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 52c01c011c6fb7dc81ceab75f75d8ecea5b9a818..89560e455ce7b2a9d54bdbff49095bcd5c742027 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -4408,6 +4408,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
                                             bt_reg_mask(bt));
                                return -EFAULT;
                        }
+                       if (insn->src_reg == BPF_REG_0 && insn->imm == BPF_FUNC_tail_call
+                           && subseq_idx - idx != 1) {
+                               if (bt_subprog_enter(bt))
+                                       return -EFAULT;
+                       }
                } else if (opcode == BPF_EXIT) {
                        bool r0_precise;
 
@@ -10995,6 +11000,10 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
        bool in_callback_fn;
        int err;
 
+       err = bpf_update_live_stack(env);
+       if (err)
+               return err;
+
        callee = state->frame[state->curframe];
        r0 = &callee->regs[BPF_REG_0];
        if (r0->type == PTR_TO_STACK) {
@@ -11912,6 +11921,25 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
                env->prog->call_get_func_ip = true;
        }
 
+       if (func_id == BPF_FUNC_tail_call) {
+               if (env->cur_state->curframe) {
+                       struct bpf_verifier_state *branch;
+
+                       mark_reg_scratched(env, BPF_REG_0);
+                       branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);
+                       if (IS_ERR(branch))
+                               return PTR_ERR(branch);
+                       clear_all_pkt_pointers(env);
+                       mark_reg_unknown(env, regs, BPF_REG_0);
+                       err = prepare_func_exit(env, &env->insn_idx);
+                       if (err)
+                               return err;
+                       env->insn_idx--;
+               } else {
+                       changes_data = false;
+               }
+       }
+
        if (changes_data)
                clear_all_pkt_pointers(env);
        return 0;
@@ -19810,9 +19838,6 @@ static int process_bpf_exit_full(struct bpf_verifier_env *env,
                return PROCESS_BPF_EXIT;
 
        if (env->cur_state->curframe) {
-               err = bpf_update_live_stack(env);
-               if (err)
-                       return err;
                /* exit from nested function */
                err = prepare_func_exit(env, &env->insn_idx);
                if (err)