bpf: Add precision marking and backtracking for stack argument slots

Extend the precision marking and backtracking infrastructure to
support stack argument slots (r11-based accesses). Without this,
precision demands for scalar values passed through stack arguments
are silently dropped, which could allow the verifier to incorrectly
prune states with different constant values in stack arg slots.

Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
Link: https://lore.kernel.org/r/20260513045025.2387526-1-yonghong.song@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 3ec3381699814de0167eb57c90d965fb935982ac..6f12fc40b682e6efbae058cc831a4454fbcd9a2f 100644 (file)
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -443,6 +443,8 @@ enum {
 
        INSN_F_DST_REG_STACK = BIT(1), /* dst_reg is PTR_TO_STACK */
        INSN_F_SRC_REG_STACK = BIT(2), /* src_reg is PTR_TO_STACK */
+
+       INSN_F_STACK_ARG_ACCESS = BIT(3),
 };
 
 struct bpf_jmp_history_entry {
@@ -858,6 +860,7 @@ struct backtrack_state {
        u32 frame;
        u32 reg_masks[MAX_CALL_FRAMES];
        u64 stack_masks[MAX_CALL_FRAMES];
+       u8 stack_arg_masks[MAX_CALL_FRAMES];
 };
 
 struct bpf_id_pair {
@@ -1256,6 +1259,11 @@ static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame,
        bt->stack_masks[frame] |= 1ull << slot;
 }
 
+static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot)
+{
+       bt->stack_arg_masks[frame] |= 1 << slot;
+}
+
 static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u32 reg)
 {
        return bt->reg_masks[frame] & (1 << reg);

diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c
index 5e93e57fb7ae8be81a50f76e758a8adf082e21ab..2e4ae0ef08609dac2ab515fc3cab9f7c62913739 100644 (file)
--- a/kernel/bpf/backtrack.c
+++ b/kernel/bpf/backtrack.c
@@ -129,11 +129,21 @@ static inline u32 bt_empty(struct backtrack_state *bt)
        int i;
 
        for (i = 0; i <= bt->frame; i++)
-               mask |= bt->reg_masks[i] | bt->stack_masks[i];
+               mask |= bt->reg_masks[i] | bt->stack_masks[i] | bt->stack_arg_masks[i];
 
        return mask == 0;
 }
 
+static inline void bt_clear_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot)
+{
+       bt->stack_arg_masks[frame] &= ~(1 << slot);
+}
+
+static inline bool bt_is_frame_stack_arg_slot_set(struct backtrack_state *bt, u32 frame, u32 slot)
+{
+       return bt->stack_arg_masks[frame] & (1 << slot);
+}
+
 static inline int bt_subprog_enter(struct backtrack_state *bt)
 {
        if (bt->frame == MAX_CALL_FRAMES - 1) {
@@ -194,6 +204,11 @@ static inline u64 bt_stack_mask(struct backtrack_state *bt)
        return bt->stack_masks[bt->frame];
 }
 
+static inline u8 bt_stack_arg_mask(struct backtrack_state *bt)
+{
+       return bt->stack_arg_masks[bt->frame];
+}
+
 static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg)
 {
        return bt->reg_masks[bt->frame] & (1 << reg);
@@ -335,6 +350,19 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
                        return 0;
                bt_clear_reg(bt, load_reg);
 
+               if (hist && hist->flags & INSN_F_STACK_ARG_ACCESS) {
+                       spi = hist->spi;
+                       /*
+                        * Stack arg read: callee reads from r11+off, but
+                        * the data lives in the caller's stack_arg_regs.
+                        * Set the mask in the caller frame so precision
+                        * is marked in the caller's slot at the callee
+                        * entry checkpoint.
+                        */
+                       bt_set_frame_stack_arg_slot(bt, bt->frame - 1, spi);
+                       return 0;
+               }
+
                /* scalars can only be spilled into stack w/o losing precision.
                 * Load from any other memory can be zero extended.
                 * The desire to keep that precision is already indicated
@@ -357,6 +385,17 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
                         * encountered a case of pointer subtraction.
                         */
                        return -ENOTSUPP;
+
+               if (hist && hist->flags & INSN_F_STACK_ARG_ACCESS) {
+                       spi = hist->spi;
+                       if (!bt_is_frame_stack_arg_slot_set(bt, bt->frame, spi))
+                               return 0;
+                       bt_clear_frame_stack_arg_slot(bt, bt->frame, spi);
+                       if (class == BPF_STX)
+                               bt_set_reg(bt, sreg);
+                       return 0;
+               }
+
                /* scalars can only be spilled into stack */
                if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
                        return 0;
@@ -425,6 +464,12 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
                                                bpf_bt_set_frame_reg(bt, bt->frame - 1, i);
                                        }
                                }
+                               if (bt_stack_arg_mask(bt)) {
+                                       verifier_bug(env,
+                                                    "static subprog leftover stack arg slots %x",
+                                                    bt_stack_arg_mask(bt));
+                                       return -EFAULT;
+                               }
                                if (bt_subprog_exit(bt))
                                        return -EFAULT;
                                return 0;
@@ -895,6 +940,17 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env,
                                        *changed = true;
                                }
                        }
+                       for (i = 0; i < func->out_stack_arg_cnt; i++) {
+                               if (!bt_is_frame_stack_arg_slot_set(bt, fr, i))
+                                       continue;
+                               reg = &func->stack_arg_regs[i];
+                               if (reg->type != SCALAR_VALUE || reg->precise) {
+                                       bt_clear_frame_stack_arg_slot(bt, fr, i);
+                               } else {
+                                       reg->precise = true;
+                                       *changed = true;
+                               }
+                       }
                        if (env->log.level & BPF_LOG_LEVEL2) {
                                fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
                                             bt_frame_reg_mask(bt, fr));

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index d15aef2fe4a1d720fab96164f28003f5a20d60aa..ebd13661933eaeda3ee5453dfca1cd8a92fa4041 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -292,6 +292,11 @@ static int arg_from_argno(argno_t a)
        return -1;
 }
 
+static int arg_idx_from_argno(argno_t a)
+{
+       return arg_from_argno(a) - 1;
+}
+
 static const char *btf_type_name(const struct btf *btf, u32 id)
 {
        return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
@@ -4115,7 +4120,8 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s
                __mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm);
        }
        state->no_stack_arg_load = true;
-       return 0;
+       return bpf_push_jmp_history(env, env->cur_state,
+                                   INSN_F_STACK_ARG_ACCESS, spi, 0, 0);
 }
 
 /*
@@ -4146,7 +4152,17 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st
        arg = &caller->stack_arg_regs[spi];
        cur = vstate->frame[vstate->curframe];
        cur->regs[dst_regno] = *arg;
-       return 0;
+       return bpf_push_jmp_history(env, env->cur_state,
+                                   INSN_F_STACK_ARG_ACCESS, spi, 0, 0);
+}
+
+static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx)
+{
+       struct bpf_func_state *caller = cur_func(env);
+       int spi = arg_idx - MAX_BPF_FUNC_REG_ARGS;
+
+       bt_set_frame_stack_arg_slot(&env->bt, caller->frameno, spi);
+       return mark_chain_precision_batch(env, env->cur_state);
 }
 
 static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller,
@@ -6875,8 +6891,14 @@ static int check_mem_size_reg(struct bpf_verifier_env *env,
        }
        err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg),
                                      access_type, zero_size_allowed, meta);
-       if (!err)
-               err = mark_chain_precision(env, reg_from_argno(size_argno));
+       if (!err) {
+               int regno = reg_from_argno(size_argno);
+
+               if (regno >= 0)
+                       err = mark_chain_precision(env, regno);
+               else
+                       err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno));
+       }
        return err;
 }
 
@@ -7325,7 +7347,7 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *
                            struct bpf_kfunc_call_arg_meta *meta)
 {
        const struct btf_type *t;
-       u32 arg_idx = arg_from_argno(argno) - 1;
+       u32 arg_idx = arg_idx_from_argno(argno);
        int spi, err, i, nr_slots, btf_id;
 
        if (reg->type != PTR_TO_STACK) {