Move stack slot index (spi) and frame number out of the flags field
in bpf_jmp_history_entry into dedicated bitfields. This simplifies
the encoding and makes room for new flags.
Previously, spi and frame were packed into the lower 9 bits of the
12-bit flags field (3 bits frame + 6 bits spi), with INSN_F_STACK_ACCESS
at BIT(9) and INSN_F_DST/SRC_REG_STACK at BIT(10)/BIT(11).
But this has no room for an INSN_F_* flag for stack arguments.
To resolve this issue, bpf_jmp_history_entry field idx is narrowed to
20 bits (sufficient for insn indices up to 1M), and the freed bits hold
spi (6 bits) and frame (3 bits) as dedicated struct fields. The flags
enum is simplified accordingly:
INSN_F_STACK_ACCESS -> BIT(0)
INSN_F_DST_REG_STACK -> BIT(1)
INSN_F_SRC_REG_STACK -> BIT(2)
which allows more room for additional INSN_F_* flags.
bpf_push_jmp_history() now takes explicit spi and frame parameters
instead of encoding them into flags. The insn_stack_access_flags(),
insn_stack_access_spi(), and insn_stack_access_frameno() helpers are
removed.
No functional change.
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
Link: https://lore.kernel.org/r/20260513045020.2385962-1-yonghong.song@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
#define MAX_CALL_FRAMES 8
-/* instruction history flags, used in bpf_jmp_history_entry.flags field */
+/* instruction history flags, used in bpf_jmp_history_entry.flags field.
+ * Frame number and SPI are stored in dedicated fields of bpf_jmp_history_entry.
+ */
enum {
- /* instruction references stack slot through PTR_TO_STACK register;
- * we also store stack's frame number in lower 3 bits (MAX_CALL_FRAMES is 8)
- * and accessed stack slot's index in next 6 bits (MAX_BPF_STACK is 512,
- * 8 bytes per slot, so slot index (spi) is [0, 63])
- */
- INSN_F_FRAMENO_MASK = 0x7, /* 3 bits */
-
- INSN_F_SPI_MASK = 0x3f, /* 6 bits */
- INSN_F_SPI_SHIFT = 3, /* shifted 3 bits to the left */
+ INSN_F_STACK_ACCESS = BIT(0),
- INSN_F_STACK_ACCESS = BIT(9),
-
- INSN_F_DST_REG_STACK = BIT(10), /* dst_reg is PTR_TO_STACK */
- INSN_F_SRC_REG_STACK = BIT(11), /* src_reg is PTR_TO_STACK */
- /* total 12 bits are used now. */
+ INSN_F_DST_REG_STACK = BIT(1), /* dst_reg is PTR_TO_STACK */
+ INSN_F_SRC_REG_STACK = BIT(2), /* src_reg is PTR_TO_STACK */
};
-static_assert(INSN_F_FRAMENO_MASK + 1 >= MAX_CALL_FRAMES);
-static_assert(INSN_F_SPI_MASK + 1 >= MAX_BPF_STACK / 8);
-
struct bpf_jmp_history_entry {
- u32 idx;
/* insn idx can't be bigger than 1 million */
+ u32 idx : 20;
+ u32 frame : 3; /* stack access frame number */
+ u32 spi : 6; /* stack slot index (0..63) */
+ u32 : 3;
u32 prev_idx : 20;
/* special INSN_F_xxx flags */
- u32 flags : 12;
+ u32 flags : 4;
+ u32 : 8;
/* additional registers that need precision tracking when this
* jump is backtracked, vector of six 10-bit records
*/
u64 linked_regs;
};
+static_assert(MAX_CALL_FRAMES <= (1 << 3));
+static_assert(MAX_BPF_STACK / 8 <= (1 << 6));
+
/* Maximum number of bpf_reg_state objects that can exist at once */
#define MAX_STACK_ARG_SLOTS (MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS)
#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE + \
void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self);
void bpf_free_backedges(struct bpf_scc_visit *visit);
int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
- int insn_flags, u64 linked_regs);
+ int insn_flags, int spi, int frame, u64 linked_regs);
void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist);
void bpf_mark_reg_not_init(const struct bpf_verifier_env *env,
struct bpf_reg_state *reg);
/* for any branch, call, exit record the history of jmps in the given state */
int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
- int insn_flags, u64 linked_regs)
+ int insn_flags, int spi, int frame, u64 linked_regs)
{
u32 cnt = cur->jmp_history_cnt;
struct bpf_jmp_history_entry *p;
env, "insn history: insn_idx %d cur flags %x new flags %x",
env->insn_idx, env->cur_hist_ent->flags, insn_flags);
env->cur_hist_ent->flags |= insn_flags;
+ env->cur_hist_ent->spi = spi;
+ env->cur_hist_ent->frame = frame;
verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env,
"insn history: insn_idx %d linked_regs: %#llx",
env->insn_idx, env->cur_hist_ent->linked_regs);
p->idx = env->insn_idx;
p->prev_idx = env->prev_insn_idx;
p->flags = insn_flags;
+ p->spi = spi;
+ p->frame = frame;
p->linked_regs = linked_regs;
cur->jmp_history_cnt = cnt;
env->cur_hist_ent = p;
(insn->imm & BPF_FETCH);
}
-static int insn_stack_access_spi(int insn_flags)
-{
- return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK;
-}
-
-static int insn_stack_access_frameno(int insn_flags)
-{
- return insn_flags & INSN_F_FRAMENO_MASK;
-}
-
/* Backtrack one insn at a time. If idx is not at the top of recorded
* history then previous instruction came from straight line execution.
* Return -ENOENT if we exhausted all instructions within given state.
* that [fp - off] slot contains scalar that needs to be
* tracked with precision
*/
- spi = insn_stack_access_spi(hist->flags);
- fr = insn_stack_access_frameno(hist->flags);
+ spi = hist->spi;
+ fr = hist->frame;
bpf_bt_set_frame_slot(bt, fr, spi);
} else if (class == BPF_STX || class == BPF_ST) {
if (bt_is_reg_set(bt, dreg))
/* scalars can only be spilled into stack */
if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
return 0;
- spi = insn_stack_access_spi(hist->flags);
- fr = insn_stack_access_frameno(hist->flags);
+ spi = hist->spi;
+ fr = hist->frame;
if (!bt_is_frame_slot_set(bt, fr, spi))
return 0;
bt_clear_frame_slot(bt, fr, spi);
*/
err = 0;
if (bpf_is_jmp_point(env, env->insn_idx))
- err = bpf_push_jmp_history(env, cur, 0, 0);
+ err = bpf_push_jmp_history(env, cur, 0, 0, 0, 0);
err = err ? : propagate_precision(env, &sl->state, cur, NULL);
if (err)
return err;
return __check_reg_arg(env, state->regs, regno, t);
}
-static int insn_stack_access_flags(int frameno, int spi)
-{
- return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno;
-}
-
static void mark_indirect_target(struct bpf_verifier_env *env, int idx)
{
env->insn_aux_data[idx].indirect_target = true;
int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
struct bpf_reg_state *reg = NULL;
- int insn_flags = insn_stack_access_flags(state->frameno, spi);
+ int insn_flags = INSN_F_STACK_ACCESS;
+ int hist_spi = spi, hist_frame = state->frameno;
/* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0,
* so it's aligned access and [off, off + size) are within stack limits
}
if (insn_flags)
- return bpf_push_jmp_history(env, env->cur_state, insn_flags, 0);
+ return bpf_push_jmp_history(env, env->cur_state, insn_flags,
+ hist_spi, hist_frame, 0);
return 0;
}
int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
struct bpf_reg_state *reg;
u8 *stype, type;
- int insn_flags = insn_stack_access_flags(reg_state->frameno, spi);
+ int insn_flags = INSN_F_STACK_ACCESS;
+ int hist_spi = spi, hist_frame = reg_state->frameno;
stype = reg_state->stack[spi].slot_type;
reg = ®_state->stack[spi].spilled_ptr;
insn_flags = 0; /* we are not restoring spilled register */
}
if (insn_flags)
- return bpf_push_jmp_history(env, env->cur_state, insn_flags, 0);
+ return bpf_push_jmp_history(env, env->cur_state, insn_flags,
+ hist_spi, hist_frame, 0);
return 0;
}
}
if (insn_flags) {
- err = bpf_push_jmp_history(env, this_branch, insn_flags, 0);
+ err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, 0);
if (err)
return err;
}
if (dst_reg->type == SCALAR_VALUE && dst_reg->id)
collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs);
if (linked_regs.cnt > 1) {
- err = bpf_push_jmp_history(env, this_branch, 0, linked_regs_pack(&linked_regs));
+ err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs));
if (err)
return err;
}
}
if (bpf_is_jmp_point(env, env->insn_idx)) {
- err = bpf_push_jmp_history(env, state, 0, 0);
+ err = bpf_push_jmp_history(env, state, 0, 0, 0, 0);
if (err)
return err;
}
projects / thirdparty / kernel / linux.git / commitdiff
