bpf: make bpf_insn_successors to return a pointer

The bpf_insn_successors() function is used to return successors
to a BPF instruction. So far, an instruction could have 0, 1 or 2
successors. Prepare the verifier code to introduction of instructions
with more than 2 successors (namely, indirect jumps).

To do this, introduce a new struct, struct bpf_iarray, containing
an array of bpf instruction indexes and make bpf_insn_successors
to return a pointer of that type. The storage for all instructions
is allocated in the env->succ, which holds an array of size 2,
to be used for all instructions.

Signed-off-by: Anton Protopopov <a.s.protopopov@gmail.com>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20251019202145.3944697-10-a.s.protopopov@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index b57222a25a4ac7d950c81c067136f5f310c42876..c6eb68b6389c2e9ceebfc3d392ae4e0ccaad5a29 100644 (file)
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -509,6 +509,15 @@ struct bpf_map_ptr_state {
 #define BPF_ALU_SANITIZE               (BPF_ALU_SANITIZE_SRC | \
                                         BPF_ALU_SANITIZE_DST)
 
+/*
+ * An array of BPF instructions.
+ * Primary usage: return value of bpf_insn_successors.
+ */
+struct bpf_iarray {
+       int cnt;
+       u32 items[];
+};
+
 struct bpf_insn_aux_data {
        union {
                enum bpf_reg_type ptr_type;     /* pointer type for load/store insns */
@@ -828,6 +837,7 @@ struct bpf_verifier_env {
        /* array of pointers to bpf_scc_info indexed by SCC id */
        struct bpf_scc_info **scc_info;
        u32 scc_cnt;
+       struct bpf_iarray *succ;
 };
 
 static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)
@@ -1050,7 +1060,7 @@ void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_st
 
 struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off);
 int bpf_jmp_offset(struct bpf_insn *insn);
-int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2]);
+struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);
 void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);
 bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx);
 

diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index baa742e6cbb65532f937e636920ae234ed711a93..bffb495bc9333287811e736e4268d05e92ecf357 100644 (file)
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -34,7 +34,7 @@
  *   - read and write marks propagation.
  * - The propagation phase is a textbook live variable data flow analysis:
  *
- *     state[cc, i].live_after = U [state[cc, s].live_before for s in insn_successors(i)]
+ *     state[cc, i].live_after = U [state[cc, s].live_before for s in bpf_insn_successors(i)]
  *     state[cc, i].live_before =
  *       (state[cc, i].live_after / state[cc, i].must_write) U state[i].may_read
  *
@@ -54,7 +54,7 @@
  *   The equation for "must_write_acc" propagation looks as follows:
  *
  *     state[cc, i].must_write_acc =
- *       ∩ [state[cc, s].must_write_acc for s in insn_successors(i)]
+ *       ∩ [state[cc, s].must_write_acc for s in bpf_insn_successors(i)]
  *       U state[cc, i].must_write
  *
  *   (An intersection of all "must_write_acc" for instruction successors
@@ -447,7 +447,12 @@ int bpf_jmp_offset(struct bpf_insn *insn)
 __diag_push();
 __diag_ignore_all("-Woverride-init", "Allow field initialization overrides for opcode_info_tbl");
 
-inline int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2])
+/*
+ * Returns an array of instructions succ, with succ->items[0], ...,
+ * succ->items[n-1] with successor instructions, where n=succ->cnt
+ */
+inline struct bpf_iarray *
+bpf_insn_successors(struct bpf_verifier_env *env, u32 idx)
 {
        static const struct opcode_info {
                bool can_jump;
@@ -474,19 +479,25 @@ inline int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2])
                _J(BPF_JSET,  {.can_jump = true,  .can_fallthrough = true}),
        #undef _J
        };
+       struct bpf_prog *prog = env->prog;
        struct bpf_insn *insn = &prog->insnsi[idx];
        const struct opcode_info *opcode_info;
-       int i = 0, insn_sz;
+       struct bpf_iarray *succ;
+       int insn_sz;
+
+       /* pre-allocated array of size up to 2; reset cnt, as it may have been used already */
+       succ = env->succ;
+       succ->cnt = 0;
 
        opcode_info = &opcode_info_tbl[BPF_CLASS(insn->code) | BPF_OP(insn->code)];
        insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
        if (opcode_info->can_fallthrough)
-               succ[i++] = idx + insn_sz;
+               succ->items[succ->cnt++] = idx + insn_sz;
 
        if (opcode_info->can_jump)
-               succ[i++] = idx + bpf_jmp_offset(insn) + 1;
+               succ->items[succ->cnt++] = idx + bpf_jmp_offset(insn) + 1;
 
-       return i;
+       return succ;
 }
 
 __diag_pop();
@@ -548,11 +559,12 @@ static inline bool update_insn(struct bpf_verifier_env *env,
        struct bpf_insn_aux_data *aux = env->insn_aux_data;
        u64 new_before, new_after, must_write_acc;
        struct per_frame_masks *insn, *succ_insn;
-       u32 succ_num, s, succ[2];
+       struct bpf_iarray *succ;
+       u32 s;
        bool changed;
 
-       succ_num = bpf_insn_successors(env->prog, insn_idx, succ);
-       if (unlikely(succ_num == 0))
+       succ = bpf_insn_successors(env, insn_idx);
+       if (succ->cnt == 0)
                return false;
 
        changed = false;
@@ -564,8 +576,8 @@ static inline bool update_insn(struct bpf_verifier_env *env,
         * of successors plus all "must_write" slots of instruction itself.
         */
        must_write_acc = U64_MAX;
-       for (s = 0; s < succ_num; ++s) {
-               succ_insn = get_frame_masks(instance, frame, succ[s]);
+       for (s = 0; s < succ->cnt; ++s) {
+               succ_insn = get_frame_masks(instance, frame, succ->items[s]);
                new_after |= succ_insn->live_before;
                must_write_acc &= succ_insn->must_write_acc;
        }

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 4579082068cacb3c644482e7d91a68db902d470d..6d175849e57acdab17277139e5a12e4af805a7d6 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -17805,6 +17805,22 @@ static int mark_fastcall_patterns(struct bpf_verifier_env *env)
        return 0;
 }
 
+static struct bpf_iarray *iarray_realloc(struct bpf_iarray *old, size_t n_elem)
+{
+       size_t new_size = sizeof(struct bpf_iarray) + n_elem * sizeof(old->items[0]);
+       struct bpf_iarray *new;
+
+       new = kvrealloc(old, new_size, GFP_KERNEL_ACCOUNT);
+       if (!new) {
+               /* this is what callers always want, so simplify the call site */
+               kvfree(old);
+               return NULL;
+       }
+
+       new->cnt = n_elem;
+       return new;
+}
+
 /* Visits the instruction at index t and returns one of the following:
  *  < 0 - an error occurred
  *  DONE_EXPLORING - the instruction was fully explored
@@ -18025,8 +18041,9 @@ err_free:
  */
 static int compute_postorder(struct bpf_verifier_env *env)
 {
-       u32 cur_postorder, i, top, stack_sz, s, succ_cnt, succ[2];
+       u32 cur_postorder, i, top, stack_sz, s;
        int *stack = NULL, *postorder = NULL, *state = NULL;
+       struct bpf_iarray *succ;
 
        postorder = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
        state = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
@@ -18050,11 +18067,11 @@ static int compute_postorder(struct bpf_verifier_env *env)
                                stack_sz--;
                                continue;
                        }
-                       succ_cnt = bpf_insn_successors(env->prog, top, succ);
-                       for (s = 0; s < succ_cnt; ++s) {
-                               if (!state[succ[s]]) {
-                                       stack[stack_sz++] = succ[s];
-                                       state[succ[s]] |= DISCOVERED;
+                       succ = bpf_insn_successors(env, top);
+                       for (s = 0; s < succ->cnt; ++s) {
+                               if (!state[succ->items[s]]) {
+                                       stack[stack_sz++] = succ->items[s];
+                                       state[succ->items[s]] |= DISCOVERED;
                                }
                        }
                        state[top] |= EXPLORED;
@@ -24313,14 +24330,13 @@ static int compute_live_registers(struct bpf_verifier_env *env)
                for (i = 0; i < env->cfg.cur_postorder; ++i) {
                        int insn_idx = env->cfg.insn_postorder[i];
                        struct insn_live_regs *live = &state[insn_idx];
-                       int succ_num;
-                       u32 succ[2];
+                       struct bpf_iarray *succ;
                        u16 new_out = 0;
                        u16 new_in = 0;
 
-                       succ_num = bpf_insn_successors(env->prog, insn_idx, succ);
-                       for (int s = 0; s < succ_num; ++s)
-                               new_out |= state[succ[s]].in;
+                       succ = bpf_insn_successors(env, insn_idx);
+                       for (int s = 0; s < succ->cnt; ++s)
+                               new_out |= state[succ->items[s]].in;
                        new_in = (new_out & ~live->def) | live->use;
                        if (new_out != live->out || new_in != live->in) {
                                live->in = new_in;
@@ -24373,11 +24389,11 @@ static int compute_scc(struct bpf_verifier_env *env)
        const u32 insn_cnt = env->prog->len;
        int stack_sz, dfs_sz, err = 0;
        u32 *stack, *pre, *low, *dfs;
-       u32 succ_cnt, i, j, t, w;
+       u32 i, j, t, w;
        u32 next_preorder_num;
        u32 next_scc_id;
        bool assign_scc;
-       u32 succ[2];
+       struct bpf_iarray *succ;
 
        next_preorder_num = 1;
        next_scc_id = 1;
@@ -24484,12 +24500,12 @@ dfs_continue:
                                stack[stack_sz++] = w;
                        }
                        /* Visit 'w' successors */
-                       succ_cnt = bpf_insn_successors(env->prog, w, succ);
-                       for (j = 0; j < succ_cnt; ++j) {
-                               if (pre[succ[j]]) {
-                                       low[w] = min(low[w], low[succ[j]]);
+                       succ = bpf_insn_successors(env, w);
+                       for (j = 0; j < succ->cnt; ++j) {
+                               if (pre[succ->items[j]]) {
+                                       low[w] = min(low[w], low[succ->items[j]]);
                                } else {
-                                       dfs[dfs_sz++] = succ[j];
+                                       dfs[dfs_sz++] = succ->items[j];
                                        goto dfs_continue;
                                }
                        }
@@ -24506,8 +24522,8 @@ dfs_continue:
                         * or if component has a self reference.
                         */
                        assign_scc = stack[stack_sz - 1] != w;
-                       for (j = 0; j < succ_cnt; ++j) {
-                               if (succ[j] == w) {
+                       for (j = 0; j < succ->cnt; ++j) {
+                               if (succ->items[j] == w) {
                                        assign_scc = true;
                                        break;
                                }
@@ -24569,6 +24585,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
                goto err_free_env;
        for (i = 0; i < len; i++)
                env->insn_aux_data[i].orig_idx = i;
+       env->succ = iarray_realloc(NULL, 2);
+       if (!env->succ)
+               goto err_free_env;
        env->prog = *prog;
        env->ops = bpf_verifier_ops[env->prog->type];
 
@@ -24817,6 +24836,7 @@ err_free_env:
        bpf_stack_liveness_free(env);
        kvfree(env->cfg.insn_postorder);
        kvfree(env->scc_info);
+       kvfree(env->succ);
        kvfree(env);
        return ret;
 }