bpf: Only enforce 8 frame call stack limit for all-static stacks

The BPF verifier currently enforces a call stack depth of 8 frames,
regardless of the actual stack space consumption of those frames. The
limit is necessary for static call stacks, because the bookkeeping data
structures used by the verifier when stepping into static functions
during verification only support 8 stack frames. However, this
limitation only matters for static stack frames: Global subprogs are
verified by themselves and do not require limiting the call depth.

Relax this limitation to only apply to static stack frames. Verification
now only fails when there is a sequence of 8 calls to non-global
subprogs. Calling into a global subprog resets the counter. This allows
deeper call stacks, provided all frames still fit in the stack.

The change does not increase the maximum size of the call stack, only
the maximum number of frames we can place in it.

Also change the progs/test_global_func3.c selftest to use static
functions, since with the new patch it would otherwise unexpectedly
pass verification.

Acked-by: Mykyta Yatsenko <yatsenko@meta.com>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
Link: https://lore.kernel.org/r/20260316161225.128011-2-emil@etsalapatis.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index e29f15419fcba06c20b48a822a992606ef662787..01c18f4268deb3355f3ab03c41ceacc7e9d712c6 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -6724,22 +6724,30 @@ static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)
        return round_up(max_t(u32, stack_depth, 1), 32);
 }
 
+/* temporary state used for call frame depth calculation */
+struct bpf_subprog_call_depth_info {
+       int ret_insn; /* caller instruction where we return to. */
+       int caller; /* caller subprogram idx */
+       int frame; /* # of consecutive static call stack frames on top of stack */
+};
+
 /* starting from main bpf function walk all instructions of the function
  * and recursively walk all callees that given function can call.
  * Ignore jump and exit insns.
- * Since recursion is prevented by check_cfg() this algorithm
- * only needs a local stack of MAX_CALL_FRAMES to remember callsites
  */
 static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
+                                        struct bpf_subprog_call_depth_info *dinfo,
                                         bool priv_stack_supported)
 {
        struct bpf_subprog_info *subprog = env->subprog_info;
        struct bpf_insn *insn = env->prog->insnsi;
        int depth = 0, frame = 0, i, subprog_end, subprog_depth;
        bool tail_call_reachable = false;
-       int ret_insn[MAX_CALL_FRAMES];
-       int ret_prog[MAX_CALL_FRAMES];
-       int j;
+       int total;
+       int tmp;
+
+       /* no caller idx */
+       dinfo[idx].caller = -1;
 
        i = subprog[idx].start;
        if (!priv_stack_supported)
@@ -6791,8 +6799,12 @@ process_func:
        } else {
                depth += subprog_depth;
                if (depth > MAX_BPF_STACK) {
+                       total = 0;
+                       for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller)
+                               total++;
+
                        verbose(env, "combined stack size of %d calls is %d. Too large\n",
-                               frame + 1, depth);
+                               total, depth);
                        return -EACCES;
                }
        }
@@ -6806,10 +6818,8 @@ continue_func:
 
                        if (!is_bpf_throw_kfunc(insn + i))
                                continue;
-                       if (subprog[idx].is_cb)
-                               err = true;
-                       for (int c = 0; c < frame && !err; c++) {
-                               if (subprog[ret_prog[c]].is_cb) {
+                       for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) {
+                               if (subprog[tmp].is_cb) {
                                        err = true;
                                        break;
                                }
@@ -6825,8 +6835,6 @@ continue_func:
                if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i))
                        continue;
                /* remember insn and function to return to */
-               ret_insn[frame] = i + 1;
-               ret_prog[frame] = idx;
 
                /* find the callee */
                next_insn = i + insn[i].imm + 1;
@@ -6846,7 +6854,16 @@ continue_func:
                                return -EINVAL;
                        }
                }
+
+               /* store caller info for after we return from callee */
+               dinfo[idx].frame = frame;
+               dinfo[idx].ret_insn = i + 1;
+
+               /* push caller idx into callee's dinfo */
+               dinfo[sidx].caller = idx;
+
                i = next_insn;
+
                idx = sidx;
                if (!priv_stack_supported)
                        subprog[idx].priv_stack_mode = NO_PRIV_STACK;
@@ -6854,7 +6871,7 @@ continue_func:
                if (subprog[idx].has_tail_call)
                        tail_call_reachable = true;
 
-               frame++;
+               frame = subprog_is_global(env, idx) ? 0 : frame + 1;
                if (frame >= MAX_CALL_FRAMES) {
                        verbose(env, "the call stack of %d frames is too deep !\n",
                                frame);
@@ -6868,12 +6885,12 @@ continue_func:
         * tail call counter throughout bpf2bpf calls combined with tailcalls
         */
        if (tail_call_reachable)
-               for (j = 0; j < frame; j++) {
-                       if (subprog[ret_prog[j]].is_exception_cb) {
+               for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) {
+                       if (subprog[tmp].is_exception_cb) {
                                verbose(env, "cannot tail call within exception cb\n");
                                return -EINVAL;
                        }
-                       subprog[ret_prog[j]].tail_call_reachable = true;
+                       subprog[tmp].tail_call_reachable = true;
                }
        if (subprog[0].tail_call_reachable)
                env->prog->aux->tail_call_reachable = true;
@@ -6881,23 +6898,33 @@ continue_func:
        /* end of for() loop means the last insn of the 'subprog'
         * was reached. Doesn't matter whether it was JA or EXIT
         */
-       if (frame == 0)
+       if (frame == 0 && dinfo[idx].caller < 0)
                return 0;
        if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE)
                depth -= round_up_stack_depth(env, subprog[idx].stack_depth);
-       frame--;
-       i = ret_insn[frame];
-       idx = ret_prog[frame];
+
+       /* pop caller idx from callee */
+       idx = dinfo[idx].caller;
+
+       /* retrieve caller state from its frame */
+       frame = dinfo[idx].frame;
+       i = dinfo[idx].ret_insn;
+
        goto continue_func;
 }
 
 static int check_max_stack_depth(struct bpf_verifier_env *env)
 {
        enum priv_stack_mode priv_stack_mode = PRIV_STACK_UNKNOWN;
+       struct bpf_subprog_call_depth_info *dinfo;
        struct bpf_subprog_info *si = env->subprog_info;
        bool priv_stack_supported;
        int ret;
 
+       dinfo = kvcalloc(env->subprog_cnt, sizeof(*dinfo), GFP_KERNEL_ACCOUNT);
+       if (!dinfo)
+               return -ENOMEM;
+
        for (int i = 0; i < env->subprog_cnt; i++) {
                if (si[i].has_tail_call) {
                        priv_stack_mode = NO_PRIV_STACK;
@@ -6919,9 +6946,12 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
        for (int i = env->subprog_cnt - 1; i >= 0; i--) {
                if (!i || si[i].is_async_cb) {
                        priv_stack_supported = !i && priv_stack_mode == PRIV_STACK_ADAPTIVE;
-                       ret = check_max_stack_depth_subprog(env, i, priv_stack_supported);
-                       if (ret < 0)
+                       ret = check_max_stack_depth_subprog(env, i, dinfo,
+                                       priv_stack_supported);
+                       if (ret < 0) {
+                               kvfree(dinfo);
                                return ret;
+                       }
                }
        }
 
@@ -6932,6 +6962,8 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
                }
        }
 
+       kvfree(dinfo);
+
        return 0;
 }
 

diff --git a/tools/testing/selftests/bpf/progs/test_global_func3.c b/tools/testing/selftests/bpf/progs/test_global_func3.c
index 142b682d3c2f08df06036c972a006fa35de00a56..974fd8c1956189d26f1b92fdf1a7c07584ebe881 100644 (file)
--- a/tools/testing/selftests/bpf/progs/test_global_func3.c
+++ b/tools/testing/selftests/bpf/progs/test_global_func3.c
@@ -5,56 +5,56 @@
 #include <bpf/bpf_helpers.h>
 #include "bpf_misc.h"
 
-__attribute__ ((noinline))
+static __attribute__ ((noinline))
 int f1(struct __sk_buff *skb)
 {
        return skb->len;
 }
 
-__attribute__ ((noinline))
+static __attribute__ ((noinline))
 int f2(int val, struct __sk_buff *skb)
 {
        return f1(skb) + val;
 }
 
-__attribute__ ((noinline))
+static __attribute__ ((noinline))
 int f3(int val, struct __sk_buff *skb, int var)
 {
        return f2(var, skb) + val;
 }
 
-__attribute__ ((noinline))
+static __attribute__ ((noinline))
 int f4(struct __sk_buff *skb)
 {
        return f3(1, skb, 2);
 }
 
-__attribute__ ((noinline))
+static __attribute__ ((noinline))
 int f5(struct __sk_buff *skb)
 {
        return f4(skb);
 }
 
-__attribute__ ((noinline))
+static __attribute__ ((noinline))
 int f6(struct __sk_buff *skb)
 {
        return f5(skb);
 }
 
-__attribute__ ((noinline))
+static __attribute__ ((noinline))
 int f7(struct __sk_buff *skb)
 {
        return f6(skb);
 }
 
-__attribute__ ((noinline))
+static __attribute__ ((noinline))
 int f8(struct __sk_buff *skb)
 {
        return f7(skb);
 }
 
 SEC("tc")
-__failure __msg("the call stack of 8 frames")
+__failure __msg("the call stack of 9 frames")
 int global_func3(struct __sk_buff *skb)
 {
        return f8(skb);