bpf: Raise maximum call chain depth to 16 frames

Bump MAX_CALL_FRAMES from 8 to 16 to allow deeper call chains
that Rust-BPF requires and update selftests.

Link: https://lore.kernel.org/r/20260613180755.29671-1-alexei.starovoitov@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index d57b339a8cb8bb948be2f44ff18d90caa9ea118c..39a851e690ec4165d634bef75431aa3109aae29c 100644 (file)
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -404,7 +404,7 @@ struct bpf_func_state {
        struct bpf_reg_state *stack_arg_regs; /* Outgoing on-stack arguments */
 };
 
-#define MAX_CALL_FRAMES 8
+#define MAX_CALL_FRAMES 16
 
 /* instruction history flags, used in bpf_jmp_history_entry.flags field.
  * Frame number and SPI are stored in dedicated fields of bpf_jmp_history_entry.
@@ -421,20 +421,21 @@ enum {
 struct bpf_jmp_history_entry {
        /* insn idx can't be bigger than 1 million */
        u32 idx : 20;
-       u32 frame : 3;  /* stack access frame number */
+       u32 frame : 4;  /* stack access frame number */
        u32 spi : 6;    /* stack slot index (0..63) */
-       u32 : 3;
+       u32 : 2;
        u32 prev_idx : 20;
        /* special INSN_F_xxx flags */
        u32 flags : 4;
        u32 : 8;
-       /* additional registers that need precision tracking when this
-        * jump is backtracked, vector of six 10-bit records
+       /*
+        * additional registers that need precision tracking when this
+        * jump is backtracked, vector of five 11-bit records
         */
        u64 linked_regs;
 };
 
-static_assert(MAX_CALL_FRAMES <= (1 << 3));
+static_assert(MAX_CALL_FRAMES <= (1 << 4));
 static_assert(MAX_BPF_STACK / 8 <= (1 << 6));
 
 /* Maximum number of bpf_reg_state objects that can exist at once */

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index eb46a81a8c51ac58e4c9bc432fd32bdc30672862..2abc79dbf281c21b477ceb09dbb6f14c9e187c6b 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -3144,7 +3144,7 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx)
        env->insn_aux_data[idx].indirect_target = true;
 }
 
-#define LR_FRAMENO_BITS        3
+#define LR_FRAMENO_BITS        4
 #define LR_SPI_BITS    6
 #define LR_ENTRY_BITS  (LR_SPI_BITS + LR_FRAMENO_BITS + 1)
 #define LR_SIZE_BITS   4
@@ -3153,7 +3153,11 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx)
 #define LR_SIZE_MASK   ((1ull << LR_SIZE_BITS)    - 1)
 #define LR_SPI_OFF     LR_FRAMENO_BITS
 #define LR_IS_REG_OFF  (LR_SPI_BITS + LR_FRAMENO_BITS)
-#define LINKED_REGS_MAX        6
+#define LINKED_REGS_MAX        5
+
+static_assert(MAX_CALL_FRAMES <= (1 << LR_FRAMENO_BITS));
+static_assert(LINKED_REGS_MAX < (1 << LR_SIZE_BITS));
+static_assert(LINKED_REGS_MAX * LR_ENTRY_BITS + LR_SIZE_BITS <= 64);
 
 struct linked_reg {
        u8 frameno;
@@ -3177,10 +3181,11 @@ static struct linked_reg *linked_regs_push(struct linked_regs *s)
        return NULL;
 }
 
-/* Use u64 as a vector of 6 10-bit values, use first 4-bits to track
+/*
+ * Use u64 as a vector of 5 11-bit values, use first 4-bits to track
  * number of elements currently in stack.
- * Pack one history entry for linked registers as 10 bits in the following format:
- * - 3-bits frameno
+ * Pack one history entry for linked registers as 11 bits in the following format:
+ * - 4-bits frameno
  * - 6-bits spi_or_reg
  * - 1-bit  is_reg
  */

diff --git a/tools/testing/selftests/bpf/progs/test_global_func3.c b/tools/testing/selftests/bpf/progs/test_global_func3.c
index 974fd8c1956189d26f1b92fdf1a7c07584ebe881..b66abb350fb0e11257adc6543dbed52e733c98a6 100644 (file)
--- a/tools/testing/selftests/bpf/progs/test_global_func3.c
+++ b/tools/testing/selftests/bpf/progs/test_global_func3.c
@@ -53,9 +53,57 @@ int f8(struct __sk_buff *skb)
        return f7(skb);
 }
 
+static __attribute__ ((noinline))
+int f9(struct __sk_buff *skb)
+{
+       return f8(skb);
+}
+
+static __attribute__ ((noinline))
+int f10(struct __sk_buff *skb)
+{
+       return f9(skb);
+}
+
+static __attribute__ ((noinline))
+int f11(struct __sk_buff *skb)
+{
+       return f10(skb);
+}
+
+static __attribute__ ((noinline))
+int f12(struct __sk_buff *skb)
+{
+       return f11(skb);
+}
+
+static __attribute__ ((noinline))
+int f13(struct __sk_buff *skb)
+{
+       return f12(skb);
+}
+
+static __attribute__ ((noinline))
+int f14(struct __sk_buff *skb)
+{
+       return f13(skb);
+}
+
+static __attribute__ ((noinline))
+int f15(struct __sk_buff *skb)
+{
+       return f14(skb);
+}
+
+static __attribute__ ((noinline))
+int f16(struct __sk_buff *skb)
+{
+       return f15(skb);
+}
+
 SEC("tc")
-__failure __msg("the call stack of 9 frames")
+__failure __msg("the call stack of 17 frames")
 int global_func3(struct __sk_buff *skb)
 {
-       return f8(skb);
+       return f16(skb);
 }

diff --git a/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c b/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c
index b058de623200a2809419ae878ad452babe626e34..72646fa2745ea937a78cd4b50ac8fdd63c369ca6 100644 (file)
--- a/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c
+++ b/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c
@@ -15,7 +15,7 @@
  * FP offset at each call site.  arg_track keys on (frame, off[]), so
  * r1=fp-8, r1=fp-16, ... r1=fp-400 produce 50 unique cache keys per level.
  *
- * This test chains 8 subprograms (the MAX_CALL_FRAMES limit).  Each
+ * This test chains 8 subprograms (within the MAX_CALL_FRAMES limit).  Each
  * intermediate function calls the next one 50 times, each time with a
  * different FP-relative offset in r1.
  *

diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
index 70ae14d6084fb082c7421d402b6214132ff114cf..e38f102da45f2ee05bd7b9f2af943bd483b7c208 100644 (file)
--- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
+++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
@@ -372,37 +372,36 @@ __naked void precision_two_ids(void)
 SEC("socket")
 __success __log_level(2)
 __flag(BPF_F_TEST_STATE_FREQ)
-/* check that r0 and r6 have different IDs after 'if',
- * collect_linked_regs() can't tie more than 6 registers for a single insn.
+/*
+ * check that r0 and r5 have different IDs after 'if',
+ * collect_linked_regs() can't tie more than 5 registers for a single insn.
  */
-__msg("8: (25) if r0 > 0x7 goto pc+0         ; R0=scalar(id=1")
-__msg("14: (bf) r6 = r6                      ; R6=scalar(id=2")
-/* check that r{0-5} are marked precise after 'if' */
-__msg("frame0: regs=r0 stack= before 8: (25) if r0 > 0x7 goto pc+0")
-__msg("frame0: parent state regs=r0,r1,r2,r3,r4,r5 stack=:")
+__msg("7: (25) if r0 > 0x7 goto pc+0         ; R0=scalar(id=1")
+__msg("12: (bf) r5 = r5                      ; R5=scalar(id=2")
+/* check that r{0-4} are marked precise after 'if' */
+__msg("frame0: regs=r0 stack= before 7: (25) if r0 > 0x7 goto pc+0")
+__msg("frame0: parent state regs=r0,r1,r2,r3,r4 stack=:")
 __naked void linked_regs_too_many_regs(void)
 {
        asm volatile (
        /* r0 = random number up to 0xff */
        "call %[bpf_ktime_get_ns];"
        "r0 &= 0xff;"
-       /* tie r{0-6} IDs */
+       /* tie r{0-5} IDs */
        "r1 = r0;"
        "r2 = r0;"
        "r3 = r0;"
        "r4 = r0;"
        "r5 = r0;"
-       "r6 = r0;"
-       /* propagate range for r{0-6} */
+       /* propagate range for r{0-5} */
        "if r0 > 7 goto +0;"
-       /* keep r{1-5} live */
+       /* keep r{1-4} live */
        "r1 = r1;"
        "r2 = r2;"
        "r3 = r3;"
        "r4 = r4;"
+       /* make r5 appear in the log */
        "r5 = r5;"
-       /* make r6 appear in the log */
-       "r6 = r6;"
        /* force r0 to be precise,
         * this would cause r{0-4} to be precise because of shared IDs
         */

diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c
index 42d523a21a435064282d15d026d29399eb5169cc..302d712e0d7ed2ad5603cde6974d7b4064d8dc28 100644 (file)
--- a/tools/testing/selftests/bpf/verifier/calls.c
+++ b/tools/testing/selftests/bpf/verifier/calls.c
@@ -1219,6 +1219,30 @@
        BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call H */
        BPF_EXIT_INSN(),
        /* H */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call I */
+       BPF_EXIT_INSN(),
+       /* I */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call J */
+       BPF_EXIT_INSN(),
+       /* J */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call K */
+       BPF_EXIT_INSN(),
+       /* K */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call L */
+       BPF_EXIT_INSN(),
+       /* L */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call M */
+       BPF_EXIT_INSN(),
+       /* M */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call N */
+       BPF_EXIT_INSN(),
+       /* N */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call O */
+       BPF_EXIT_INSN(),
+       /* O */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call P */
+       BPF_EXIT_INSN(),
+       /* P */
        BPF_MOV64_IMM(BPF_REG_0, 0),
        BPF_EXIT_INSN(),
        },
@@ -1257,6 +1281,30 @@
        BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call H */
        BPF_EXIT_INSN(),
        /* H */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call I */
+       BPF_EXIT_INSN(),
+       /* I */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call J */
+       BPF_EXIT_INSN(),
+       /* J */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call K */
+       BPF_EXIT_INSN(),
+       /* K */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call L */
+       BPF_EXIT_INSN(),
+       /* L */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call M */
+       BPF_EXIT_INSN(),
+       /* M */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call N */
+       BPF_EXIT_INSN(),
+       /* N */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call O */
+       BPF_EXIT_INSN(),
+       /* O */
+       BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call P */
+       BPF_EXIT_INSN(),
+       /* P */
        BPF_MOV64_IMM(BPF_REG_0, 0),
        BPF_EXIT_INSN(),
        },