bpf: Fix s16 truncation for large bpf-to-bpf call offsets

Currently, the BPF instruction set allows bpf-to-bpf calls (or internal
calls, pseudo calls) to use a 32-bit imm field to represent the relative
jump offset.

However, when JIT is disabled or falls back to the interpreter, the
verifier invokes bpf_patch_call_args() to rewrite the call instruction.
In this function, the 32-bit imm is downcast to s16 and stored in the off
field.

    void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
    {
        stack_depth = max_t(u32, stack_depth, 1);
        insn->off = (s16) insn->imm;
        insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] -
            __bpf_call_base_args;
        insn->code = BPF_JMP | BPF_CALL_ARGS;
    }

If the original imm exceeds the s16 range (i.e., a jump offset greater
than 32767 instructions), this downcast silently truncates the offset,
resulting in an incorrect call target.

Fix this by:
1. In bpf_patch_call_args(), keeping the imm field unchanged and using the
   off field to store the index of the interpreter function.
2. In ___bpf_prog_run() for the JMP_CALL_ARGS case, retrieving the
   interpreter function pointer from the interpreters_args array using the
   off field as the index, and passing the original imm to calculate the
   last argument of the interpreter function.

After these changes, the truncation issue is resolved, and __bpf_call_base_args
is also no longer needed and can be removed, which makes the code cleaner.

Performance: In ___bpf_prog_run() for the JMP_CALL_ARGS case, changing the
retrieval of the interpreter function pointer from pointer addition to
direct array indexing improves performance. The possible reason is that the
latter has better instruction-level parallelism. See the v5 discussion [1]
for more details.

[1] https://lore.kernel.org/bpf/f120c3c4-6999-414a-b514-518bb64b4758@zju.edu.cn/

To avoid requiring bpftool changes, keep the new imm/off encoding internal
and restore the legacy xlated dump layout in bpf_insn_prepare_dump().
For bpf-to-bpf call offsets that do not fit in s16, export off as 0 instead
of a truncated and misleading value.

Fixes: 1ea47e01ad6e ("bpf: add support for bpf_call to interpreter")
Fixes: 7105e828c087 ("bpf: allow for correlation of maps and helpers in dump")
Suggested-by: Xu Kuohai <xukuohai@huaweicloud.com>
Suggested-by: Puranjay Mohan <puranjay@kernel.org>
Co-developed-by: Tianci Cao <ziye@zju.edu.cn>
Signed-off-by: Tianci Cao <ziye@zju.edu.cn>
Co-developed-by: Shenghao Yuan <shenghaoyuan0928@163.com>
Signed-off-by: Shenghao Yuan <shenghaoyuan0928@163.com>
Signed-off-by: Yazhou Tang <tangyazhou518@outlook.com>
Link: https://lore.kernel.org/r/20260506094714.419842-3-tangyazhou@zju.edu.cn
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 52b30e9ea431b5acaed6cc50622affd592a69c26..cd191c5fdb0a5ee954939f61577d6c5cbc7828c4 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2918,6 +2918,12 @@ int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 ua
 
 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
 int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
+s32 bpf_call_args_imm(s16 idx);
+#else
+static inline s32 bpf_call_args_imm(s16 idx)
+{
+       return 0;
+}
 #endif
 
 struct btf *bpf_get_btf_vmlinux(void);

diff --git a/include/linux/filter.h b/include/linux/filter.h
index 1ec6d5ba64cc4106eb2f1bc1c40a84a85e79ff52..88a241aac36a277983859c9a5f3dc5015afea7e2 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1151,9 +1151,6 @@ bool sk_filter_charge(struct sock *sk, struct sk_filter *fp);
 void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
 
 u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
-#define __bpf_call_base_args \
-       ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
-        (void *)__bpf_call_base)
 
 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog);
 void bpf_jit_compile(struct bpf_prog *prog);

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 63044ebe572134506d14724048f7ae9feaa73d43..6aa2a8b2403065a285357efaf56cfa33d20110b9 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1771,6 +1771,9 @@ static u32 abs_s32(s32 x)
        return x >= 0 ? (u32)x : -(u32)x;
 }
 
+static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5,
+                                 const struct bpf_insn *insn);
+
 /**
  *     ___bpf_prog_run - run eBPF program on a given context
  *     @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers
@@ -2077,10 +2080,9 @@ select_insn:
                CONT;
 
        JMP_CALL_ARGS:
-               BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2,
-                                                           BPF_R3, BPF_R4,
-                                                           BPF_R5,
-                                                           insn + insn->off + 1);
+               BPF_R0 = interpreters_args[insn->off](BPF_R1, BPF_R2, BPF_R3,
+                                                     BPF_R4, BPF_R5,
+                                                     insn + insn->imm + 1);
                CONT;
 
        JMP_TAIL_CALL: {
@@ -2400,12 +2402,17 @@ int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
        /* Prevent out-of-bounds read to interpreters_args */
        if (stack_depth > MAX_BPF_STACK)
                return -EINVAL;
-       insn->off = (s16) insn->imm;
-       insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] -
-               __bpf_call_base_args;
+       insn->off = (round_up(stack_depth, 32) / 32) - 1;
        insn->code = BPF_JMP | BPF_CALL_ARGS;
        return 0;
 }
+
+s32 bpf_call_args_imm(s16 idx)
+{
+       if (WARN_ON_ONCE(idx < 0 || idx >= ARRAY_SIZE(interpreters_args)))
+               return 0;
+       return BPF_CALL_IMM(interpreters_args[idx]);
+}
 #endif
 #endif
 

diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index df8f48091321f7954689aef03b0dc0b75283dc8a..3692adf625586a06412747ef000e9d0ab9c5bc9e 100644 (file)
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -1250,9 +1250,9 @@ static int jit_subprogs(struct bpf_verifier_env *env)
                }
                if (!bpf_pseudo_call(insn))
                        continue;
-               insn->off = env->insn_aux_data[i].call_imm;
-               subprog = bpf_find_subprog(env, i + insn->off + 1);
-               insn->imm = subprog;
+               insn->imm = env->insn_aux_data[i].call_imm;
+               subprog = bpf_find_subprog(env, i + insn->imm + 1);
+               insn->off = subprog;
        }
 
        prog->jited = 1;

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index a3c0214ca9341067eff2f6fc1897497f9cb3a9ec..630d530782fe8a115432c5594cdd2bd8fd427a05 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -4919,6 +4919,29 @@ out:
        return map;
 }
 
+static void prepare_dump_pseudo_call(struct bpf_insn *insn)
+{
+       s32 call_off = insn->imm;
+
+       /*
+        * BPF_CALL_ARGS only exists for interpreter fallback.
+        * 1. For interpreter (BPF_CALL_ARGS): insn->off is the index of
+        *    interpreters_args array, so here using bpf_call_args_imm()
+        *    to get the real address offset.
+        * 2. For JIT (BPF_CALL): insn->off is the subprog id.
+        */
+       if (insn->code == (BPF_JMP | BPF_CALL_ARGS))
+               insn->imm = bpf_call_args_imm(insn->off);
+       else
+               insn->imm = insn->off;
+
+       /* Avoid dumping a truncated and misleading pc-relative offset. */
+       if (call_off > S16_MAX || call_off < S16_MIN)
+               insn->off = 0;
+       else
+               insn->off = call_off;
+}
+
 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
                                              const struct cred *f_cred)
 {
@@ -4944,6 +4967,9 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
                }
                if (code == (BPF_JMP | BPF_CALL) ||
                    code == (BPF_JMP | BPF_CALL_ARGS)) {
+                       /* Restore the legacy xlated dump layout. */
+                       if (insns[i].src_reg == BPF_PSEUDO_CALL)
+                               prepare_dump_pseudo_call(&insns[i]);
                        if (code == (BPF_JMP | BPF_CALL_ARGS))
                                insns[i].code = BPF_JMP | BPF_CALL;
                        if (!bpf_dump_raw_ok(f_cred))