bpf,x86: adjust the "jmp" mode for bpf trampoline

In the origin call case, if BPF_TRAMP_F_SKIP_FRAME is not set, it means
that the trampoline is not called, but "jmp".

Introduce the function bpf_trampoline_use_jmp() to check if the trampoline
is in "jmp" mode.

Do some adjustment on the "jmp" mode for the x86_64. The main adjustment
that we make is for the stack parameter passing case, as the stack
alignment logic changes in the "jmp" mode without the "rip". What's more,
the location of the parameters on the stack also changes.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
Link: https://lore.kernel.org/r/20251118123639.688444-5-dongml2@chinatelecom.cn
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 808d4343f6cfafc5bb656ac1021a8610a989d00f..632a83381c2d43afecbabe964d9ee65e719d34a8 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -2847,9 +2847,10 @@ static int get_nr_used_regs(const struct btf_func_model *m)
 }
 
 static void save_args(const struct btf_func_model *m, u8 **prog,
-                     int stack_size, bool for_call_origin)
+                     int stack_size, bool for_call_origin, u32 flags)
 {
        int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0;
+       bool use_jmp = bpf_trampoline_use_jmp(flags);
        int i, j;
 
        /* Store function arguments to stack.
@@ -2890,7 +2891,7 @@ static void save_args(const struct btf_func_model *m, u8 **prog,
                         */
                        for (j = 0; j < arg_regs; j++) {
                                emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP,
-                                        nr_stack_slots * 8 + 0x18);
+                                        nr_stack_slots * 8 + 16 + (!use_jmp) * 8);
                                emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0,
                                         -stack_size);
 
@@ -3284,7 +3285,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
                 * should be 16-byte aligned. Following code depend on
                 * that stack_size is already 8-byte aligned.
                 */
-               stack_size += (stack_size % 16) ? 0 : 8;
+               if (bpf_trampoline_use_jmp(flags)) {
+                       /* no rip in the "jmp" case */
+                       stack_size += (stack_size % 16) ? 8 : 0;
+               } else {
+                       stack_size += (stack_size % 16) ? 0 : 8;
+               }
        }
 
        arg_stack_off = stack_size;
@@ -3344,7 +3350,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
                emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -ip_off);
        }
 
-       save_args(m, &prog, regs_off, false);
+       save_args(m, &prog, regs_off, false, flags);
 
        if (flags & BPF_TRAMP_F_CALL_ORIG) {
                /* arg1: mov rdi, im */
@@ -3377,7 +3383,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 
        if (flags & BPF_TRAMP_F_CALL_ORIG) {
                restore_regs(m, &prog, regs_off);
-               save_args(m, &prog, arg_stack_off, true);
+               save_args(m, &prog, arg_stack_off, true, flags);
 
                if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
                        /* Before calling the original function, load the

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 30fb4042140573844a2dd02d6591e68c512e614a..2f79afe81482fe1112608003cc394f90677e0042 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1264,6 +1264,18 @@ typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start,
 bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog);
 bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog);
 
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP
+static inline bool bpf_trampoline_use_jmp(u64 flags)
+{
+       return flags & BPF_TRAMP_F_CALL_ORIG && !(flags & BPF_TRAMP_F_SKIP_FRAME);
+}
+#else
+static inline bool bpf_trampoline_use_jmp(u64 flags)
+{
+       return false;
+}
+#endif
+
 struct bpf_ksym {
        unsigned long            start;
        unsigned long            end;