bpf: implement "jmp" mode for trampoline

Implement the "jmp" mode for the bpf trampoline. For the ftrace_managed
case, we need only to set the FTRACE_OPS_FL_JMP on the tr->fops if "jmp"
is needed.

For the bpf poke case, we will check the origin poke type with the
"origin_flags", and current poke type with "tr->flags". The function
bpf_trampoline_update_fentry() is introduced to do the job.

The "jmp" mode will only be enabled with CONFIG_DYNAMIC_FTRACE_WITH_JMP
enabled and BPF_TRAMP_F_SHARE_IPMODIFY is not set. With
BPF_TRAMP_F_SHARE_IPMODIFY, we need to get the origin call ip from the
stack, so we can't use the "jmp" mode.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
Acked-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/r/20251118123639.688444-7-dongml2@chinatelecom.cn
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 0230ad19533e9196cef8d1b2d92087e48f659253..976d89011b15702fc81ce6b8544cd3b1ff3d2e3c 100644 (file)
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -175,24 +175,42 @@ out:
        return tr;
 }
 
-static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)
+static int bpf_trampoline_update_fentry(struct bpf_trampoline *tr, u32 orig_flags,
+                                       void *old_addr, void *new_addr)
 {
+       enum bpf_text_poke_type new_t = BPF_MOD_CALL, old_t = BPF_MOD_CALL;
        void *ip = tr->func.addr;
+
+       if (!new_addr)
+               new_t = BPF_MOD_NOP;
+       else if (bpf_trampoline_use_jmp(tr->flags))
+               new_t = BPF_MOD_JUMP;
+
+       if (!old_addr)
+               old_t = BPF_MOD_NOP;
+       else if (bpf_trampoline_use_jmp(orig_flags))
+               old_t = BPF_MOD_JUMP;
+
+       return bpf_arch_text_poke(ip, old_t, new_t, old_addr, new_addr);
+}
+
+static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags,
+                            void *old_addr)
+{
        int ret;
 
        if (tr->func.ftrace_managed)
                ret = unregister_ftrace_direct(tr->fops, (long)old_addr, false);
        else
-               ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, BPF_MOD_NOP,
-                                        old_addr, NULL);
+               ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, NULL);
 
        return ret;
 }
 
-static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr,
+static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags,
+                        void *old_addr, void *new_addr,
                         bool lock_direct_mutex)
 {
-       void *ip = tr->func.addr;
        int ret;
 
        if (tr->func.ftrace_managed) {
@@ -201,10 +219,8 @@ static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_ad
                else
                        ret = modify_ftrace_direct_nolock(tr->fops, (long)new_addr);
        } else {
-               ret = bpf_arch_text_poke(ip,
-                                        old_addr ? BPF_MOD_CALL : BPF_MOD_NOP,
-                                        new_addr ? BPF_MOD_CALL : BPF_MOD_NOP,
-                                        old_addr, new_addr);
+               ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr,
+                                                  new_addr);
        }
        return ret;
 }
@@ -229,8 +245,7 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
                        return ret;
                ret = register_ftrace_direct(tr->fops, (long)new_addr);
        } else {
-               ret = bpf_arch_text_poke(ip, BPF_MOD_NOP, BPF_MOD_CALL,
-                                        NULL, new_addr);
+               ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr);
        }
 
        return ret;
@@ -416,7 +431,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
                return PTR_ERR(tlinks);
 
        if (total == 0) {
-               err = unregister_fentry(tr, tr->cur_image->image);
+               err = unregister_fentry(tr, orig_flags, tr->cur_image->image);
                bpf_tramp_image_put(tr->cur_image);
                tr->cur_image = NULL;
                goto out;
@@ -440,9 +455,20 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
 
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
 again:
-       if ((tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY) &&
-           (tr->flags & BPF_TRAMP_F_CALL_ORIG))
-               tr->flags |= BPF_TRAMP_F_ORIG_STACK;
+       if (tr->flags & BPF_TRAMP_F_CALL_ORIG) {
+               if (tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY) {
+                       /* The BPF_TRAMP_F_SKIP_FRAME can be cleared in the
+                        * first try, reset it in the second try.
+                        */
+                       tr->flags |= BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SKIP_FRAME;
+               } else if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_JMP)) {
+                       /* Use "jmp" instead of "call" for the trampoline
+                        * in the origin call case, and we don't need to
+                        * skip the frame.
+                        */
+                       tr->flags &= ~BPF_TRAMP_F_SKIP_FRAME;
+               }
+       }
 #endif
 
        size = arch_bpf_trampoline_size(&tr->func.model, tr->flags,
@@ -473,10 +499,18 @@ again:
        if (err)
                goto out_free;
 
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP
+       if (bpf_trampoline_use_jmp(tr->flags))
+               tr->fops->flags |= FTRACE_OPS_FL_JMP;
+       else
+               tr->fops->flags &= ~FTRACE_OPS_FL_JMP;
+#endif
+
        WARN_ON(tr->cur_image && total == 0);
        if (tr->cur_image)
                /* progs already running at this address */
-               err = modify_fentry(tr, tr->cur_image->image, im->image, lock_direct_mutex);
+               err = modify_fentry(tr, orig_flags, tr->cur_image->image,
+                                   im->image, lock_direct_mutex);
        else
                /* first time registering */
                err = register_fentry(tr, im->image);
@@ -499,8 +533,15 @@ again:
        tr->cur_image = im;
 out:
        /* If any error happens, restore previous flags */
-       if (err)
+       if (err) {
                tr->flags = orig_flags;
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP
+               if (bpf_trampoline_use_jmp(tr->flags))
+                       tr->fops->flags |= FTRACE_OPS_FL_JMP;
+               else
+                       tr->fops->flags &= ~FTRACE_OPS_FL_JMP;
+#endif
+       }
        kfree(tlinks);
        return err;