bpf: Move trampoline image setup into bpf_trampoline_ops callbacks

Moving trampoline image setup into bpf_trampoline_ops callbacks,
so we can have different image handling for multi attachment which
is coming in following changes.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Link: https://lore.kernel.org/r/20260606123955.345967-7-jolsa@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 5c943832fb9d394546eb6ccec5cea1468ede7151..1006031ea0212d654cb7b3cb6021ce03f417c546 100644 (file)
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -60,11 +60,10 @@ static void trampoline_unlock(struct bpf_trampoline *tr)
 }
 
 struct bpf_trampoline_ops {
-       int (*register_fentry)(struct bpf_trampoline *tr, void *new_addr, void *data);
-       int (*unregister_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *old_addr,
-                                void *data);
-       int (*modify_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *old_addr,
-                            void *new_addr, bool lock_direct_mutex, void *data);
+       int (*register_fentry)(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *data);
+       int (*unregister_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *data);
+       int (*modify_fentry)(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im,
+                            bool lock_direct_mutex, void *data);
 };
 
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
@@ -426,9 +425,11 @@ static int bpf_trampoline_update_fentry(struct bpf_trampoline *tr, u32 orig_flag
        return bpf_arch_text_poke(ip, old_t, new_t, old_addr, new_addr);
 }
 
-static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags,
-                            void *old_addr, void *data __maybe_unused)
+static void bpf_tramp_image_put(struct bpf_tramp_image *im);
+
+static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags, void *data __maybe_unused)
 {
+       void *old_addr = tr->cur_image->image;
        int ret;
 
        if (tr->func.ftrace_managed)
@@ -436,13 +437,19 @@ static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags,
        else
                ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, NULL);
 
-       return ret;
+       if (ret)
+               return ret;
+
+       bpf_tramp_image_put(tr->cur_image);
+       tr->cur_image = NULL;
+       return 0;
 }
 
-static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags,
-                        void *old_addr, void *new_addr,
+static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im,
                         bool lock_direct_mutex, void *data __maybe_unused)
 {
+       void *old_addr = tr->cur_image->image;
+       void *new_addr = im->image;
        int ret;
 
        if (tr->func.ftrace_managed) {
@@ -451,12 +458,20 @@ static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags,
                ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr,
                                                   new_addr);
        }
-       return ret;
+
+       if (ret)
+               return ret;
+
+       bpf_tramp_image_put(tr->cur_image);
+       tr->cur_image = im;
+       return 0;
 }
 
 /* first time registering */
-static int register_fentry(struct bpf_trampoline *tr, void *new_addr, void *data __maybe_unused)
+static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im,
+                          void *data __maybe_unused)
 {
+       void *new_addr = im->image;
        void *ip = tr->func.addr;
        unsigned long faddr;
        int ret;
@@ -474,7 +489,11 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr, void *data
                ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr);
        }
 
-       return ret;
+       if (ret)
+               return ret;
+
+       tr->cur_image = im;
+       return 0;
 }
 
 static const struct bpf_trampoline_ops trampoline_ops = {
@@ -664,9 +683,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
                return PTR_ERR(tlinks);
 
        if (total == 0) {
-               err = ops->unregister_fentry(tr, orig_flags, tr->cur_image->image, data);
-               bpf_tramp_image_put(tr->cur_image);
-               tr->cur_image = NULL;
+               err = ops->unregister_fentry(tr, orig_flags, data);
                goto out;
        }
 
@@ -734,11 +751,10 @@ again:
 
        if (tr->cur_image)
                /* progs already running at this address */
-               err = ops->modify_fentry(tr, orig_flags, tr->cur_image->image,
-                                        im->image, lock_direct_mutex, data);
+               err = ops->modify_fentry(tr, orig_flags, im, lock_direct_mutex, data);
        else
                /* first time registering */
-               err = ops->register_fentry(tr, im->image, data);
+               err = ops->register_fentry(tr, im, data);
 
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
        if (err == -EAGAIN) {
@@ -750,22 +766,16 @@ again:
                goto again;
        }
 #endif
-       if (err)
-               goto out_free;
 
-       if (tr->cur_image)
-               bpf_tramp_image_put(tr->cur_image);
-       tr->cur_image = im;
+out_free:
+       if (err)
+               bpf_tramp_image_free(im);
 out:
        /* If any error happens, restore previous flags */
        if (err)
                tr->flags = orig_flags;
        kfree(tlinks);
        return err;
-
-out_free:
-       bpf_tramp_image_free(im);
-       goto out;
 }
 
 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)