bpf: Add struct bpf_trampoline_ops object

In following changes we will need to override ftrace direct attachment
behaviour. In order to do that we are adding struct bpf_trampoline_ops
object that defines callbacks for ftrace direct attachment:

   register_fentry
   unregister_fentry
   modify_fentry

The new struct bpf_trampoline_ops object is passed as an argument to
__bpf_trampoline_link/unlink_prog functions.

At the moment the default trampoline_ops is set to the current ftrace
direct attachment functions, so there's no functional change for the
current code.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Link: https://lore.kernel.org/r/20260606123955.345967-6-jolsa@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index c0b4732627befc97ebf2bc0d29edbab40daf076a..5c943832fb9d394546eb6ccec5cea1468ede7151 100644 (file)
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -59,8 +59,18 @@ static void trampoline_unlock(struct bpf_trampoline *tr)
        mutex_unlock(select_trampoline_lock(tr));
 }
 
+struct bpf_trampoline_ops {
+       int (*register_fentry)(struct bpf_trampoline *tr, void *new_addr, void *data);
+       int (*unregister_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *old_addr,
+                                void *data);
+       int (*modify_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *old_addr,
+                            void *new_addr, bool lock_direct_mutex, void *data);
+};
+
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
-static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex);
+static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex,
+                                const struct bpf_trampoline_ops *ops, void *data);
+static const struct bpf_trampoline_ops trampoline_ops;
 
 #ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS
 static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip)
@@ -145,13 +155,15 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip,
 
                if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
                    !(tr->flags & BPF_TRAMP_F_ORIG_STACK))
-                       ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
+                       ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */,
+                                                   &trampoline_ops, NULL);
                break;
        case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER:
                tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY;
 
                if (tr->flags & BPF_TRAMP_F_ORIG_STACK)
-                       ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */);
+                       ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */,
+                                                   &trampoline_ops, NULL);
                break;
        default:
                ret = -EINVAL;
@@ -415,7 +427,7 @@ static int bpf_trampoline_update_fentry(struct bpf_trampoline *tr, u32 orig_flag
 }
 
 static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags,
-                            void *old_addr)
+                            void *old_addr, void *data __maybe_unused)
 {
        int ret;
 
@@ -429,7 +441,7 @@ static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags,
 
 static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags,
                         void *old_addr, void *new_addr,
-                        bool lock_direct_mutex)
+                        bool lock_direct_mutex, void *data __maybe_unused)
 {
        int ret;
 
@@ -443,7 +455,7 @@ static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags,
 }
 
 /* first time registering */
-static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
+static int register_fentry(struct bpf_trampoline *tr, void *new_addr, void *data __maybe_unused)
 {
        void *ip = tr->func.addr;
        unsigned long faddr;
@@ -465,6 +477,12 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
        return ret;
 }
 
+static const struct bpf_trampoline_ops trampoline_ops = {
+       .register_fentry   = register_fentry,
+       .unregister_fentry = unregister_fentry,
+       .modify_fentry     = modify_fentry,
+};
+
 static struct bpf_tramp_links *
 bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg)
 {
@@ -632,7 +650,8 @@ out:
        return ERR_PTR(err);
 }
 
-static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex)
+static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex,
+                                const struct bpf_trampoline_ops *ops, void *data)
 {
        struct bpf_tramp_image *im;
        struct bpf_tramp_links *tlinks;
@@ -645,7 +664,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
                return PTR_ERR(tlinks);
 
        if (total == 0) {
-               err = unregister_fentry(tr, orig_flags, tr->cur_image->image);
+               err = ops->unregister_fentry(tr, orig_flags, tr->cur_image->image, data);
                bpf_tramp_image_put(tr->cur_image);
                tr->cur_image = NULL;
                goto out;
@@ -715,11 +734,11 @@ again:
 
        if (tr->cur_image)
                /* progs already running at this address */
-               err = modify_fentry(tr, orig_flags, tr->cur_image->image,
-                                   im->image, lock_direct_mutex);
+               err = ops->modify_fentry(tr, orig_flags, tr->cur_image->image,
+                                        im->image, lock_direct_mutex, data);
        else
                /* first time registering */
-               err = register_fentry(tr, im->image);
+               err = ops->register_fentry(tr, im->image, data);
 
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
        if (err == -EAGAIN) {
@@ -793,7 +812,9 @@ static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog)
 
 static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link,
                                      struct bpf_trampoline *tr,
-                                     struct bpf_prog *tgt_prog)
+                                     struct bpf_prog *tgt_prog,
+                                     const struct bpf_trampoline_ops *ops,
+                                     void *data)
 {
        struct bpf_fsession_link *fslink = NULL;
        enum bpf_tramp_prog_type kind;
@@ -851,7 +872,7 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link,
        } else {
                tr->progs_cnt[kind]++;
        }
-       err = bpf_trampoline_update(tr, true /* lock_direct_mutex */);
+       err = bpf_trampoline_update(tr, true /* lock_direct_mutex */, ops, data);
        if (err) {
                hlist_del_init(&link->tramp_hlist);
                if (kind == BPF_TRAMP_FSESSION) {
@@ -872,14 +893,16 @@ int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
        int err;
 
        trampoline_lock(tr);
-       err = __bpf_trampoline_link_prog(link, tr, tgt_prog);
+       err = __bpf_trampoline_link_prog(link, tr, tgt_prog, &trampoline_ops, NULL);
        trampoline_unlock(tr);
        return err;
 }
 
 static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
                                        struct bpf_trampoline *tr,
-                                       struct bpf_prog *tgt_prog)
+                                       struct bpf_prog *tgt_prog,
+                                       const struct bpf_trampoline_ops *ops,
+                                       void *data)
 {
        enum bpf_tramp_prog_type kind;
        int err;
@@ -904,7 +927,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
        }
        hlist_del_init(&link->tramp_hlist);
        tr->progs_cnt[kind]--;
-       return bpf_trampoline_update(tr, true /* lock_direct_mutex */);
+       return bpf_trampoline_update(tr, true /* lock_direct_mutex */, ops, data);
 }
 
 /* bpf_trampoline_unlink_prog() should never fail. */
@@ -915,7 +938,7 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
        int err;
 
        trampoline_lock(tr);
-       err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog);
+       err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog, &trampoline_ops, NULL);
        trampoline_unlock(tr);
        return err;
 }
@@ -1044,7 +1067,7 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
                goto err;
        }
 
-       err = __bpf_trampoline_link_prog(&shim_link->link, tr, NULL);
+       err = __bpf_trampoline_link_prog(&shim_link->link, tr, NULL, &trampoline_ops, NULL);
        if (err)
                goto err;