bpf,x86: Use single ftrace_ops for direct calls

Using single ftrace_ops for direct calls update instead of allocating
ftrace_ops object for each trampoline.

With single ftrace_ops object we can use update_ftrace_direct_* api
that allows multiple ip sites updates on single ftrace_ops object.

Adding HAVE_SINGLE_FTRACE_DIRECT_OPS config option to be enabled on
each arch that supports this.

At the moment we can enable this only on x86 arch, because arm relies
on ftrace_ops object representing just single trampoline image (stored
in ftrace_ops::direct_call). Archs that do not support this will continue
to use *_ftrace_direct api.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Reviewed-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/bpf/20251230145010.103439-10-jolsa@kernel.org

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 80527299f859aaab7e357f59a410ac436daafd1f..53bf2cf7ff6ff9aefad11f6c4856f1753ede2588 100644 (file)
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -336,6 +336,7 @@ config X86
        select SCHED_SMT                        if SMP
        select ARCH_SUPPORTS_SCHED_CLUSTER      if SMP
        select ARCH_SUPPORTS_SCHED_MC           if SMP
+       select HAVE_SINGLE_FTRACE_DIRECT_OPS    if X86_64 && DYNAMIC_FTRACE_WITH_DIRECT_CALLS
 
 config INSTRUCTION_DECODER
        def_bool y

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 64556aa0007b09ef47c777a9220c38ee049a70eb..952cd793246188d5b4381c056a4e49d6ba580caf 100644 (file)
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -33,12 +33,40 @@ static DEFINE_MUTEX(trampoline_mutex);
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
 static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex);
 
+#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS
+static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip)
+{
+       struct hlist_head *head_ip;
+       struct bpf_trampoline *tr;
+
+       mutex_lock(&trampoline_mutex);
+       head_ip = &trampoline_ip_table[hash_64(ip, TRAMPOLINE_HASH_BITS)];
+       hlist_for_each_entry(tr, head_ip, hlist_ip) {
+               if (tr->ip == ip)
+                       goto out;
+       }
+       tr = NULL;
+out:
+       mutex_unlock(&trampoline_mutex);
+       return tr;
+}
+#else
+static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip)
+{
+       return ops->private;
+}
+#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */
+
 static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip,
                                     enum ftrace_ops_cmd cmd)
 {
-       struct bpf_trampoline *tr = ops->private;
+       struct bpf_trampoline *tr;
        int ret = 0;
 
+       tr = direct_ops_ip_lookup(ops, ip);
+       if (!tr)
+               return -EINVAL;
+
        if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) {
                /* This is called inside register_ftrace_direct_multi(), so
                 * tr->mutex is already locked.
@@ -144,6 +172,162 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym)
                           PAGE_SIZE, true, ksym->name);
 }
 
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS
+/*
+ * We have only single direct_ops which contains all the direct call
+ * sites and is the only global ftrace_ops for all trampolines.
+ *
+ * We use 'update_ftrace_direct_*' api for attachment.
+ */
+struct ftrace_ops direct_ops = {
+       .ops_func = bpf_tramp_ftrace_ops_func,
+};
+
+static int direct_ops_alloc(struct bpf_trampoline *tr)
+{
+       tr->fops = &direct_ops;
+       return 0;
+}
+
+static void direct_ops_free(struct bpf_trampoline *tr) { }
+
+static struct ftrace_hash *hash_from_ip(struct bpf_trampoline *tr, void *ptr)
+{
+       unsigned long ip, addr = (unsigned long) ptr;
+       struct ftrace_hash *hash;
+
+       ip = ftrace_location(tr->ip);
+       if (!ip)
+               return NULL;
+       hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
+       if (!hash)
+               return NULL;
+       if (bpf_trampoline_use_jmp(tr->flags))
+               addr = ftrace_jmp_set(addr);
+       if (!add_ftrace_hash_entry_direct(hash, ip, addr)) {
+               free_ftrace_hash(hash);
+               return NULL;
+       }
+       return hash;
+}
+
+static int direct_ops_add(struct bpf_trampoline *tr, void *addr)
+{
+       struct ftrace_hash *hash = hash_from_ip(tr, addr);
+       int err;
+
+       if (!hash)
+               return -ENOMEM;
+       err = update_ftrace_direct_add(tr->fops, hash);
+       free_ftrace_hash(hash);
+       return err;
+}
+
+static int direct_ops_del(struct bpf_trampoline *tr, void *addr)
+{
+       struct ftrace_hash *hash = hash_from_ip(tr, addr);
+       int err;
+
+       if (!hash)
+               return -ENOMEM;
+       err = update_ftrace_direct_del(tr->fops, hash);
+       free_ftrace_hash(hash);
+       return err;
+}
+
+static int direct_ops_mod(struct bpf_trampoline *tr, void *addr, bool lock_direct_mutex)
+{
+       struct ftrace_hash *hash = hash_from_ip(tr, addr);
+       int err;
+
+       if (!hash)
+               return -ENOMEM;
+       err = update_ftrace_direct_mod(tr->fops, hash, lock_direct_mutex);
+       free_ftrace_hash(hash);
+       return err;
+}
+#else
+/*
+ * We allocate ftrace_ops object for each trampoline and it contains
+ * call site specific for that trampoline.
+ *
+ * We use *_ftrace_direct api for attachment.
+ */
+static int direct_ops_alloc(struct bpf_trampoline *tr)
+{
+       tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);
+       if (!tr->fops)
+               return -ENOMEM;
+       tr->fops->private = tr;
+       tr->fops->ops_func = bpf_tramp_ftrace_ops_func;
+       return 0;
+}
+
+static void direct_ops_free(struct bpf_trampoline *tr)
+{
+       if (!tr->fops)
+               return;
+       ftrace_free_filter(tr->fops);
+       kfree(tr->fops);
+}
+
+static int direct_ops_add(struct bpf_trampoline *tr, void *ptr)
+{
+       unsigned long addr = (unsigned long) ptr;
+       struct ftrace_ops *ops = tr->fops;
+       int ret;
+
+       if (bpf_trampoline_use_jmp(tr->flags))
+               addr = ftrace_jmp_set(addr);
+
+       ret = ftrace_set_filter_ip(ops, tr->ip, 0, 1);
+       if (ret)
+               return ret;
+       return register_ftrace_direct(ops, addr);
+}
+
+static int direct_ops_del(struct bpf_trampoline *tr, void *addr)
+{
+       return unregister_ftrace_direct(tr->fops, (long)addr, false);
+}
+
+static int direct_ops_mod(struct bpf_trampoline *tr, void *ptr, bool lock_direct_mutex)
+{
+       unsigned long addr = (unsigned long) ptr;
+       struct ftrace_ops *ops = tr->fops;
+
+       if (bpf_trampoline_use_jmp(tr->flags))
+               addr = ftrace_jmp_set(addr);
+       if (lock_direct_mutex)
+               return modify_ftrace_direct(ops, addr);
+       return modify_ftrace_direct_nolock(ops, addr);
+}
+#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */
+#else
+static void direct_ops_free(struct bpf_trampoline *tr) { }
+
+static int direct_ops_alloc(struct bpf_trampoline *tr)
+{
+       return 0;
+}
+
+static int direct_ops_add(struct bpf_trampoline *tr, void *addr)
+{
+       return -ENODEV;
+}
+
+static int direct_ops_del(struct bpf_trampoline *tr, void *addr)
+{
+       return -ENODEV;
+}
+
+static int direct_ops_mod(struct bpf_trampoline *tr, void *ptr, bool lock_direct_mutex)
+{
+       return -ENODEV;
+}
+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
+
 static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip)
 {
        struct bpf_trampoline *tr;
@@ -161,16 +345,11 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip)
        tr = kzalloc(sizeof(*tr), GFP_KERNEL);
        if (!tr)
                goto out;
-#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
-       tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);
-       if (!tr->fops) {
+       if (direct_ops_alloc(tr)) {
                kfree(tr);
                tr = NULL;
                goto out;
        }
-       tr->fops->private = tr;
-       tr->fops->ops_func = bpf_tramp_ftrace_ops_func;
-#endif
 
        tr->key = key;
        tr->ip = ftrace_location(ip);
@@ -213,7 +392,7 @@ static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags,
        int ret;
 
        if (tr->func.ftrace_managed)
-               ret = unregister_ftrace_direct(tr->fops, (long)old_addr, false);
+               ret = direct_ops_del(tr, old_addr);
        else
                ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, NULL);
 
@@ -227,15 +406,7 @@ static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags,
        int ret;
 
        if (tr->func.ftrace_managed) {
-               unsigned long addr = (unsigned long) new_addr;
-
-               if (bpf_trampoline_use_jmp(tr->flags))
-                       addr = ftrace_jmp_set(addr);
-
-               if (lock_direct_mutex)
-                       ret = modify_ftrace_direct(tr->fops, addr);
-               else
-                       ret = modify_ftrace_direct_nolock(tr->fops, addr);
+               ret = direct_ops_mod(tr, new_addr, lock_direct_mutex);
        } else {
                ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr,
                                                   new_addr);
@@ -258,15 +429,7 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
        }
 
        if (tr->func.ftrace_managed) {
-               unsigned long addr = (unsigned long) new_addr;
-
-               if (bpf_trampoline_use_jmp(tr->flags))
-                       addr = ftrace_jmp_set(addr);
-
-               ret = ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1);
-               if (ret)
-                       return ret;
-               ret = register_ftrace_direct(tr->fops, addr);
+               ret = direct_ops_add(tr, new_addr);
        } else {
                ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr);
        }
@@ -947,10 +1110,7 @@ void bpf_trampoline_put(struct bpf_trampoline *tr)
         */
        hlist_del(&tr->hlist_key);
        hlist_del(&tr->hlist_ip);
-       if (tr->fops) {
-               ftrace_free_filter(tr->fops);
-               kfree(tr->fops);
-       }
+       direct_ops_free(tr);
        kfree(tr);
 out:
        mutex_unlock(&trampoline_mutex);

diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index bfa2ec46e0752e3e281aab1499829aa92f211970..d7042a09fe469af8817dcfaa8cf37be889a125f8 100644 (file)
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -50,6 +50,9 @@ config HAVE_DYNAMIC_FTRACE_WITH_REGS
 config HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
        bool
 
+config HAVE_SINGLE_FTRACE_DIRECT_OPS
+       bool
+
 config HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
        bool
 

diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index ee2d7b8a43724dbf1bffdd73d29f0b266fb08ca7..8574932e66b634c8a45c9a20831e442502a8810f 100644 (file)
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -2631,8 +2631,13 @@ unsigned long ftrace_find_rec_direct(unsigned long ip)
 static void call_direct_funcs(unsigned long ip, unsigned long pip,
                              struct ftrace_ops *ops, struct ftrace_regs *fregs)
 {
-       unsigned long addr = READ_ONCE(ops->direct_call);
+       unsigned long addr;
 
+#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS
+       addr = ftrace_find_rec_direct(ip);
+#else
+       addr = READ_ONCE(ops->direct_call);
+#endif
        if (!addr)
                return;