bpf: Add trampoline ip hash table

Following changes need to lookup trampoline based on its ip address,
adding hash table for that.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20251230145010.103439-8-jolsa@kernel.org

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 9272a237cced25d38a18c6d28387819cc66a3906..5524f9429e7626166615757a640863a89617020a 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1329,14 +1329,17 @@ struct bpf_tramp_image {
 };
 
 struct bpf_trampoline {
-       /* hlist for trampoline_table */
-       struct hlist_node hlist;
+       /* hlist for trampoline_key_table */
+       struct hlist_node hlist_key;
+       /* hlist for trampoline_ip_table */
+       struct hlist_node hlist_ip;
        struct ftrace_ops *fops;
        /* serializes access to fields of this trampoline */
        struct mutex mutex;
        refcount_t refcnt;
        u32 flags;
        u64 key;
+       unsigned long ip;
        struct {
                struct btf_func_model model;
                void *addr;

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 468de930b2eda0828b116d1504385b1f2195e1f5..1d4f618c2402b4e2e24c7a9d8d0abd9b5a59d8b9 100644 (file)
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -24,9 +24,10 @@ const struct bpf_prog_ops bpf_extension_prog_ops = {
 #define TRAMPOLINE_HASH_BITS 10
 #define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS)
 
-static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE];
+static struct hlist_head trampoline_key_table[TRAMPOLINE_TABLE_SIZE];
+static struct hlist_head trampoline_ip_table[TRAMPOLINE_TABLE_SIZE];
 
-/* serializes access to trampoline_table */
+/* serializes access to trampoline tables */
 static DEFINE_MUTEX(trampoline_mutex);
 
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
@@ -142,15 +143,15 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym)
                           PAGE_SIZE, true, ksym->name);
 }
 
-static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
+static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip)
 {
        struct bpf_trampoline *tr;
        struct hlist_head *head;
        int i;
 
        mutex_lock(&trampoline_mutex);
-       head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)];
-       hlist_for_each_entry(tr, head, hlist) {
+       head = &trampoline_key_table[hash_64(key, TRAMPOLINE_HASH_BITS)];
+       hlist_for_each_entry(tr, head, hlist_key) {
                if (tr->key == key) {
                        refcount_inc(&tr->refcnt);
                        goto out;
@@ -171,8 +172,12 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
 #endif
 
        tr->key = key;
-       INIT_HLIST_NODE(&tr->hlist);
-       hlist_add_head(&tr->hlist, head);
+       tr->ip = ftrace_location(ip);
+       INIT_HLIST_NODE(&tr->hlist_key);
+       INIT_HLIST_NODE(&tr->hlist_ip);
+       hlist_add_head(&tr->hlist_key, head);
+       head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)];
+       hlist_add_head(&tr->hlist_ip, head);
        refcount_set(&tr->refcnt, 1);
        mutex_init(&tr->mutex);
        for (i = 0; i < BPF_TRAMP_MAX; i++)
@@ -883,7 +888,7 @@ void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
                                         prog->aux->attach_btf_id);
 
        bpf_lsm_find_cgroup_shim(prog, &bpf_func);
-       tr = bpf_trampoline_lookup(key);
+       tr = bpf_trampoline_lookup(key, 0);
        if (WARN_ON_ONCE(!tr))
                return;
 
@@ -903,7 +908,7 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key,
 {
        struct bpf_trampoline *tr;
 
-       tr = bpf_trampoline_lookup(key);
+       tr = bpf_trampoline_lookup(key, tgt_info->tgt_addr);
        if (!tr)
                return NULL;
 
@@ -939,7 +944,8 @@ void bpf_trampoline_put(struct bpf_trampoline *tr)
         * fexit progs. The fentry-only trampoline will be freed via
         * multiple rcu callbacks.
         */
-       hlist_del(&tr->hlist);
+       hlist_del(&tr->hlist_key);
+       hlist_del(&tr->hlist_ip);
        if (tr->fops) {
                ftrace_free_filter(tr->fops);
                kfree(tr->fops);
@@ -1212,7 +1218,9 @@ static int __init init_trampolines(void)
        int i;
 
        for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++)
-               INIT_HLIST_HEAD(&trampoline_table[i]);
+               INIT_HLIST_HEAD(&trampoline_key_table[i]);
+       for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++)
+               INIT_HLIST_HEAD(&trampoline_ip_table[i]);
        return 0;
 }
 late_initcall(init_trampolines);