bpf: Add support for tracing_multi link cookies

Add support to specify cookies for tracing_multi link.

Cookies are provided in array where each value is paired with provided
BTF ID value with the same array index.

Such cookie can be retrieved by bpf program with bpf_get_attach_cookie
helper call.

We need to sort cookies array together with ids array in check_dup_ids,
to keep the id->cookie relation.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Link: https://lore.kernel.org/r/20260606123955.345967-15-jolsa@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index bcf70f810d2ca1ebab2ed1f69ec9571df2bc6e72..e9d2b42a39816a192f2e3b657b4c438f63322408 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1971,6 +1971,7 @@ struct bpf_tracing_multi_data {
 struct bpf_tracing_multi_link {
        struct bpf_link link;
        struct bpf_tracing_multi_data data;
+       u64 *cookies;
        int nodes_cnt;
        struct bpf_tracing_multi_node nodes[] __counted_by(nodes_cnt);
 };

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 9f603731d26753e080154f748c9ecf544762e637..569c15e1cae31cde628f7ff472e2d4276bf974f0 100644 (file)
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -1880,6 +1880,7 @@ union bpf_attr {
                        } cgroup;
                        struct {
                                __aligned_u64   ids;
+                               __aligned_u64   cookies;
                                __u32           cnt;
                        } tracing_multi;
                };

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 957e5d7f9554ad395d81ab4cfb88cc91716d2280..a3537fda50cf18e9bfaf0c5faeb5428b392e4009 100644 (file)
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -1613,6 +1613,7 @@ int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids,
 
                mnode->trampoline = tr;
                mnode->node.link = &link->link;
+               mnode->node.cookie = link->cookies ? link->cookies[i] : 0;
 
                cond_resched();
        }

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 9e3cb547651e53e0f54b084c9ed98ec7f0c7b81f..e33492739ed186c9c604b85d5d5d55869eca17c9 100644 (file)
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -3659,6 +3659,7 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link)
        struct bpf_tracing_multi_link *tr_link =
                container_of(link, struct bpf_tracing_multi_link, link);
 
+       kvfree(tr_link->cookies);
        kvfree(tr_link);
 }
 
@@ -3678,13 +3679,24 @@ static int ids_cmp_r(const void *pa, const void *pb, const void *priv __maybe_un
 static void ids_swap_r(void *a, void *b, int size __maybe_unused,
                       const void *priv __maybe_unused)
 {
-       u32 *id_a = a, *id_b = b;
+       u64 *cookie_a, *cookie_b, *cookies;
+       u32 *id_a = a, *id_b = b, *ids;
+       void **data = (void **) priv;
 
+       ids     = data[0];
+       cookies = data[1];
+
+       if (cookies) {
+               cookie_a = cookies + (id_a - ids);
+               cookie_b = cookies + (id_b - ids);
+               swap(*cookie_a, *cookie_b);
+       }
        swap(*id_a, *id_b);
 }
 
-static int check_dup_ids(u32 *ids, u32 cnt)
+static int check_dup_ids(u32 *ids, u64 *cookies, u32 cnt)
 {
+       void *data[2] = { ids, cookies };
        int err = 0;
 
        /*
@@ -3692,7 +3704,7 @@ static int check_dup_ids(u32 *ids, u32 cnt)
         * and check it for duplicates. The ids and cookies arrays
         * are left sorted.
         */
-       sort_r_nonatomic(ids, cnt, sizeof(ids[0]), ids_cmp_r, ids_swap_r, NULL);
+       sort_r_nonatomic(ids, cnt, sizeof(ids[0]), ids_cmp_r, ids_swap_r, data);
 
        for (int i = 1; i < cnt; i++) {
                if (ids[i] == ids[i - 1]) {
@@ -3708,6 +3720,8 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr)
        struct bpf_tracing_multi_link *link = NULL;
        struct bpf_link_primer link_primer;
        u32 cnt, *ids = NULL;
+       u64 __user *ucookies;
+       u64 *cookies = NULL;
        u32 __user *uids;
        int err;
 
@@ -3730,7 +3744,20 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr)
                goto error;
        }
 
-       err = check_dup_ids(ids, cnt);
+       ucookies = u64_to_user_ptr(attr->link_create.tracing_multi.cookies);
+       if (ucookies) {
+               cookies = kvmalloc_objs(*cookies, cnt);
+               if (!cookies) {
+                       err = -ENOMEM;
+                       goto error;
+               }
+               if (copy_from_user(cookies, ucookies, cnt * sizeof(*cookies))) {
+                       err = -EFAULT;
+                       goto error;
+               }
+       }
+
+       err = check_dup_ids(ids, cookies, cnt);
        if (err)
                goto error;
 
@@ -3748,6 +3775,7 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr)
                goto error;
 
        link->nodes_cnt = cnt;
+       link->cookies = cookies;
 
        err = bpf_trampoline_multi_attach(prog, ids, link);
        kvfree(ids);
@@ -3758,6 +3786,7 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr)
        return bpf_link_settle(&link_primer);
 
 error:
+       kvfree(cookies);
        kvfree(ids);
        kvfree(link);
        return err;

diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 9f603731d26753e080154f748c9ecf544762e637..569c15e1cae31cde628f7ff472e2d4276bf974f0 100644 (file)
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -1880,6 +1880,7 @@ union bpf_attr {
                        } cgroup;
                        struct {
                                __aligned_u64   ids;
+                               __aligned_u64   cookies;
                                __u32           cnt;
                        } tracing_multi;
                };