bpf: add fsession support

The fsession is something that similar to kprobe session. It allow to
attach a single BPF program to both the entry and the exit of the target
functions.

Introduce the struct bpf_fsession_link, which allows to add the link to
both the fentry and fexit progs_hlist of the trampoline.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
Co-developed-by: Leon Hwang <leon.hwang@linux.dev>
Signed-off-by: Leon Hwang <leon.hwang@linux.dev>
Link: https://lore.kernel.org/r/20260124062008.8657-2-dongml2@chinatelecom.cn
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 5936f8e2996f19e34f8f0d9249108f3835ed42d1..41228b0add52ecb985ce32bd8c6bed2d4835021e 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1309,6 +1309,7 @@ enum bpf_tramp_prog_type {
        BPF_TRAMP_MODIFY_RETURN,
        BPF_TRAMP_MAX,
        BPF_TRAMP_REPLACE, /* more than MAX */
+       BPF_TRAMP_FSESSION,
 };
 
 struct bpf_tramp_image {
@@ -1875,6 +1876,11 @@ struct bpf_tracing_link {
        struct bpf_prog *tgt_prog;
 };
 
+struct bpf_fsession_link {
+       struct bpf_tracing_link link;
+       struct bpf_tramp_link fexit;
+};
+
 struct bpf_raw_tp_link {
        struct bpf_link link;
        struct bpf_raw_event_map *btp;
@@ -2169,6 +2175,19 @@ static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_op
 
 #endif
 
+static inline int bpf_fsession_cnt(struct bpf_tramp_links *links)
+{
+       struct bpf_tramp_links fentries = links[BPF_TRAMP_FENTRY];
+       int cnt = 0;
+
+       for (int i = 0; i < links[BPF_TRAMP_FENTRY].nr_links; i++) {
+               if (fentries.links[i]->link.prog->expected_attach_type == BPF_TRACE_FSESSION)
+                       cnt++;
+       }
+
+       return cnt;
+}
+
 int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
                               const struct bpf_ctx_arg_aux *info, u32 cnt);
 

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 2a2ade4be60f2c9febc38b6e427bf32cc4b15efc..44e7dbc278e37179f50a57afbaf30e776fe8644b 100644 (file)
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -1145,6 +1145,7 @@ enum bpf_attach_type {
        BPF_NETKIT_PEER,
        BPF_TRACE_KPROBE_SESSION,
        BPF_TRACE_UPROBE_SESSION,
+       BPF_TRACE_FSESSION,
        __MAX_BPF_ATTACH_TYPE
 };
 

diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index d10b3404260f354e03a01a32702c7c4b41534117..8959f3bc1e927be3b574179da8acb5a129702006 100644 (file)
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -6219,6 +6219,7 @@ static int btf_validate_prog_ctx_type(struct bpf_verifier_log *log, const struct
                case BPF_TRACE_FENTRY:
                case BPF_TRACE_FEXIT:
                case BPF_MODIFY_RETURN:
+               case BPF_TRACE_FSESSION:
                        /* allow u64* as ctx */
                        if (btf_is_int(t) && t->size == 8)
                                return 0;
@@ -6820,6 +6821,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
                        fallthrough;
                case BPF_LSM_CGROUP:
                case BPF_TRACE_FEXIT:
+               case BPF_TRACE_FSESSION:
                        /* When LSM programs are attached to void LSM hooks
                         * they use FEXIT trampolines and when attached to
                         * int LSM hooks, they use MODIFY_RETURN trampolines.

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 3c5c03d43f5f74380dd23549971b666e89ecbe61..b9184545c3fd09956c3917f2f83f4b782918e4be 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -3577,6 +3577,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
        case BPF_PROG_TYPE_TRACING:
                if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
                    prog->expected_attach_type != BPF_TRACE_FEXIT &&
+                   prog->expected_attach_type != BPF_TRACE_FSESSION &&
                    prog->expected_attach_type != BPF_MODIFY_RETURN) {
                        err = -EINVAL;
                        goto out_put_prog;
@@ -3626,7 +3627,21 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
                key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);
        }
 
-       link = kzalloc(sizeof(*link), GFP_USER);
+       if (prog->expected_attach_type == BPF_TRACE_FSESSION) {
+               struct bpf_fsession_link *fslink;
+
+               fslink = kzalloc(sizeof(*fslink), GFP_USER);
+               if (fslink) {
+                       bpf_link_init(&fslink->fexit.link, BPF_LINK_TYPE_TRACING,
+                                     &bpf_tracing_link_lops, prog, attach_type);
+                       fslink->fexit.cookie = bpf_cookie;
+                       link = &fslink->link;
+               } else {
+                       link = NULL;
+               }
+       } else {
+               link = kzalloc(sizeof(*link), GFP_USER);
+       }
        if (!link) {
                err = -ENOMEM;
                goto out_put_prog;
@@ -4350,6 +4365,7 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type)
        case BPF_TRACE_RAW_TP:
        case BPF_TRACE_FENTRY:
        case BPF_TRACE_FEXIT:
+       case BPF_TRACE_FSESSION:
        case BPF_MODIFY_RETURN:
                return BPF_PROG_TYPE_TRACING;
        case BPF_LSM_MAC:

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 2a125d063e62b534d7413f64448f22512b5608c1..edf9da43762d7e8a2bcdc443e52b8c225633c0e7 100644 (file)
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -109,10 +109,17 @@ bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
        enum bpf_attach_type eatype = prog->expected_attach_type;
        enum bpf_prog_type ptype = prog->type;
 
-       return (ptype == BPF_PROG_TYPE_TRACING &&
-               (eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||
-                eatype == BPF_MODIFY_RETURN)) ||
-               (ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);
+       switch (ptype) {
+       case BPF_PROG_TYPE_TRACING:
+               if (eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||
+                   eatype == BPF_MODIFY_RETURN || eatype == BPF_TRACE_FSESSION)
+                       return true;
+               return false;
+       case BPF_PROG_TYPE_LSM:
+               return eatype == BPF_LSM_MAC;
+       default:
+               return false;
+       }
 }
 
 void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym)
@@ -559,6 +566,8 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
                return BPF_TRAMP_MODIFY_RETURN;
        case BPF_TRACE_FEXIT:
                return BPF_TRAMP_FEXIT;
+       case BPF_TRACE_FSESSION:
+               return BPF_TRAMP_FSESSION;
        case BPF_LSM_MAC:
                if (!prog->aux->attach_func_proto->type)
                        /* The function returns void, we cannot modify its
@@ -594,8 +603,10 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link,
                                      struct bpf_trampoline *tr,
                                      struct bpf_prog *tgt_prog)
 {
+       struct bpf_fsession_link *fslink = NULL;
        enum bpf_tramp_prog_type kind;
        struct bpf_tramp_link *link_exiting;
+       struct hlist_head *prog_list;
        int err = 0;
        int cnt = 0, i;
 
@@ -621,24 +632,43 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link,
                                          BPF_MOD_JUMP, NULL,
                                          link->link.prog->bpf_func);
        }
+       if (kind == BPF_TRAMP_FSESSION) {
+               prog_list = &tr->progs_hlist[BPF_TRAMP_FENTRY];
+               cnt++;
+       } else {
+               prog_list = &tr->progs_hlist[kind];
+       }
        if (cnt >= BPF_MAX_TRAMP_LINKS)
                return -E2BIG;
        if (!hlist_unhashed(&link->tramp_hlist))
                /* prog already linked */
                return -EBUSY;
-       hlist_for_each_entry(link_exiting, &tr->progs_hlist[kind], tramp_hlist) {
+       hlist_for_each_entry(link_exiting, prog_list, tramp_hlist) {
                if (link_exiting->link.prog != link->link.prog)
                        continue;
                /* prog already linked */
                return -EBUSY;
        }
 
-       hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]);
-       tr->progs_cnt[kind]++;
+       hlist_add_head(&link->tramp_hlist, prog_list);
+       if (kind == BPF_TRAMP_FSESSION) {
+               tr->progs_cnt[BPF_TRAMP_FENTRY]++;
+               fslink = container_of(link, struct bpf_fsession_link, link.link);
+               hlist_add_head(&fslink->fexit.tramp_hlist, &tr->progs_hlist[BPF_TRAMP_FEXIT]);
+               tr->progs_cnt[BPF_TRAMP_FEXIT]++;
+       } else {
+               tr->progs_cnt[kind]++;
+       }
        err = bpf_trampoline_update(tr, true /* lock_direct_mutex */);
        if (err) {
                hlist_del_init(&link->tramp_hlist);
-               tr->progs_cnt[kind]--;
+               if (kind == BPF_TRAMP_FSESSION) {
+                       tr->progs_cnt[BPF_TRAMP_FENTRY]--;
+                       hlist_del_init(&fslink->fexit.tramp_hlist);
+                       tr->progs_cnt[BPF_TRAMP_FEXIT]--;
+               } else {
+                       tr->progs_cnt[kind]--;
+               }
        }
        return err;
 }
@@ -672,6 +702,13 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
                guard(mutex)(&tgt_prog->aux->ext_mutex);
                tgt_prog->aux->is_extended = false;
                return err;
+       } else if (kind == BPF_TRAMP_FSESSION) {
+               struct bpf_fsession_link *fslink =
+                       container_of(link, struct bpf_fsession_link, link.link);
+
+               hlist_del_init(&fslink->fexit.tramp_hlist);
+               tr->progs_cnt[BPF_TRAMP_FEXIT]--;
+               kind = BPF_TRAMP_FENTRY;
        }
        hlist_del_init(&link->tramp_hlist);
        tr->progs_cnt[kind]--;

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index c7f5234d5fd220de456cf5808b623f7a3fca440f..41bbed6418b5198c959914fc1299b9a8f16e1da8 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -17848,6 +17848,7 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char
                switch (env->prog->expected_attach_type) {
                case BPF_TRACE_FENTRY:
                case BPF_TRACE_FEXIT:
+               case BPF_TRACE_FSESSION:
                        range = retval_range(0, 0);
                        break;
                case BPF_TRACE_RAW_TP:
@@ -23774,6 +23775,7 @@ patch_map_ops_generic:
                if (prog_type == BPF_PROG_TYPE_TRACING &&
                    insn->imm == BPF_FUNC_get_func_ret) {
                        if (eatype == BPF_TRACE_FEXIT ||
+                           eatype == BPF_TRACE_FSESSION ||
                            eatype == BPF_MODIFY_RETURN) {
                                /* Load nr_args from ctx - 8 */
                                insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
@@ -24725,7 +24727,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
                if (tgt_prog->type == BPF_PROG_TYPE_TRACING &&
                    prog_extension &&
                    (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY ||
-                    tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) {
+                    tgt_prog->expected_attach_type == BPF_TRACE_FEXIT ||
+                    tgt_prog->expected_attach_type == BPF_TRACE_FSESSION)) {
                        /* Program extensions can extend all program types
                         * except fentry/fexit. The reason is the following.
                         * The fentry/fexit programs are used for performance
@@ -24740,7 +24743,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
                         * beyond reasonable stack size. Hence extending fentry
                         * is not allowed.
                         */
-                       bpf_log(log, "Cannot extend fentry/fexit\n");
+                       bpf_log(log, "Cannot extend fentry/fexit/fsession\n");
                        return -EINVAL;
                }
        } else {
@@ -24824,6 +24827,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
        case BPF_LSM_CGROUP:
        case BPF_TRACE_FENTRY:
        case BPF_TRACE_FEXIT:
+       case BPF_TRACE_FSESSION:
                if (!btf_type_is_func(t)) {
                        bpf_log(log, "attach_btf_id %u is not a function\n",
                                btf_id);
@@ -24990,6 +24994,7 @@ static bool can_be_sleepable(struct bpf_prog *prog)
                case BPF_TRACE_FEXIT:
                case BPF_MODIFY_RETURN:
                case BPF_TRACE_ITER:
+               case BPF_TRACE_FSESSION:
                        return true;
                default:
                        return false;
@@ -25071,9 +25076,10 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
                        tgt_info.tgt_name);
                return -EINVAL;
        } else if ((prog->expected_attach_type == BPF_TRACE_FEXIT ||
+                  prog->expected_attach_type == BPF_TRACE_FSESSION ||
                   prog->expected_attach_type == BPF_MODIFY_RETURN) &&
                   btf_id_set_contains(&noreturn_deny, btf_id)) {
-               verbose(env, "Attaching fexit/fmod_ret to __noreturn function '%s' is rejected.\n",
+               verbose(env, "Attaching fexit/fsession/fmod_ret to __noreturn function '%s' is rejected.\n",
                        tgt_info.tgt_name);
                return -EINVAL;
        }

diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index 26cfcfdc45eb72305e4cceff707adec2bfcb83c3..178c4738e63bec0a9390209f9c3726786e631342 100644 (file)
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -685,6 +685,7 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog,
        switch (prog->expected_attach_type) {
        case BPF_TRACE_FENTRY:
        case BPF_TRACE_FEXIT:
+       case BPF_TRACE_FSESSION:
                if (bpf_fentry_test1(1) != 2 ||
                    bpf_fentry_test2(2, 3) != 5 ||
                    bpf_fentry_test3(4, 5, 6) != 15 ||

diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
index 850dd736ccd12504fb8982f84d304cc04ef1be86..de111818f3a037a86bf43aa3248bd5d355fb37a7 100644 (file)
--- a/net/core/bpf_sk_storage.c
+++ b/net/core/bpf_sk_storage.c
@@ -365,6 +365,7 @@ static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog)
                return true;
        case BPF_TRACE_FENTRY:
        case BPF_TRACE_FEXIT:
+       case BPF_TRACE_FSESSION:
                return !!strncmp(prog->aux->attach_func_name, "bpf_sk_storage",
                                 strlen("bpf_sk_storage"));
        default:

diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index b816bc53d2e159c5f2e154eccf72929fabcd9278..3ca7d76e05f04457ae09d1b09680299f135fa6e4 100644 (file)
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -1145,6 +1145,7 @@ enum bpf_attach_type {
        BPF_NETKIT_PEER,
        BPF_TRACE_KPROBE_SESSION,
        BPF_TRACE_UPROBE_SESSION,
+       BPF_TRACE_FSESSION,
        __MAX_BPF_ATTACH_TYPE
 };
 

diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_failure.c b/tools/testing/selftests/bpf/prog_tests/tracing_failure.c
index 10e231965589e9ce2dd59494bd0dd587a9784c7f..f9f9e1cb87bf1ca594938ed56b86b7405ad49043 100644 (file)
--- a/tools/testing/selftests/bpf/prog_tests/tracing_failure.c
+++ b/tools/testing/selftests/bpf/prog_tests/tracing_failure.c
@@ -73,7 +73,7 @@ static void test_tracing_deny(void)
 static void test_fexit_noreturns(void)
 {
        test_tracing_fail_prog("fexit_noreturns",
-                              "Attaching fexit/fmod_ret to __noreturn function 'do_exit' is rejected.");
+                              "Attaching fexit/fsession/fmod_ret to __noreturn function 'do_exit' is rejected.");
 }
 
 void test_tracing_failure(void)