bpf: support fsession for bpf_session_cookie

Implement session cookie for fsession. The session cookies will be stored
in the stack, and the layout of the stack will look like this:
  return value	-> 8 bytes
  argN		-> 8 bytes
  ...
  arg1		-> 8 bytes
  nr_args	-> 8 bytes
  ip (optional)	-> 8 bytes
  cookie2	-> 8 bytes
  cookie1	-> 8 bytes

The offset of the cookie for the current bpf program, which is in 8-byte
units, is stored in the
"(((u64 *)ctx)[-1] >> BPF_TRAMP_COOKIE_INDEX_SHIFT) & 0xFF". Therefore, we
can get the session cookie with ((u64 *)ctx)[-offset].

Implement and inline the bpf_session_cookie() for the fsession in the
verifier.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
Link: https://lore.kernel.org/r/20260124062008.8657-6-dongml2@chinatelecom.cn
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 29eecd79352e9e2ea489b9bbfa95a721ecbe7b30..4427c6e9833196615d02468d78572c09e14f342a 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1229,6 +1229,7 @@ enum {
 #endif
 };
 
+#define BPF_TRAMP_COOKIE_INDEX_SHIFT   8
 #define BPF_TRAMP_IS_RETURN_SHIFT      63
 
 struct bpf_tramp_links {
@@ -1782,6 +1783,7 @@ struct bpf_prog {
                                enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
                                call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
                                call_get_func_ip:1, /* Do we call get_func_ip() */
+                               call_session_cookie:1, /* Do we call bpf_session_cookie() */
                                tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */
                                sleepable:1;    /* BPF program is sleepable */
        enum bpf_prog_type      type;           /* Type of BPF program */
@@ -2190,6 +2192,19 @@ static inline int bpf_fsession_cnt(struct bpf_tramp_links *links)
        return cnt;
 }
 
+static inline int bpf_fsession_cookie_cnt(struct bpf_tramp_links *links)
+{
+       struct bpf_tramp_links fentries = links[BPF_TRAMP_FENTRY];
+       int cnt = 0;
+
+       for (int i = 0; i < links[BPF_TRAMP_FENTRY].nr_links; i++) {
+               if (fentries.links[i]->link.prog->call_session_cookie)
+                       cnt++;
+       }
+
+       return cnt;
+}
+
 int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
                               const struct bpf_ctx_arg_aux *info, u32 cnt);
 

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index d04aea235a128f0e5fb1949617a7240454e3a2a1..c2f2650db9fdf71737eecd56febbe3cbf8fe42b4 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -14406,6 +14406,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                        return err;
        }
 
+       if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie])
+               env->prog->call_session_cookie = true;
+
        return 0;
 }
 
@@ -23024,6 +23027,23 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                insn_buf[1] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_0, BPF_TRAMP_IS_RETURN_SHIFT);
                insn_buf[2] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 1);
                *cnt = 3;
+       } else if (desc->func_id == special_kfunc_list[KF_bpf_session_cookie] &&
+                  env->prog->expected_attach_type == BPF_TRACE_FSESSION) {
+               /*
+                * inline bpf_session_cookie() for fsession:
+                *   __u64 *bpf_session_cookie(void *ctx)
+                *   {
+                *       u64 off = (((u64 *)ctx)[-1] >> BPF_TRAMP_COOKIE_INDEX_SHIFT) & 0xFF;
+                *       return &((u64 *)ctx)[-off];
+                *   }
+                */
+               insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
+               insn_buf[1] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_0, BPF_TRAMP_COOKIE_INDEX_SHIFT);
+               insn_buf[2] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF);
+               insn_buf[3] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
+               insn_buf[4] = BPF_ALU64_REG(BPF_SUB, BPF_REG_0, BPF_REG_1);
+               insn_buf[5] = BPF_ALU64_IMM(BPF_NEG, BPF_REG_0, 0);
+               *cnt = 6;
        }
 
        if (env->insn_aux_data[insn_idx].arg_prog) {