bpf: Add dynptr type for skb metadata

Add a dynptr type, similar to skb dynptr, but for the skb metadata access.

The dynptr provides an alternative to __sk_buff->data_meta for accessing
the custom metadata area allocated using the bpf_xdp_adjust_meta() helper.

More importantly, it abstracts away the fact where the storage for the
custom metadata lives, which opens up the way to persist the metadata by
relocating it as the skb travels through the network stack layers.

Writes to skb metadata invalidate any existing skb payload and metadata
slices. While this is more restrictive that needed at the moment, it leaves
the door open to reallocating the metadata on writes, and should be only a
minor inconvenience to the users.

Only the program types which can access __sk_buff->data_meta today are
allowed to create a dynptr for skb metadata at the moment. We need to
modify the network stack to persist the metadata across layers before
opening up access to other BPF hooks.

Once more BPF hooks gain access to skb_meta dynptr, we will also need to
add a read-only variant of the helper similar to
bpf_dynptr_from_skb_rdonly.

skb_meta dynptr ops are stubbed out and implemented by subsequent changes.

Signed-off-by: Jakub Sitnicki <jakub@cloudflare.com>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Reviewed-by: Jesse Brandeburg <jbrandeburg@cloudflare.com>
Link: https://patch.msgid.link/20250814-skb-metadata-thru-dynptr-v7-1-8a39e636e0fb@cloudflare.com

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index cc700925b802fe0bb88844cc36120a9136788eaf..ec527b476dba71bd8b74480fa4cc65d58480cac4 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -767,12 +767,15 @@ enum bpf_type_flag {
         */
        MEM_WRITE               = BIT(18 + BPF_BASE_TYPE_BITS),
 
+       /* DYNPTR points to skb_metadata_end()-skb_metadata_len() */
+       DYNPTR_TYPE_SKB_META    = BIT(19 + BPF_BASE_TYPE_BITS),
+
        __BPF_TYPE_FLAG_MAX,
        __BPF_TYPE_LAST_FLAG    = __BPF_TYPE_FLAG_MAX - 1,
 };
 
 #define DYNPTR_TYPE_FLAG_MASK  (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \
-                                | DYNPTR_TYPE_XDP)
+                                | DYNPTR_TYPE_XDP | DYNPTR_TYPE_SKB_META)
 
 /* Max number of base types. */
 #define BPF_BASE_TYPE_LIMIT    (1UL << BPF_BASE_TYPE_BITS)
@@ -1358,6 +1361,8 @@ enum bpf_dynptr_type {
        BPF_DYNPTR_TYPE_SKB,
        /* Underlying data is a xdp_buff */
        BPF_DYNPTR_TYPE_XDP,
+       /* Points to skb_metadata_end()-skb_metadata_len() */
+       BPF_DYNPTR_TYPE_SKB_META,
 };
 
 int bpf_dynptr_check_size(u32 size);

diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 6b4877e85a68c9d92260dfc8405a867dc9498f29..9552b32208c54171e2933c3d0aee72bcef214f32 100644 (file)
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -1780,6 +1780,8 @@ static int __bpf_dynptr_read(void *dst, u32 len, const struct bpf_dynptr_kern *s
                return __bpf_skb_load_bytes(src->data, src->offset + offset, dst, len);
        case BPF_DYNPTR_TYPE_XDP:
                return __bpf_xdp_load_bytes(src->data, src->offset + offset, dst, len);
+       case BPF_DYNPTR_TYPE_SKB_META:
+               return -EOPNOTSUPP; /* not implemented */
        default:
                WARN_ONCE(true, "bpf_dynptr_read: unknown dynptr type %d\n", type);
                return -EFAULT;
@@ -1836,6 +1838,8 @@ int __bpf_dynptr_write(const struct bpf_dynptr_kern *dst, u32 offset, void *src,
                if (flags)
                        return -EINVAL;
                return __bpf_xdp_store_bytes(dst->data, dst->offset + offset, src, len);
+       case BPF_DYNPTR_TYPE_SKB_META:
+               return -EOPNOTSUPP; /* not implemented */
        default:
                WARN_ONCE(true, "bpf_dynptr_write: unknown dynptr type %d\n", type);
                return -EFAULT;
@@ -1882,6 +1886,7 @@ BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u3
                return (unsigned long)(ptr->data + ptr->offset + offset);
        case BPF_DYNPTR_TYPE_SKB:
        case BPF_DYNPTR_TYPE_XDP:
+       case BPF_DYNPTR_TYPE_SKB_META:
                /* skb and xdp dynptrs should use bpf_dynptr_slice / bpf_dynptr_slice_rdwr */
                return 0;
        default:
@@ -2710,6 +2715,8 @@ __bpf_kfunc void *bpf_dynptr_slice(const struct bpf_dynptr *p, u32 offset,
                bpf_xdp_copy_buf(ptr->data, ptr->offset + offset, buffer__opt, len, false);
                return buffer__opt;
        }
+       case BPF_DYNPTR_TYPE_SKB_META:
+               return NULL; /* not implemented */
        default:
                WARN_ONCE(true, "unknown dynptr type %d\n", type);
                return NULL;

diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
index 38050f4ee40030ff252624f3ac84a40460296acd..e4983c1303e76003ba971a71c5ff6b0c9aa751af 100644 (file)
--- a/kernel/bpf/log.c
+++ b/kernel/bpf/log.c
@@ -498,6 +498,8 @@ const char *dynptr_type_str(enum bpf_dynptr_type type)
                return "skb";
        case BPF_DYNPTR_TYPE_XDP:
                return "xdp";
+       case BPF_DYNPTR_TYPE_SKB_META:
+               return "skb_meta";
        case BPF_DYNPTR_TYPE_INVALID:
                return "<invalid>";
        default:

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index c4f69a9e9af6904d21177c851bb059541ba98057..5964bed40ffbf0083b385d931a1550453c0c8636 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -674,6 +674,8 @@ static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
                return BPF_DYNPTR_TYPE_SKB;
        case DYNPTR_TYPE_XDP:
                return BPF_DYNPTR_TYPE_XDP;
+       case DYNPTR_TYPE_SKB_META:
+               return BPF_DYNPTR_TYPE_SKB_META;
        default:
                return BPF_DYNPTR_TYPE_INVALID;
        }
@@ -690,6 +692,8 @@ static enum bpf_type_flag get_dynptr_type_flag(enum bpf_dynptr_type type)
                return DYNPTR_TYPE_SKB;
        case BPF_DYNPTR_TYPE_XDP:
                return DYNPTR_TYPE_XDP;
+       case BPF_DYNPTR_TYPE_SKB_META:
+               return DYNPTR_TYPE_SKB_META;
        default:
                return 0;
        }
@@ -2274,7 +2278,8 @@ static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
 static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg)
 {
        return base_type(reg->type) == PTR_TO_MEM &&
-               (reg->type & DYNPTR_TYPE_SKB || reg->type & DYNPTR_TYPE_XDP);
+              (reg->type &
+               (DYNPTR_TYPE_SKB | DYNPTR_TYPE_XDP | DYNPTR_TYPE_SKB_META));
 }
 
 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
@@ -11641,7 +11646,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
                if (dynptr_type == BPF_DYNPTR_TYPE_INVALID)
                        return -EFAULT;
 
-               if (dynptr_type == BPF_DYNPTR_TYPE_SKB)
+               if (dynptr_type == BPF_DYNPTR_TYPE_SKB ||
+                   dynptr_type == BPF_DYNPTR_TYPE_SKB_META)
                        /* this will trigger clear_all_pkt_pointers(), which will
                         * invalidate all dynptr slices associated with the skb
                         */
@@ -12228,6 +12234,7 @@ enum special_kfunc_type {
        KF_bpf_rbtree_right,
        KF_bpf_dynptr_from_skb,
        KF_bpf_dynptr_from_xdp,
+       KF_bpf_dynptr_from_skb_meta,
        KF_bpf_dynptr_slice,
        KF_bpf_dynptr_slice_rdwr,
        KF_bpf_dynptr_clone,
@@ -12277,9 +12284,11 @@ BTF_ID(func, bpf_rbtree_right)
 #ifdef CONFIG_NET
 BTF_ID(func, bpf_dynptr_from_skb)
 BTF_ID(func, bpf_dynptr_from_xdp)
+BTF_ID(func, bpf_dynptr_from_skb_meta)
 #else
 BTF_ID_UNUSED
 BTF_ID_UNUSED
+BTF_ID_UNUSED
 #endif
 BTF_ID(func, bpf_dynptr_slice)
 BTF_ID(func, bpf_dynptr_slice_rdwr)
@@ -13253,6 +13262,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
                                dynptr_arg_type |= DYNPTR_TYPE_SKB;
                        } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) {
                                dynptr_arg_type |= DYNPTR_TYPE_XDP;
+                       } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb_meta]) {
+                               dynptr_arg_type |= DYNPTR_TYPE_SKB_META;
                        } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] &&
                                   (dynptr_arg_type & MEM_UNINIT)) {
                                enum bpf_dynptr_type parent_type = meta->initialized_dynptr.type;

diff --git a/net/core/filter.c b/net/core/filter.c
index da391e2b0788d0f125c260fcafee5f43d5d6067f..31b4b50dbadf3f02ecc2f70ddf95a1c62caa5350 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -12007,6 +12007,36 @@ __bpf_kfunc int bpf_dynptr_from_skb(struct __sk_buff *s, u64 flags,
        return 0;
 }
 
+/**
+ * bpf_dynptr_from_skb_meta() - Initialize a dynptr to the skb metadata area.
+ * @skb_: socket buffer carrying the metadata
+ * @flags: future use, must be zero
+ * @ptr__uninit: dynptr to initialize
+ *
+ * Set up a dynptr for access to the metadata area earlier allocated from the
+ * XDP context with bpf_xdp_adjust_meta(). Serves as an alternative to
+ * &__sk_buff->data_meta.
+ *
+ * Return:
+ * * %0         - dynptr ready to use
+ * * %-EINVAL   - invalid flags, dynptr set to null
+ */
+__bpf_kfunc int bpf_dynptr_from_skb_meta(struct __sk_buff *skb_, u64 flags,
+                                        struct bpf_dynptr *ptr__uninit)
+{
+       struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)ptr__uninit;
+       struct sk_buff *skb = (struct sk_buff *)skb_;
+
+       if (flags) {
+               bpf_dynptr_set_null(ptr);
+               return -EINVAL;
+       }
+
+       bpf_dynptr_init(ptr, skb, BPF_DYNPTR_TYPE_SKB_META, 0, skb_metadata_len(skb));
+
+       return 0;
+}
+
 __bpf_kfunc int bpf_dynptr_from_xdp(struct xdp_md *x, u64 flags,
                                    struct bpf_dynptr *ptr__uninit)
 {
@@ -12181,6 +12211,10 @@ BTF_KFUNCS_START(bpf_kfunc_check_set_skb)
 BTF_ID_FLAGS(func, bpf_dynptr_from_skb, KF_TRUSTED_ARGS)
 BTF_KFUNCS_END(bpf_kfunc_check_set_skb)
 
+BTF_KFUNCS_START(bpf_kfunc_check_set_skb_meta)
+BTF_ID_FLAGS(func, bpf_dynptr_from_skb_meta, KF_TRUSTED_ARGS)
+BTF_KFUNCS_END(bpf_kfunc_check_set_skb_meta)
+
 BTF_KFUNCS_START(bpf_kfunc_check_set_xdp)
 BTF_ID_FLAGS(func, bpf_dynptr_from_xdp)
 BTF_KFUNCS_END(bpf_kfunc_check_set_xdp)
@@ -12202,6 +12236,11 @@ static const struct btf_kfunc_id_set bpf_kfunc_set_skb = {
        .set = &bpf_kfunc_check_set_skb,
 };
 
+static const struct btf_kfunc_id_set bpf_kfunc_set_skb_meta = {
+       .owner = THIS_MODULE,
+       .set = &bpf_kfunc_check_set_skb_meta,
+};
+
 static const struct btf_kfunc_id_set bpf_kfunc_set_xdp = {
        .owner = THIS_MODULE,
        .set = &bpf_kfunc_check_set_xdp,
@@ -12237,6 +12276,8 @@ static int __init bpf_kfunc_init(void)
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_NETFILTER, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_kfunc_set_skb);
+       ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_skb_meta);
+       ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_ACT, &bpf_kfunc_set_skb_meta);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &bpf_kfunc_set_xdp);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
                                               &bpf_kfunc_set_sock_addr);