bpf: Make bpf_skb_adjust_room metadata-safe

bpf_skb_adjust_room() may push or pull bytes from skb->data. In both cases,
skb metadata must be moved accordingly to stay accessible.

Replace existing memmove() calls, which only move payload, with a helper
that also handles metadata. Reserve enough space for metadata to fit after
skb_push.

Signed-off-by: Jakub Sitnicki <jakub@cloudflare.com>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://patch.msgid.link/20251105-skb-meta-rx-path-v4-7-5ceb08a9b37b@cloudflare.com

diff --git a/net/core/filter.c b/net/core/filter.c
index 673299fd3d5896cdd81dbf32c7b27a4c9f03b0e3..297f0ae9dc1fe5c939683c7707b649b8121beded 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -3253,11 +3253,11 @@ static void bpf_skb_change_protocol(struct sk_buff *skb, u16 proto)
 
 static int bpf_skb_generic_push(struct sk_buff *skb, u32 off, u32 len)
 {
-       /* Caller already did skb_cow() with len as headroom,
+       /* Caller already did skb_cow() with meta_len+len as headroom,
         * so no need to do it here.
         */
        skb_push(skb, len);
-       memmove(skb->data, skb->data + len, off);
+       skb_postpush_data_move(skb, len, off);
        memset(skb->data + off, 0, len);
 
        /* No skb_postpush_rcsum(skb, skb->data + off, len)
@@ -3281,7 +3281,7 @@ static int bpf_skb_generic_pop(struct sk_buff *skb, u32 off, u32 len)
        old_data = skb->data;
        __skb_pull(skb, len);
        skb_postpull_rcsum(skb, old_data + off, len);
-       memmove(skb->data, old_data, off);
+       skb_postpull_data_move(skb, len, off);
 
        return 0;
 }
@@ -3489,6 +3489,7 @@ static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff,
        u8 inner_mac_len = flags >> BPF_ADJ_ROOM_ENCAP_L2_SHIFT;
        bool encap = flags & BPF_F_ADJ_ROOM_ENCAP_L3_MASK;
        u16 mac_len = 0, inner_net = 0, inner_trans = 0;
+       const u8 meta_len = skb_metadata_len(skb);
        unsigned int gso_type = SKB_GSO_DODGY;
        int ret;
 
@@ -3499,7 +3500,7 @@ static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff,
                        return -ENOTSUPP;
        }
 
-       ret = skb_cow_head(skb, len_diff);
+       ret = skb_cow_head(skb, meta_len + len_diff);
        if (unlikely(ret < 0))
                return ret;