bpf: Allow bpf_xdp_shrink_data to shrink a frag from head and tail

Move skb_frag_t adjustment into bpf_xdp_shrink_data() and extend its
functionality to be able to shrink an xdp fragment from both head and
tail. In a later patch, bpf_xdp_pull_data() will reuse it to shrink an
xdp fragment from head.

Additionally, in bpf_xdp_frags_shrink_tail(), breaking the loop when
bpf_xdp_shrink_data() returns false (i.e., not releasing the current
fragment) is not necessary as the loop condition, offset > 0, has the
same effect. Remove the else branch to simplify the code.

Signed-off-by: Amery Hung <ameryhung@gmail.com>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Reviewed-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
Link: https://patch.msgid.link/20250922233356.3356453-3-ameryhung@gmail.com

diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h
index 513c8e9704f657f05101f83127d66960407ac291..4f2d3268a6769dc6ac69a8bc3c8c4ef31b602dd8 100644 (file)
--- a/include/net/xdp_sock_drv.h
+++ b/include/net/xdp_sock_drv.h
@@ -160,13 +160,23 @@ static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first)
        return ret;
 }
 
-static inline void xsk_buff_del_tail(struct xdp_buff *tail)
+static inline void xsk_buff_del_frag(struct xdp_buff *xdp)
 {
-       struct xdp_buff_xsk *xskb = container_of(tail, struct xdp_buff_xsk, xdp);
+       struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
 
        list_del(&xskb->list_node);
 }
 
+static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first)
+{
+       struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp);
+       struct xdp_buff_xsk *frag;
+
+       frag = list_first_entry(&xskb->pool->xskb_list, struct xdp_buff_xsk,
+                               list_node);
+       return &frag->xdp;
+}
+
 static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first)
 {
        struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp);
@@ -389,8 +399,13 @@ static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first)
        return NULL;
 }
 
-static inline void xsk_buff_del_tail(struct xdp_buff *tail)
+static inline void xsk_buff_del_frag(struct xdp_buff *xdp)
+{
+}
+
+static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first)
 {
+       return NULL;
 }
 
 static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first)

diff --git a/net/core/filter.c b/net/core/filter.c
index 5837534f4352bf297a03cd5240361fdcbff101a3..8cae575ad437468e642816d95dcf2035b2e95ba9 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -4153,34 +4153,45 @@ static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset)
        return 0;
 }
 
-static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink,
-                                  enum xdp_mem_type mem_type, bool release)
+static struct xdp_buff *bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink,
+                                              bool tail, bool release)
 {
-       struct xdp_buff *zc_frag = xsk_buff_get_tail(xdp);
+       struct xdp_buff *zc_frag = tail ? xsk_buff_get_tail(xdp) :
+                                         xsk_buff_get_head(xdp);
 
        if (release) {
-               xsk_buff_del_tail(zc_frag);
-               __xdp_return(0, mem_type, false, zc_frag);
+               xsk_buff_del_frag(zc_frag);
        } else {
-               zc_frag->data_end -= shrink;
+               if (tail)
+                       zc_frag->data_end -= shrink;
+               else
+                       zc_frag->data += shrink;
        }
+
+       return zc_frag;
 }
 
 static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag,
-                               int shrink)
+                               int shrink, bool tail)
 {
        enum xdp_mem_type mem_type = xdp->rxq->mem.type;
        bool release = skb_frag_size(frag) == shrink;
+       netmem_ref netmem = skb_frag_netmem(frag);
+       struct xdp_buff *zc_frag = NULL;
 
        if (mem_type == MEM_TYPE_XSK_BUFF_POOL) {
-               bpf_xdp_shrink_data_zc(xdp, shrink, mem_type, release);
-               goto out;
+               netmem = 0;
+               zc_frag = bpf_xdp_shrink_data_zc(xdp, shrink, tail, release);
        }
 
-       if (release)
-               __xdp_return(skb_frag_netmem(frag), mem_type, false, NULL);
+       if (release) {
+               __xdp_return(netmem, mem_type, false, zc_frag);
+       } else {
+               if (!tail)
+                       skb_frag_off_add(frag, shrink);
+               skb_frag_size_sub(frag, shrink);
+       }
 
-out:
        return release;
 }
 
@@ -4198,12 +4209,8 @@ static int bpf_xdp_frags_shrink_tail(struct xdp_buff *xdp, int offset)
 
                len_free += shrink;
                offset -= shrink;
-               if (bpf_xdp_shrink_data(xdp, frag, shrink)) {
+               if (bpf_xdp_shrink_data(xdp, frag, shrink, true))
                        n_frags_free++;
-               } else {
-                       skb_frag_size_sub(frag, shrink);
-                       break;
-               }
        }
        sinfo->nr_frags -= n_frags_free;
        sinfo->xdp_frags_size -= len_free;