xdp: add generic xdp_buff_add_frag()

The code piece which would attach a frag to &xdp_buff is almost
identical across the drivers supporting XDP multi-buffer on Rx.
Make it a generic elegant "oneliner".
Also, I see lots of drivers calculating frags_truesize as
`xdp->frame_sz * nr_frags`. I can't say this is fully correct, since
frags might be backed by chunks of different sizes, especially with
stuff like the header split. Even page_pool_alloc() can give you two
different truesizes on two subsequent requests to allocate the same
buffer size. Add a field to &skb_shared_info (unionized as there's no
free slot currently on x86_64) to track the "true" truesize. It can
be used later when updating the skb.

Reviewed-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Link: https://patch.msgid.link/20241218174435.1445282-3-aleksander.lobakin@intel.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index b2509cd0b930688d9ae94bc4e37a7549797e608f..bb2b751d274acff931281a72e8b4b0c699b4e8af 100644 (file)
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -608,11 +608,19 @@ struct skb_shared_info {
         * Warning : all fields before dataref are cleared in __alloc_skb()
         */
        atomic_t        dataref;
-       unsigned int    xdp_frags_size;
 
-       /* Intermediate layers must ensure that destructor_arg
-        * remains valid until skb destructor */
-       void *          destructor_arg;
+       union {
+               struct {
+                       u32             xdp_frags_size;
+                       u32             xdp_frags_truesize;
+               };
+
+               /*
+                * Intermediate layers must ensure that destructor_arg
+                * remains valid until skb destructor.
+                */
+               void            *destructor_arg;
+       };
 
        /* must be last field, see pskb_expand_head() */
        skb_frag_t      frags[MAX_SKB_FRAGS];

diff --git a/include/net/xdp.h b/include/net/xdp.h
index d2089cfecefd5f102e1c7e32c2d0e95837530cae..11139c210b498a9a69b0db1deb96d7ceac79873b 100644 (file)
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -167,6 +167,93 @@ out:
        return len;
 }
 
+void xdp_return_frag(netmem_ref netmem, const struct xdp_buff *xdp);
+
+/**
+ * __xdp_buff_add_frag - attach frag to &xdp_buff
+ * @xdp: XDP buffer to attach the frag to
+ * @netmem: network memory containing the frag
+ * @offset: offset at which the frag starts
+ * @size: size of the frag
+ * @truesize: total memory size occupied by the frag
+ * @try_coalesce: whether to try coalescing the frags (not valid for XSk)
+ *
+ * Attach frag to the XDP buffer. If it currently has no frags attached,
+ * initialize the related fields, otherwise check that the frag number
+ * didn't reach the limit of ``MAX_SKB_FRAGS``. If possible, try coalescing
+ * the frag with the previous one.
+ * The function doesn't check/update the pfmemalloc bit. Please use the
+ * non-underscored wrapper in drivers.
+ *
+ * Return: true on success, false if there's no space for the frag in
+ * the shared info struct.
+ */
+static inline bool __xdp_buff_add_frag(struct xdp_buff *xdp, netmem_ref netmem,
+                                      u32 offset, u32 size, u32 truesize,
+                                      bool try_coalesce)
+{
+       struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
+       skb_frag_t *prev;
+       u32 nr_frags;
+
+       if (!xdp_buff_has_frags(xdp)) {
+               xdp_buff_set_frags_flag(xdp);
+
+               nr_frags = 0;
+               sinfo->xdp_frags_size = 0;
+               sinfo->xdp_frags_truesize = 0;
+
+               goto fill;
+       }
+
+       nr_frags = sinfo->nr_frags;
+       prev = &sinfo->frags[nr_frags - 1];
+
+       if (try_coalesce && netmem == skb_frag_netmem(prev) &&
+           offset == skb_frag_off(prev) + skb_frag_size(prev)) {
+               skb_frag_size_add(prev, size);
+               /* Guaranteed to only decrement the refcount */
+               xdp_return_frag(netmem, xdp);
+       } else if (unlikely(nr_frags == MAX_SKB_FRAGS)) {
+               return false;
+       } else {
+fill:
+               __skb_fill_netmem_desc_noacc(sinfo, nr_frags++, netmem,
+                                            offset, size);
+       }
+
+       sinfo->nr_frags = nr_frags;
+       sinfo->xdp_frags_size += size;
+       sinfo->xdp_frags_truesize += truesize;
+
+       return true;
+}
+
+/**
+ * xdp_buff_add_frag - attach frag to &xdp_buff
+ * @xdp: XDP buffer to attach the frag to
+ * @netmem: network memory containing the frag
+ * @offset: offset at which the frag starts
+ * @size: size of the frag
+ * @truesize: total memory size occupied by the frag
+ *
+ * Version of __xdp_buff_add_frag() which takes care of the pfmemalloc bit.
+ *
+ * Return: true on success, false if there's no space for the frag in
+ * the shared info struct.
+ */
+static inline bool xdp_buff_add_frag(struct xdp_buff *xdp, netmem_ref netmem,
+                                    u32 offset, u32 size, u32 truesize)
+{
+       if (!__xdp_buff_add_frag(xdp, netmem, offset, size, truesize, true))
+               return false;
+
+       if (unlikely(netmem_is_pfmemalloc(netmem)))
+               xdp_buff_set_frag_pfmemalloc(xdp);
+
+       return true;
+}
+
 struct xdp_frame {
        void *data;
        u32 len;
@@ -230,7 +317,14 @@ xdp_update_skb_shared_info(struct sk_buff *skb, u8 nr_frags,
                           unsigned int size, unsigned int truesize,
                           bool pfmemalloc)
 {
-       skb_shinfo(skb)->nr_frags = nr_frags;
+       struct skb_shared_info *sinfo = skb_shinfo(skb);
+
+       sinfo->nr_frags = nr_frags;
+       /*
+        * ``destructor_arg`` is unionized with ``xdp_frags_{,true}size``,
+        * reset it after that these fields aren't used anymore.
+        */
+       sinfo->destructor_arg = NULL;
 
        skb->len += size;
        skb->data_len += size;

diff --git a/net/core/xdp.c b/net/core/xdp.c
index f1165a35411bcae72c5a78d5f5480ac0e8c69bbe..a66a4e036f53fd5ade48a2ae43cb1a127cc47adf 100644 (file)
--- a/net/core/xdp.c
+++ b/net/core/xdp.c
@@ -535,6 +535,17 @@ void xdp_return_frame_bulk(struct xdp_frame *xdpf,
 }
 EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);
 
+/**
+ * xdp_return_frag -- free one XDP frag or decrement its refcount
+ * @netmem: network memory reference to release
+ * @xdp: &xdp_buff to release the frag for
+ */
+void xdp_return_frag(netmem_ref netmem, const struct xdp_buff *xdp)
+{
+       __xdp_return(netmem, xdp->rxq->mem.type, true, NULL);
+}
+EXPORT_SYMBOL_GPL(xdp_return_frag);
+
 void xdp_return_buff(struct xdp_buff *xdp)
 {
        struct skb_shared_info *sinfo;