xsk: wrap generic metadata handling onto separate function

xsk_build_skb() has gone wild with its size and one of the things we can
do about it is to pull out a branch that takes care of metadata handling
and make it a separate function.

While at it, let us add metadata SW support for devices supporting
IFF_TX_SKB_NO_LINEAR flag, that happen to have separate logic for
building skb in xsk's generic xmit path.

Acked-by: Stanislav Fomichev <sdf@fomichev.me>
Reviewed-by: Jason Xing <kerneljasonxing@gmail.com>
Signed-off-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
Acked-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://patch.msgid.link/20250925160009.2474816-4-maciej.fijalkowski@intel.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index f7e0d254a7237c175d2d3df4a5c597e46498ea72..7b0c68a70888cda896de1b832ddded11dfcaa344 100644 (file)
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -657,6 +657,45 @@ static void xsk_drop_skb(struct sk_buff *skb)
        xsk_consume_skb(skb);
 }
 
+static int xsk_skb_metadata(struct sk_buff *skb, void *buffer,
+                           struct xdp_desc *desc, struct xsk_buff_pool *pool,
+                           u32 hr)
+{
+       struct xsk_tx_metadata *meta = NULL;
+
+       if (unlikely(pool->tx_metadata_len == 0))
+               return -EINVAL;
+
+       meta = buffer - pool->tx_metadata_len;
+       if (unlikely(!xsk_buff_valid_tx_metadata(meta)))
+               return -EINVAL;
+
+       if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) {
+               if (unlikely(meta->request.csum_start +
+                            meta->request.csum_offset +
+                            sizeof(__sum16) > desc->len))
+                       return -EINVAL;
+
+               skb->csum_start = hr + meta->request.csum_start;
+               skb->csum_offset = meta->request.csum_offset;
+               skb->ip_summed = CHECKSUM_PARTIAL;
+
+               if (unlikely(pool->tx_sw_csum)) {
+                       int err;
+
+                       err = skb_checksum_help(skb);
+                       if (err)
+                               return err;
+               }
+       }
+
+       if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
+               skb->skb_mstamp_ns = meta->request.launch_time;
+       xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta);
+
+       return 0;
+}
+
 static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
                                              struct xdp_desc *desc)
 {
@@ -669,6 +708,9 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
        int err, i;
        u64 addr;
 
+       addr = desc->addr;
+       buffer = xsk_buff_raw_get_data(pool, addr);
+
        if (!skb) {
                hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(xs->dev->needed_headroom));
 
@@ -679,6 +721,11 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
                skb_reserve(skb, hr);
 
                xsk_skb_init_misc(skb, xs, desc->addr);
+               if (desc->options & XDP_TX_METADATA) {
+                       err = xsk_skb_metadata(skb, buffer, desc, pool, hr);
+                       if (unlikely(err))
+                               return ERR_PTR(err);
+               }
        } else {
                xsk_addr = kmem_cache_zalloc(xsk_tx_generic_cache, GFP_KERNEL);
                if (!xsk_addr)
@@ -692,11 +739,9 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
                list_add_tail(&xsk_addr->addr_node, &XSKCB(skb)->addrs_list);
        }
 
-       addr = desc->addr;
        len = desc->len;
        ts = pool->unaligned ? len : pool->chunk_size;
 
-       buffer = xsk_buff_raw_get_data(pool, addr);
        offset = offset_in_page(buffer);
        addr = buffer - pool->addrs;
 
@@ -727,7 +772,6 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
 static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
                                     struct xdp_desc *desc)
 {
-       struct xsk_tx_metadata *meta = NULL;
        struct net_device *dev = xs->dev;
        struct sk_buff *skb = xs->skb;
        int err;
@@ -761,6 +805,12 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
                                goto free_err;
 
                        xsk_skb_init_misc(skb, xs, desc->addr);
+                       if (desc->options & XDP_TX_METADATA) {
+                               err = xsk_skb_metadata(skb, buffer, desc,
+                                                      xs->pool, hr);
+                               if (unlikely(err))
+                                       goto free_err;
+                       }
                } else {
                        int nr_frags = skb_shinfo(skb)->nr_frags;
                        struct xsk_addr_node *xsk_addr;
@@ -795,42 +845,6 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
                        xsk_addr->addr = desc->addr;
                        list_add_tail(&xsk_addr->addr_node, &XSKCB(skb)->addrs_list);
                }
-
-               if (!skb_shinfo(skb)->nr_frags && desc->options & XDP_TX_METADATA) {
-                       if (unlikely(xs->pool->tx_metadata_len == 0)) {
-                               err = -EINVAL;
-                               goto free_err;
-                       }
-
-                       meta = buffer - xs->pool->tx_metadata_len;
-                       if (unlikely(!xsk_buff_valid_tx_metadata(meta))) {
-                               err = -EINVAL;
-                               goto free_err;
-                       }
-
-                       if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) {
-                               if (unlikely(meta->request.csum_start +
-                                            meta->request.csum_offset +
-                                            sizeof(__sum16) > len)) {
-                                       err = -EINVAL;
-                                       goto free_err;
-                               }
-
-                               skb->csum_start = hr + meta->request.csum_start;
-                               skb->csum_offset = meta->request.csum_offset;
-                               skb->ip_summed = CHECKSUM_PARTIAL;
-
-                               if (unlikely(xs->pool->tx_sw_csum)) {
-                                       err = skb_checksum_help(skb);
-                                       if (err)
-                                               goto free_err;
-                               }
-                       }
-
-                       if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
-                               skb->skb_mstamp_ns = meta->request.launch_time;
-                       xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta);
-               }
        }
 
        xsk_inc_num_desc(skb);