gve: implement DQO RX datapath and control path for AF_XDP zero-copy

Add the RX datapath for AF_XDP zero-copy for DQ RDA. The RX path is
quite similar to that of the normal XDP case. Parallel methods are
introduced to properly handle XSKs instead of normal driver buffers.

To properly support posting from XSKs, queues are destroyed and
recreated, as the driver was initially making use of page pool buffers
instead of the XSK pool memory.

Expose support for AF_XDP zero-copy, as the TX and RX datapaths both
exist.

Reviewed-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: Praveen Kaligineedi <pkaligineedi@google.com>
Signed-off-by: Joshua Washington <joshwash@google.com>
Signed-off-by: Jeroen de Borst <jeroendb@google.com>
Link: https://patch.msgid.link/20250717152839.973004-6-jeroendb@google.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index ff7dc06e7fa467c2b510dd1793aeac96a1dd8257..bceaf9b05cb422413d2f4eda03e60a147989d30c 100644 (file)
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -190,6 +190,9 @@ struct gve_rx_buf_state_dqo {
        /* The page posted to HW. */
        struct gve_rx_slot_page_info page_info;
 
+       /* XSK buffer */
+       struct xdp_buff *xsk_buff;
+
        /* The DMA address corresponding to `page_info`. */
        dma_addr_t addr;
 

diff --git a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
index 6c3c459a1b5e341b377de54e2c766997523f70d8..8f5021e59e0a9cb5ab5523a8091606711e7c8f0a 100644 (file)
--- a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c
@@ -4,6 +4,7 @@
  * Copyright (C) 2015-2024 Google, Inc.
  */
 
+#include <net/xdp_sock_drv.h>
 #include "gve.h"
 #include "gve_utils.h"
 
@@ -29,6 +30,10 @@ struct gve_rx_buf_state_dqo *gve_alloc_buf_state(struct gve_rx_ring *rx)
        /* Point buf_state to itself to mark it as allocated */
        buf_state->next = buffer_id;
 
+       /* Clear the buffer pointers */
+       buf_state->page_info.page = NULL;
+       buf_state->xsk_buff = NULL;
+
        return buf_state;
 }
 
@@ -286,7 +291,24 @@ int gve_alloc_buffer(struct gve_rx_ring *rx, struct gve_rx_desc_dqo *desc)
 {
        struct gve_rx_buf_state_dqo *buf_state;
 
-       if (rx->dqo.page_pool) {
+       if (rx->xsk_pool) {
+               buf_state = gve_alloc_buf_state(rx);
+               if (unlikely(!buf_state))
+                       return -ENOMEM;
+
+               buf_state->xsk_buff = xsk_buff_alloc(rx->xsk_pool);
+               if (unlikely(!buf_state->xsk_buff)) {
+                       xsk_set_rx_need_wakeup(rx->xsk_pool);
+                       gve_free_buf_state(rx, buf_state);
+                       return -ENOMEM;
+               }
+               /* Allocated xsk buffer. Clear wakeup in case it was set. */
+               xsk_clear_rx_need_wakeup(rx->xsk_pool);
+               desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states);
+               desc->buf_addr =
+                       cpu_to_le64(xsk_buff_xdp_get_dma(buf_state->xsk_buff));
+               return 0;
+       } else if (rx->dqo.page_pool) {
                buf_state = gve_alloc_buf_state(rx);
                if (WARN_ON_ONCE(!buf_state))
                        return -ENOMEM;

diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index c6ccc0bb40c9a3c2b3c265e78a0cd16d220b3bc0..6ea306947417f5ccc924fafa0b5f87a9a94e80d8 100644 (file)
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -1610,13 +1610,24 @@ static int gve_xsk_pool_enable(struct net_device *dev,
                return 0;
 
        err = gve_reg_xsk_pool(priv, dev, pool, qid);
-       if (err) {
-               clear_bit(qid, priv->xsk_pools);
-               xsk_pool_dma_unmap(pool,
-                                  DMA_ATTR_SKIP_CPU_SYNC |
-                                  DMA_ATTR_WEAK_ORDERING);
+       if (err)
+               goto err_xsk_pool_dma_mapped;
+
+       /* Stop and start RDA queues to repost buffers. */
+       if (!gve_is_qpl(priv)) {
+               err = gve_configure_rings_xdp(priv, priv->rx_cfg.num_queues);
+               if (err)
+                       goto err_xsk_pool_registered;
        }
+       return 0;
 
+err_xsk_pool_registered:
+       gve_unreg_xsk_pool(priv, qid);
+err_xsk_pool_dma_mapped:
+       clear_bit(qid, priv->xsk_pools);
+       xsk_pool_dma_unmap(pool,
+                          DMA_ATTR_SKIP_CPU_SYNC |
+                          DMA_ATTR_WEAK_ORDERING);
        return err;
 }
 
@@ -1628,6 +1639,7 @@ static int gve_xsk_pool_disable(struct net_device *dev,
        struct napi_struct *napi_tx;
        struct xsk_buff_pool *pool;
        int tx_qid;
+       int err;
 
        if (qid >= priv->rx_cfg.num_queues)
                return -EINVAL;
@@ -1643,6 +1655,13 @@ static int gve_xsk_pool_disable(struct net_device *dev,
        if (!netif_running(dev) || !priv->tx_cfg.num_xdp_queues)
                return 0;
 
+       /* Stop and start RDA queues to repost buffers. */
+       if (!gve_is_qpl(priv) && priv->xdp_prog) {
+               err = gve_configure_rings_xdp(priv, priv->rx_cfg.num_queues);
+               if (err)
+                       return err;
+       }
+
        napi_rx = &priv->ntfy_blocks[priv->rx[qid].ntfy_id].napi;
        napi_disable(napi_rx); /* make sure current rx poll is done */
 
@@ -1654,12 +1673,14 @@ static int gve_xsk_pool_disable(struct net_device *dev,
        smp_mb(); /* Make sure it is visible to the workers on datapath */
 
        napi_enable(napi_rx);
-       if (gve_rx_work_pending(&priv->rx[qid]))
-               napi_schedule(napi_rx);
-
        napi_enable(napi_tx);
-       if (gve_tx_clean_pending(priv, &priv->tx[tx_qid]))
-               napi_schedule(napi_tx);
+       if (gve_is_gqi(priv)) {
+               if (gve_rx_work_pending(&priv->rx[qid]))
+                       napi_schedule(napi_rx);
+
+               if (gve_tx_clean_pending(priv, &priv->tx[tx_qid]))
+                       napi_schedule(napi_tx);
+       }
 
        return 0;
 }
@@ -2286,6 +2307,7 @@ static void gve_set_netdev_xdp_features(struct gve_priv *priv)
        } else if (priv->queue_format == GVE_DQO_RDA_FORMAT) {
                xdp_features = NETDEV_XDP_ACT_BASIC;
                xdp_features |= NETDEV_XDP_ACT_REDIRECT;
+               xdp_features |= NETDEV_XDP_ACT_XSK_ZEROCOPY;
        } else {
                xdp_features = 0;
        }

diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
index afaa822b12273cf45b937ad82ee4bd45c334a536..7380c2b7a2d85a6ca428ad64d0f95b1b9afe4a6e 100644 (file)
--- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c
@@ -16,6 +16,7 @@
 #include <net/ip6_checksum.h>
 #include <net/ipv6.h>
 #include <net/tcp.h>
+#include <net/xdp_sock_drv.h>
 
 static void gve_rx_free_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx)
 {
@@ -149,6 +150,10 @@ void gve_rx_free_ring_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
                        gve_free_to_page_pool(rx, bs, false);
                else
                        gve_free_qpl_page_dqo(bs);
+               if (gve_buf_state_is_allocated(rx, bs) && bs->xsk_buff) {
+                       xsk_buff_free(bs->xsk_buff);
+                       bs->xsk_buff = NULL;
+               }
        }
 
        if (rx->dqo.qpl) {
@@ -580,8 +585,11 @@ static int gve_xdp_tx_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
        int err;
 
        xdpf = xdp_convert_buff_to_frame(xdp);
-       if (unlikely(!xdpf))
+       if (unlikely(!xdpf)) {
+               if (rx->xsk_pool)
+                       xsk_buff_free(xdp);
                return -ENOSPC;
+       }
 
        tx_qid = gve_xdp_tx_queue_id(priv, rx->q_num);
        tx = &priv->tx[tx_qid];
@@ -592,6 +600,41 @@ static int gve_xdp_tx_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
        return err;
 }
 
+static void gve_xsk_done_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
+                            struct xdp_buff *xdp, struct bpf_prog *xprog,
+                            int xdp_act)
+{
+       switch (xdp_act) {
+       case XDP_ABORTED:
+       case XDP_DROP:
+       default:
+               xsk_buff_free(xdp);
+               break;
+       case XDP_TX:
+               if (unlikely(gve_xdp_tx_dqo(priv, rx, xdp)))
+                       goto err;
+               break;
+       case XDP_REDIRECT:
+               if (unlikely(xdp_do_redirect(priv->dev, xdp, xprog)))
+                       goto err;
+               break;
+       }
+
+       u64_stats_update_begin(&rx->statss);
+       if ((u32)xdp_act < GVE_XDP_ACTIONS)
+               rx->xdp_actions[xdp_act]++;
+       u64_stats_update_end(&rx->statss);
+       return;
+
+err:
+       u64_stats_update_begin(&rx->statss);
+       if (xdp_act == XDP_TX)
+               rx->xdp_tx_errors++;
+       if (xdp_act == XDP_REDIRECT)
+               rx->xdp_redirect_errors++;
+       u64_stats_update_end(&rx->statss);
+}
+
 static void gve_xdp_done_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
                             struct xdp_buff *xdp, struct bpf_prog *xprog,
                             int xdp_act,
@@ -633,6 +676,48 @@ err:
        return;
 }
 
+static int gve_rx_xsk_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
+                         struct gve_rx_buf_state_dqo *buf_state, int buf_len,
+                         struct bpf_prog *xprog)
+{
+       struct xdp_buff *xdp = buf_state->xsk_buff;
+       struct gve_priv *priv = rx->gve;
+       int xdp_act;
+
+       xdp->data_end = xdp->data + buf_len;
+       xsk_buff_dma_sync_for_cpu(xdp);
+
+       if (xprog) {
+               xdp_act = bpf_prog_run_xdp(xprog, xdp);
+               buf_len = xdp->data_end - xdp->data;
+               if (xdp_act != XDP_PASS) {
+                       gve_xsk_done_dqo(priv, rx, xdp, xprog, xdp_act);
+                       gve_free_buf_state(rx, buf_state);
+                       return 0;
+               }
+       }
+
+       /* Copy the data to skb */
+       rx->ctx.skb_head = gve_rx_copy_data(priv->dev, napi,
+                                           xdp->data, buf_len);
+       if (unlikely(!rx->ctx.skb_head)) {
+               xsk_buff_free(xdp);
+               gve_free_buf_state(rx, buf_state);
+               return -ENOMEM;
+       }
+       rx->ctx.skb_tail = rx->ctx.skb_head;
+
+       /* Free XSK buffer and Buffer state */
+       xsk_buff_free(xdp);
+       gve_free_buf_state(rx, buf_state);
+
+       /* Update Stats */
+       u64_stats_update_begin(&rx->statss);
+       rx->xdp_actions[XDP_PASS]++;
+       u64_stats_update_end(&rx->statss);
+       return 0;
+}
+
 /* Returns 0 if descriptor is completed successfully.
  * Returns -EINVAL if descriptor is invalid.
  * Returns -ENOMEM if data cannot be copied to skb.
@@ -671,7 +756,11 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
        buf_len = compl_desc->packet_len;
        hdr_len = compl_desc->header_len;
 
-       /* Page might have not been used for a while and was likely last written
+       xprog = READ_ONCE(priv->xdp_prog);
+       if (buf_state->xsk_buff)
+               return gve_rx_xsk_dqo(napi, rx, buf_state, buf_len, xprog);
+
+       /* Page might have not been used for awhile and was likely last written
         * by a different thread.
         */
        if (rx->dqo.page_pool) {
@@ -721,7 +810,6 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
                return 0;
        }
 
-       xprog = READ_ONCE(priv->xdp_prog);
        if (xprog) {
                struct xdp_buff xdp;
                void *old_data;