]> git.ipfire.org Git - thirdparty/linux.git/commitdiff
libeth: convert to netmem
authorAlexander Lobakin <aleksander.lobakin@intel.com>
Thu, 12 Jun 2025 16:02:19 +0000 (18:02 +0200)
committerTony Nguyen <anthony.l.nguyen@intel.com>
Mon, 16 Jun 2025 18:40:14 +0000 (11:40 -0700)
Back when the libeth Rx core was initially written, devmem was a draft
and netmem_ref didn't exist in the mainline. Now that it's here, make
libeth MP-agnostic before introducing any new code or any new library
users.
When it's known that the created PP/FQ is for header buffers, use faster
"unsafe" underscored netmem <--> virt accessors as netmem_is_net_iov()
is always false in that case, but consumes some cycles (bit test +
true branch).

Reviewed-by: Mina Almasry <almasrymina@google.com>
Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Signed-off-by: Tony Nguyen <anthony.l.nguyen@intel.com>
drivers/net/ethernet/intel/iavf/iavf_txrx.c
drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c
drivers/net/ethernet/intel/idpf/idpf_txrx.c
drivers/net/ethernet/intel/libeth/rx.c
include/net/libeth/rx.h

index 23e786b9793d3394ad56fdf27b8947afbae71a87..aaf70c6256556285cd0dd52413115069aa736de4 100644 (file)
@@ -723,7 +723,7 @@ static void iavf_clean_rx_ring(struct iavf_ring *rx_ring)
        for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) {
                const struct libeth_fqe *rx_fqes = &rx_ring->rx_fqes[i];
 
-               page_pool_put_full_page(rx_ring->pp, rx_fqes->page, false);
+               libeth_rx_recycle_slow(rx_fqes->netmem);
 
                if (unlikely(++i == rx_ring->count))
                        i = 0;
@@ -1197,10 +1197,11 @@ static void iavf_add_rx_frag(struct sk_buff *skb,
                             const struct libeth_fqe *rx_buffer,
                             unsigned int size)
 {
-       u32 hr = rx_buffer->page->pp->p.offset;
+       u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset;
 
-       skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
-                       rx_buffer->offset + hr, size, rx_buffer->truesize);
+       skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags,
+                              rx_buffer->netmem, rx_buffer->offset + hr,
+                              size, rx_buffer->truesize);
 }
 
 /**
@@ -1214,12 +1215,13 @@ static void iavf_add_rx_frag(struct sk_buff *skb,
 static struct sk_buff *iavf_build_skb(const struct libeth_fqe *rx_buffer,
                                      unsigned int size)
 {
-       u32 hr = rx_buffer->page->pp->p.offset;
+       struct page *buf_page = __netmem_to_page(rx_buffer->netmem);
+       u32 hr = buf_page->pp->p.offset;
        struct sk_buff *skb;
        void *va;
 
        /* prefetch first cache line of first page */
-       va = page_address(rx_buffer->page) + rx_buffer->offset;
+       va = page_address(buf_page) + rx_buffer->offset;
        net_prefetch(va + hr);
 
        /* build an skb around the page buffer */
index 993c354aa27adf7fa4f35295dd72a87b103bd2ab..555879b1248d8474d85a0412738b9b4d80f96d18 100644 (file)
@@ -1006,7 +1006,7 @@ static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget)
                        break;
 
 skip_data:
-               rx_buf->page = NULL;
+               rx_buf->netmem = 0;
 
                IDPF_SINGLEQ_BUMP_RING_IDX(rx_q, ntc);
                cleaned_count++;
index 5cf440e09d0a6701633e91a9f46b07ba1eff5bfb..cef9dfb877e8d675f9f5abf6df64e18b825d121c 100644 (file)
@@ -383,12 +383,12 @@ err_out:
  */
 static void idpf_rx_page_rel(struct libeth_fqe *rx_buf)
 {
-       if (unlikely(!rx_buf->page))
+       if (unlikely(!rx_buf->netmem))
                return;
 
-       page_pool_put_full_page(rx_buf->page->pp, rx_buf->page, false);
+       libeth_rx_recycle_slow(rx_buf->netmem);
 
-       rx_buf->page = NULL;
+       rx_buf->netmem = 0;
        rx_buf->offset = 0;
 }
 
@@ -3240,10 +3240,10 @@ idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb,
 void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb,
                      unsigned int size)
 {
-       u32 hr = rx_buf->page->pp->p.offset;
+       u32 hr = netmem_get_pp(rx_buf->netmem)->p.offset;
 
-       skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buf->page,
-                       rx_buf->offset + hr, size, rx_buf->truesize);
+       skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, rx_buf->netmem,
+                              rx_buf->offset + hr, size, rx_buf->truesize);
 }
 
 /**
@@ -3266,16 +3266,20 @@ static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr,
                             struct libeth_fqe *buf, u32 data_len)
 {
        u32 copy = data_len <= L1_CACHE_BYTES ? data_len : ETH_HLEN;
+       struct page *hdr_page, *buf_page;
        const void *src;
        void *dst;
 
-       if (!libeth_rx_sync_for_cpu(buf, copy))
+       if (unlikely(netmem_is_net_iov(buf->netmem)) ||
+           !libeth_rx_sync_for_cpu(buf, copy))
                return 0;
 
-       dst = page_address(hdr->page) + hdr->offset + hdr->page->pp->p.offset;
-       src = page_address(buf->page) + buf->offset + buf->page->pp->p.offset;
-       memcpy(dst, src, LARGEST_ALIGN(copy));
+       hdr_page = __netmem_to_page(hdr->netmem);
+       buf_page = __netmem_to_page(buf->netmem);
+       dst = page_address(hdr_page) + hdr->offset + hdr_page->pp->p.offset;
+       src = page_address(buf_page) + buf->offset + buf_page->pp->p.offset;
 
+       memcpy(dst, src, LARGEST_ALIGN(copy));
        buf->offset += copy;
 
        return copy;
@@ -3291,11 +3295,12 @@ static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr,
  */
 struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size)
 {
-       u32 hr = buf->page->pp->p.offset;
+       struct page *buf_page = __netmem_to_page(buf->netmem);
+       u32 hr = buf_page->pp->p.offset;
        struct sk_buff *skb;
        void *va;
 
-       va = page_address(buf->page) + buf->offset;
+       va = page_address(buf_page) + buf->offset;
        prefetch(va + hr);
 
        skb = napi_build_skb(va, buf->truesize);
@@ -3429,7 +3434,8 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget)
 
                if (unlikely(!hdr_len && !skb)) {
                        hdr_len = idpf_rx_hsplit_wa(hdr, rx_buf, pkt_len);
-                       pkt_len -= hdr_len;
+                       /* If failed, drop both buffers by setting len to 0 */
+                       pkt_len -= hdr_len ? : pkt_len;
 
                        u64_stats_update_begin(&rxq->stats_sync);
                        u64_stats_inc(&rxq->q_stats.hsplit_buf_ovf);
@@ -3446,7 +3452,7 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget)
                        u64_stats_update_end(&rxq->stats_sync);
                }
 
-               hdr->page = NULL;
+               hdr->netmem = 0;
 
 payload:
                if (!libeth_rx_sync_for_cpu(rx_buf, pkt_len))
@@ -3462,7 +3468,7 @@ payload:
                        break;
 
 skip_data:
-               rx_buf->page = NULL;
+               rx_buf->netmem = 0;
 
                idpf_rx_post_buf_refill(refillq, buf_id);
                IDPF_RX_BUMP_NTC(rxq, ntc);
index c2c53552c4403ba1081c96d9a3d86dcfb4170708..2afa6e33f1609ab25859c9f3e4393689af4c4f6f 100644 (file)
@@ -204,14 +204,14 @@ void libeth_rx_fq_destroy(struct libeth_fq *fq)
 EXPORT_SYMBOL_GPL(libeth_rx_fq_destroy);
 
 /**
- * libeth_rx_recycle_slow - recycle a libeth page from the NAPI context
- * @page: page to recycle
+ * libeth_rx_recycle_slow - recycle libeth netmem
+ * @netmem: network memory to recycle
  *
  * To be used on exceptions or rare cases not requiring fast inline recycling.
  */
-void libeth_rx_recycle_slow(struct page *page)
+void __cold libeth_rx_recycle_slow(netmem_ref netmem)
 {
-       page_pool_recycle_direct(page->pp, page);
+       page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false);
 }
 EXPORT_SYMBOL_GPL(libeth_rx_recycle_slow);
 
index ab05024be5186c0ae4812e45458698bcbead997d..7d5dc58984b1840e5fababbe5e5b5399b64229a3 100644 (file)
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (C) 2024 Intel Corporation */
+/* Copyright (C) 2024-2025 Intel Corporation */
 
 #ifndef __LIBETH_RX_H
 #define __LIBETH_RX_H
@@ -31,7 +31,7 @@
 
 /**
  * struct libeth_fqe - structure representing an Rx buffer (fill queue element)
- * @page: page holding the buffer
+ * @netmem: network memory reference holding the buffer
  * @offset: offset from the page start (to the headroom)
  * @truesize: total space occupied by the buffer (w/ headroom and tailroom)
  *
@@ -40,7 +40,7 @@
  * former, @offset is always 0 and @truesize is always ```PAGE_SIZE```.
  */
 struct libeth_fqe {
-       struct page             *page;
+       netmem_ref              netmem;
        u32                     offset;
        u32                     truesize;
 } __aligned_largest;
@@ -102,15 +102,16 @@ static inline dma_addr_t libeth_rx_alloc(const struct libeth_fq_fp *fq, u32 i)
        struct libeth_fqe *buf = &fq->fqes[i];
 
        buf->truesize = fq->truesize;
-       buf->page = page_pool_dev_alloc(fq->pp, &buf->offset, &buf->truesize);
-       if (unlikely(!buf->page))
+       buf->netmem = page_pool_dev_alloc_netmem(fq->pp, &buf->offset,
+                                                &buf->truesize);
+       if (unlikely(!buf->netmem))
                return DMA_MAPPING_ERROR;
 
-       return page_pool_get_dma_addr(buf->page) + buf->offset +
+       return page_pool_get_dma_addr_netmem(buf->netmem) + buf->offset +
               fq->pp->p.offset;
 }
 
-void libeth_rx_recycle_slow(struct page *page);
+void libeth_rx_recycle_slow(netmem_ref netmem);
 
 /**
  * libeth_rx_sync_for_cpu - synchronize or recycle buffer post DMA
@@ -126,18 +127,19 @@ void libeth_rx_recycle_slow(struct page *page);
 static inline bool libeth_rx_sync_for_cpu(const struct libeth_fqe *fqe,
                                          u32 len)
 {
-       struct page *page = fqe->page;
+       netmem_ref netmem = fqe->netmem;
 
        /* Very rare, but possible case. The most common reason:
         * the last fragment contained FCS only, which was then
         * stripped by the HW.
         */
        if (unlikely(!len)) {
-               libeth_rx_recycle_slow(page);
+               libeth_rx_recycle_slow(netmem);
                return false;
        }
 
-       page_pool_dma_sync_for_cpu(page->pp, page, fqe->offset, len);
+       page_pool_dma_sync_netmem_for_cpu(netmem_get_pp(netmem), netmem,
+                                         fqe->offset, len);
 
        return true;
 }