page_pool: make page_pool_put_page_bulk() handle array of netmems

Currently, page_pool_put_page_bulk() indeed takes an array of pointers
to the data, not pages, despite the name. As one side effect, when
you're freeing frags from &skb_shared_info, xdp_return_frame_bulk()
converts page pointers to virtual addresses and then
page_pool_put_page_bulk() converts them back. Moreover, data pointers
assume every frag is placed in the host memory, making this function
non-universal.
Make page_pool_put_page_bulk() handle array of netmems. Pass frag
netmems directly and use virt_to_netmem() when freeing xdpf->data,
so that the PP core will then get the compound netmem and take care
of the rest.

Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://patch.msgid.link/20241203173733.3181246-9-aleksander.lobakin@intel.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h
index c022c410abe39d588c0933479ab131b3c07391fd..1ea16b0e9c793ad1edeee237c055c5d8d01b2704 100644 (file)
--- a/include/net/page_pool/types.h
+++ b/include/net/page_pool/types.h
@@ -259,8 +259,8 @@ void page_pool_disable_direct_recycling(struct page_pool *pool);
 void page_pool_destroy(struct page_pool *pool);
 void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),
                           const struct xdp_mem_info *mem);
-void page_pool_put_page_bulk(struct page_pool *pool, void **data,
-                            int count);
+void page_pool_put_netmem_bulk(struct page_pool *pool, netmem_ref *data,
+                              u32 count);
 #else
 static inline void page_pool_destroy(struct page_pool *pool)
 {
@@ -272,8 +272,8 @@ static inline void page_pool_use_xdp_mem(struct page_pool *pool,
 {
 }
 
-static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data,
-                                          int count)
+static inline void page_pool_put_netmem_bulk(struct page_pool *pool,
+                                            netmem_ref *data, u32 count)
 {
 }
 #endif

diff --git a/include/net/xdp.h b/include/net/xdp.h
index 1253fe21ede79d8fe9e86b2245fe09e3c8c569b8..f4020b29122fa8eeef3e493b686b0b68869882ae 100644 (file)
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -194,7 +194,7 @@ xdp_frame_is_frag_pfmemalloc(const struct xdp_frame *frame)
 struct xdp_frame_bulk {
        int count;
        void *xa;
-       void *q[XDP_BULK_QUEUE_SIZE];
+       netmem_ref q[XDP_BULK_QUEUE_SIZE];
 };
 
 static __always_inline void xdp_frame_bulk_init(struct xdp_frame_bulk *bq)

diff --git a/net/core/page_pool.c b/net/core/page_pool.c
index f89cf93f6eb45a95f0a47d1592dde44caa78e711..4c85b77cfdac93eb2c364acf25bf14d6979678e5 100644 (file)
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
@@ -840,22 +840,22 @@ void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
 EXPORT_SYMBOL(page_pool_put_unrefed_page);
 
 /**
- * page_pool_put_page_bulk() - release references on multiple pages
+ * page_pool_put_netmem_bulk() - release references on multiple netmems
  * @pool:      pool from which pages were allocated
- * @data:      array holding page pointers
- * @count:     number of pages in @data
+ * @data:      array holding netmem references
+ * @count:     number of entries in @data
  *
- * Tries to refill a number of pages into the ptr_ring cache holding ptr_ring
- * producer lock. If the ptr_ring is full, page_pool_put_page_bulk()
- * will release leftover pages to the page allocator.
- * page_pool_put_page_bulk() is suitable to be run inside the driver NAPI tx
+ * Tries to refill a number of netmems into the ptr_ring cache holding ptr_ring
+ * producer lock. If the ptr_ring is full, page_pool_put_netmem_bulk()
+ * will release leftover netmems to the memory provider.
+ * page_pool_put_netmem_bulk() is suitable to be run inside the driver NAPI tx
  * completion loop for the XDP_REDIRECT use case.
  *
  * Please note the caller must not use data area after running
- * page_pool_put_page_bulk(), as this function overwrites it.
+ * page_pool_put_netmem_bulk(), as this function overwrites it.
  */
-void page_pool_put_page_bulk(struct page_pool *pool, void **data,
-                            int count)
+void page_pool_put_netmem_bulk(struct page_pool *pool, netmem_ref *data,
+                              u32 count)
 {
        int i, bulk_len = 0;
        bool allow_direct;
@@ -864,7 +864,7 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
        allow_direct = page_pool_napi_local(pool);
 
        for (i = 0; i < count; i++) {
-               netmem_ref netmem = page_to_netmem(virt_to_head_page(data[i]));
+               netmem_ref netmem = netmem_compound_head(data[i]);
 
                /* It is not the last user for the page frag case */
                if (!page_pool_is_last_ref(netmem))
@@ -873,7 +873,7 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
                netmem = __page_pool_put_page(pool, netmem, -1, allow_direct);
                /* Approved for bulk recycling in ptr_ring cache */
                if (netmem)
-                       data[bulk_len++] = (__force void *)netmem;
+                       data[bulk_len++] = netmem;
        }
 
        if (!bulk_len)
@@ -882,7 +882,7 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
        /* Bulk producer into ptr_ring page_pool cache */
        in_softirq = page_pool_producer_lock(pool);
        for (i = 0; i < bulk_len; i++) {
-               if (__ptr_ring_produce(&pool->ring, data[i])) {
+               if (__ptr_ring_produce(&pool->ring, (__force void *)data[i])) {
                        /* ring full */
                        recycle_stat_inc(pool, ring_full);
                        break;
@@ -899,9 +899,9 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
         * since put_page() with refcnt == 1 can be an expensive operation
         */
        for (; i < bulk_len; i++)
-               page_pool_return_page(pool, (__force netmem_ref)data[i]);
+               page_pool_return_page(pool, data[i]);
 }
-EXPORT_SYMBOL(page_pool_put_page_bulk);
+EXPORT_SYMBOL(page_pool_put_netmem_bulk);
 
 static netmem_ref page_pool_drain_frag(struct page_pool *pool,
                                       netmem_ref netmem)

diff --git a/net/core/xdp.c b/net/core/xdp.c
index de1e9cb78718131202d1a89e0352432a4a8d182d..938ad15c985778c6389ac232d8e62be67051c45c 100644 (file)
--- a/net/core/xdp.c
+++ b/net/core/xdp.c
@@ -518,7 +518,7 @@ void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq)
        if (unlikely(!xa || !bq->count))
                return;
 
-       page_pool_put_page_bulk(xa->page_pool, bq->q, bq->count);
+       page_pool_put_netmem_bulk(xa->page_pool, bq->q, bq->count);
        /* bq->xa is not cleared to save lookup, if mem.id same in next bulk */
        bq->count = 0;
 }
@@ -559,12 +559,12 @@ void xdp_return_frame_bulk(struct xdp_frame *xdpf,
                for (i = 0; i < sinfo->nr_frags; i++) {
                        skb_frag_t *frag = &sinfo->frags[i];
 
-                       bq->q[bq->count++] = skb_frag_address(frag);
+                       bq->q[bq->count++] = skb_frag_netmem(frag);
                        if (bq->count == XDP_BULK_QUEUE_SIZE)
                                xdp_flush_frame_bulk(bq);
                }
        }
-       bq->q[bq->count++] = xdpf->data;
+       bq->q[bq->count++] = virt_to_netmem(xdpf->data);
 }
 EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);