net/mlx5e: SHAMPO, Fix header formulas for higher MTUs and 64K pages

The MLX5E_SHAMPO_WQ_HEADER_PER_PAGE and
MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE macros are used directly in
several places under the assumption that there will always be more
headers per WQE than headers per page. However, this assumption doesn't
hold for 64K page sizes and higher MTUs (> 4K). This can be first
observed during header page allocation: ksm_entries will become 0 during
alignment to MLX5E_SHAMPO_WQ_HEADER_PER_PAGE.

This patch introduces 2 additional members to the mlx5e_shampo_hd struct
which are meant to be used instead of the macrose mentioned above.
When the number of headers per WQE goes below
MLX5E_SHAMPO_WQ_HEADER_PER_PAGE, clamp the number of headers per
page and expand the header size accordingly so that the headers
for one WQE cover a full page.

All the formulas are adapted to use these two new members.

Fixes: 945ca432bfd0 ("net/mlx5e: SHAMPO, Drop info array")
Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
Reviewed-by: Simon Horman <horms@kernel.org>
Link: https://patch.msgid.link/1762238915-1027590-4-git-send-email-tariqt@nvidia.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 14e3207b14e745be9beb853151c428b399ff6273..a163f81f07c13d1b2cd9790eb5670a246b3e80a1 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -634,7 +634,10 @@ struct mlx5e_dma_info {
 struct mlx5e_shampo_hd {
        struct mlx5e_frag_page *pages;
        u32 hd_per_wq;
+       u32 hd_per_page;
        u16 hd_per_wqe;
+       u8 log_hd_per_page;
+       u8 log_hd_entry_size;
        unsigned long *bitmap;
        u16 pi;
        u16 ci;

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 9c46511e7b43766ccc47d78dc87aee328d37bfbc..6023bbbf3f39cc40b086aeb3c9f7ef1d751dbec1 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -791,8 +791,9 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
                                int node)
 {
        void *wqc = MLX5_ADDR_OF(rqc, rqp->rqc, wq);
+       u8 log_hd_per_page, log_hd_entry_size;
+       u16 hd_per_wq, hd_per_wqe;
        u32 hd_pool_size;
-       u16 hd_per_wq;
        int wq_size;
        int err;
 
@@ -815,11 +816,24 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
        if (err)
                goto err_umr_mkey;
 
-       rq->mpwqe.shampo->hd_per_wqe =
-               mlx5e_shampo_hd_per_wqe(mdev, params, rqp);
+       hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rqp);
        wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz));
-       hd_pool_size = (rq->mpwqe.shampo->hd_per_wqe * wq_size) /
-               MLX5E_SHAMPO_WQ_HEADER_PER_PAGE;
+
+       BUILD_BUG_ON(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE > PAGE_SHIFT);
+       if (hd_per_wqe >= MLX5E_SHAMPO_WQ_HEADER_PER_PAGE) {
+               log_hd_per_page = MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE;
+               log_hd_entry_size = MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE;
+       } else {
+               log_hd_per_page = order_base_2(hd_per_wqe);
+               log_hd_entry_size = order_base_2(PAGE_SIZE / hd_per_wqe);
+       }
+
+       rq->mpwqe.shampo->hd_per_wqe = hd_per_wqe;
+       rq->mpwqe.shampo->hd_per_page = BIT(log_hd_per_page);
+       rq->mpwqe.shampo->log_hd_per_page = log_hd_per_page;
+       rq->mpwqe.shampo->log_hd_entry_size = log_hd_entry_size;
+
+       hd_pool_size = (hd_per_wqe * wq_size) >> log_hd_per_page;
 
        if (netif_rxq_has_unreadable_mp(rq->netdev, rq->ix)) {
                /* Separate page pool for shampo headers */

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index f2a06752ce37517278de9c340b39475ca6ff8ca2..687cf123211d7553b5304b3cd764f82d4c075ae9 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -648,17 +648,20 @@ static void build_ksm_umr(struct mlx5e_icosq *sq, struct mlx5e_umr_wqe *umr_wqe,
        umr_wqe->hdr.uctrl.mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
 }
 
-static struct mlx5e_frag_page *mlx5e_shampo_hd_to_frag_page(struct mlx5e_rq *rq, int header_index)
+static struct mlx5e_frag_page *mlx5e_shampo_hd_to_frag_page(struct mlx5e_rq *rq,
+                                                           int header_index)
 {
-       BUILD_BUG_ON(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE > PAGE_SHIFT);
+       struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
 
-       return &rq->mpwqe.shampo->pages[header_index >> MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE];
+       return &shampo->pages[header_index >> shampo->log_hd_per_page];
 }
 
-static u64 mlx5e_shampo_hd_offset(int header_index)
+static u64 mlx5e_shampo_hd_offset(struct mlx5e_rq *rq, int header_index)
 {
-       return (header_index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) <<
-               MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE;
+       struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
+       u32 hd_per_page = shampo->hd_per_page;
+
+       return (header_index & (hd_per_page - 1)) << shampo->log_hd_entry_size;
 }
 
 static void mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index);
@@ -684,7 +687,7 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq,
                u64 addr;
 
                frag_page = mlx5e_shampo_hd_to_frag_page(rq, index);
-               header_offset = mlx5e_shampo_hd_offset(index);
+               header_offset = mlx5e_shampo_hd_offset(rq, index);
                if (!header_offset) {
                        err = mlx5e_page_alloc_fragmented(rq->hd_page_pool,
                                                          frag_page);
@@ -714,7 +717,7 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq,
 err_unmap:
        while (--i >= 0) {
                --index;
-               header_offset = mlx5e_shampo_hd_offset(index);
+               header_offset = mlx5e_shampo_hd_offset(rq, index);
                if (!header_offset) {
                        struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, index);
 
@@ -738,7 +741,7 @@ static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq)
        ksm_entries = bitmap_find_window(shampo->bitmap,
                                         shampo->hd_per_wqe,
                                         shampo->hd_per_wq, shampo->pi);
-       ksm_entries = ALIGN_DOWN(ksm_entries, MLX5E_SHAMPO_WQ_HEADER_PER_PAGE);
+       ksm_entries = ALIGN_DOWN(ksm_entries, shampo->hd_per_page);
        if (!ksm_entries)
                return 0;
 
@@ -856,7 +859,7 @@ mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index)
 {
        struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
 
-       if (((header_index + 1) & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) == 0) {
+       if (((header_index + 1) & (shampo->hd_per_page - 1)) == 0) {
                struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index);
 
                mlx5e_page_release_fragmented(rq->hd_page_pool, frag_page);
@@ -1223,9 +1226,10 @@ static unsigned int mlx5e_lro_update_hdr(struct sk_buff *skb,
 static void *mlx5e_shampo_get_packet_hd(struct mlx5e_rq *rq, u16 header_index)
 {
        struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index);
-       u16 head_offset = mlx5e_shampo_hd_offset(header_index) + rq->buff.headroom;
+       u16 head_offset = mlx5e_shampo_hd_offset(rq, header_index);
+       void *addr = netmem_address(frag_page->netmem);
 
-       return netmem_address(frag_page->netmem) + head_offset;
+       return addr + head_offset + rq->buff.headroom;
 }
 
 static void mlx5e_shampo_update_ipv4_udp_hdr(struct mlx5e_rq *rq, struct iphdr *ipv4)
@@ -2265,7 +2269,8 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
                          struct mlx5_cqe64 *cqe, u16 header_index)
 {
        struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index);
-       u16 head_offset = mlx5e_shampo_hd_offset(header_index);
+       u16 head_offset = mlx5e_shampo_hd_offset(rq, header_index);
+       struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
        u16 head_size = cqe->shampo.header_size;
        u16 rx_headroom = rq->buff.headroom;
        struct sk_buff *skb = NULL;
@@ -2281,7 +2286,7 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
        data            = hdr + rx_headroom;
        frag_size       = MLX5_SKB_FRAG_SZ(rx_headroom + head_size);
 
-       if (likely(frag_size <= BIT(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE))) {
+       if (likely(frag_size <= BIT(shampo->log_hd_entry_size))) {
                /* build SKB around header */
                dma_sync_single_range_for_cpu(rq->pdev, dma_addr, 0, frag_size, rq->buff.map_dir);
                net_prefetchw(hdr);