net/mlx5: Store the global doorbell in mlx5_priv

The global doorbell is used for more than just Ethernet resources, so
move it out of mlx5e_hw_objs into a common place (mlx5_priv), to avoid
non-Ethernet modules (e.g. HWS, ASO) depending on Ethernet structs.

Use this opportunity to consolidate it with the 'uar' pointer already
there, which was used as an RX doorbell. Underneath the 'uar' pointer is
identical to 'bfreg->up', so store a single resource and use that
instead.

For CQ doorbells, care is taken to always use bfreg->up->index instead
of bfreg->index, which may refer to a subsequent UAR page from the same
ALLOC_UAR batch on some NICs.

This paves the way for cleanly supporting multiple doorbells in the
Ethernet driver.

Signed-off-by: Cosmin Ratiu <cratiu@nvidia.com>
Reviewed-by: Dragos Tatulea <dtatulea@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
Reviewed-by: Simon Horman <horms@kernel.org>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 9c8003a7833433a6bfad2a089f8427d37d92e6d3..a23b364e24ffeb1bcfe983318c01f437cee023a1 100644 (file)
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -648,7 +648,7 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 {
        struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev;
        struct mlx5_ib_cq *cq = to_mcq(ibcq);
-       void __iomem *uar_page = mdev->priv.uar->map;
+       void __iomem *uar_page = mdev->priv.bfreg.up->map;
        unsigned long irq_flags;
        int ret = 0;
 
@@ -923,7 +923,7 @@ static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
                 cq->buf.frag_buf.page_shift -
                 MLX5_ADAPTER_PAGE_SHIFT);
 
-       *index = dev->mdev->priv.uar->index;
+       *index = dev->mdev->priv.bfreg.up->index;
 
        return 0;
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cq.c b/drivers/net/ethernet/mellanox/mlx5/core/cq.c
index 1fd403713bafc5ffead5ccecf45197559fedce99..35039a95dcfd43697208a959733b4a7afdadfa11 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/cq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/cq.c
@@ -145,7 +145,7 @@ int mlx5_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
                mlx5_core_dbg(dev, "failed adding CP 0x%x to debug file system\n",
                              cq->cqn);
 
-       cq->uar = dev->priv.uar;
+       cq->uar = dev->priv.bfreg.up;
        cq->irqn = eq->core.irqn;
 
        return 0;

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
index 31e7f59bc19b1eaeae26e9fdf17a61f3ed1990ec..b6b4ae7c59fa04c12058730d689ca48f4254f254 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
@@ -810,7 +810,7 @@ static void mlx5e_build_common_cq_param(struct mlx5_core_dev *mdev,
 {
        void *cqc = param->cqc;
 
-       MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
+       MLX5_SET(cqc, cqc, uar_page, mdev->priv.bfreg.up->index);
        if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
                MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD);
 }

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
index 391b4e9c9dc49121cb2d1af1c03887b30f902c18..7c1d9a9ea46456c880656fad0d668fa218f706fb 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
@@ -334,7 +334,7 @@ static int mlx5e_ptp_alloc_txqsq(struct mlx5e_ptp *c, int txq_ix,
        sq->mdev      = mdev;
        sq->ch_ix     = MLX5E_PTP_CHANNEL_IX;
        sq->txq_ix    = txq_ix;
-       sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
+       sq->uar_map   = mdev->priv.bfreg.map;
        sq->min_inline_mode = params->tx_min_inline_mode;
        sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);
        sq->stats     = &c->priv->ptp_stats.sq[tc];

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
index 6ed3a32b7e226d497234e4fa7b244bf9629b5710..e9e36358c39d1647ec32ddcbc52bf4a4c4a7dc59 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
@@ -163,17 +163,11 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises)
                goto err_dealloc_transport_domain;
        }
 
-       err = mlx5_alloc_bfreg(mdev, &res->bfreg, false, false);
-       if (err) {
-               mlx5_core_err(mdev, "alloc bfreg failed, %d\n", err);
-               goto err_destroy_mkey;
-       }
-
        if (create_tises) {
                err = mlx5e_create_tises(mdev, res->tisn);
                if (err) {
                        mlx5_core_err(mdev, "alloc tises failed, %d\n", err);
-                       goto err_destroy_bfreg;
+                       goto err_destroy_mkey;
                }
                res->tisn_valid = true;
        }
@@ -190,8 +184,6 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises)
 
        return 0;
 
-err_destroy_bfreg:
-       mlx5_free_bfreg(mdev, &res->bfreg);
 err_destroy_mkey:
        mlx5_core_destroy_mkey(mdev, res->mkey);
 err_dealloc_transport_domain:
@@ -209,7 +201,6 @@ void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev)
        mdev->mlx5e_res.dek_priv = NULL;
        if (res->tisn_valid)
                mlx5e_destroy_tises(mdev, res->tisn);
-       mlx5_free_bfreg(mdev, &res->bfreg);
        mlx5_core_destroy_mkey(mdev, res->mkey);
        mlx5_core_dealloc_transport_domain(mdev, res->td.tdn);
        mlx5_core_dealloc_pd(mdev, res->pdn);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index fe5f5ae433b7ddedd85e43424907f1d6afd3ee3a..22e3bc72a265bd8a85e3d48d505590c903455df3 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -1536,7 +1536,7 @@ static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
        sq->pdev      = c->pdev;
        sq->mkey_be   = c->mkey_be;
        sq->channel   = c;
-       sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
+       sq->uar_map   = mdev->priv.bfreg.map;
        sq->min_inline_mode = params->tx_min_inline_mode;
        sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN;
        sq->xsk_pool  = xsk_pool;
@@ -1621,7 +1621,7 @@ static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
        int err;
 
        sq->channel   = c;
-       sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
+       sq->uar_map   = mdev->priv.bfreg.map;
        sq->reserved_room = param->stop_room;
 
        param->wq.db_numa_node = cpu_to_node(c->cpu);
@@ -1706,7 +1706,7 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
        sq->priv      = c->priv;
        sq->ch_ix     = c->ix;
        sq->txq_ix    = txq_ix;
-       sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
+       sq->uar_map   = mdev->priv.bfreg.map;
        sq->min_inline_mode = params->tx_min_inline_mode;
        sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);
        sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
@@ -1782,7 +1782,7 @@ static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
        MLX5_SET(sqc,  sqc, flush_in_error_en, 1);
 
        MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
-       MLX5_SET(wq,   wq, uar_page,      mdev->mlx5e_res.hw_objs.bfreg.index);
+       MLX5_SET(wq,   wq, uar_page,      mdev->priv.bfreg.index);
        MLX5_SET(wq,   wq, log_wq_pg_sz,  csp->wq_ctrl->buf.page_shift -
                                          MLX5_ADAPTER_PAGE_SHIFT);
        MLX5_SET64(wq, wq, dbr_addr,      csp->wq_ctrl->db.dma);
@@ -2277,7 +2277,7 @@ static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
        MLX5_SET(cqc, cqc, cq_period_mode, mlx5e_cq_period_mode(param->cq_period_mode));
 
        MLX5_SET(cqc,   cqc, c_eqn_or_apu_element, eqn);
-       MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);
+       MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.bfreg.up->index);
        MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
                                            MLX5_ADAPTER_PAGE_SHIFT);
        MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index f3c714ebd9cb417c6f14897393a831ee1bf4110d..25499da177bc4c73aa5a64033cd97eafb64adbd7 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -307,7 +307,7 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
 
        eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
        MLX5_SET(eqc, eqc, log_eq_size, eq->fbc.log_sz);
-       MLX5_SET(eqc, eqc, uar_page, priv->uar->index);
+       MLX5_SET(eqc, eqc, uar_page, priv->bfreg.up->index);
        MLX5_SET(eqc, eqc, intr, vecidx);
        MLX5_SET(eqc, eqc, log_page_size,
                 eq->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
@@ -320,7 +320,7 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
        eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
        eq->irqn = pci_irq_vector(dev->pdev, vecidx);
        eq->dev = dev;
-       eq->doorbell = priv->uar->map + MLX5_EQ_DOORBELL_OFFSET;
+       eq->doorbell = priv->bfreg.up->map + MLX5_EQ_DOORBELL_OFFSET;
 
        err = mlx5_debug_eq_add(dev, eq);
        if (err)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c
index 58bd749b5e4de07a19320e223a0103b8ae7ded25..129725159a93565687b4bed5467de2d6fd2e18d3 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c
@@ -100,7 +100,7 @@ static int create_aso_cq(struct mlx5_aso_cq *cq, void *cqc_data)
 
        MLX5_SET(cqc,   cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
        MLX5_SET(cqc,   cqc, c_eqn_or_apu_element, eqn);
-       MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);
+       MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.bfreg.up->index);
        MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
                                            MLX5_ADAPTER_PAGE_SHIFT);
        MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
@@ -129,7 +129,7 @@ static int mlx5_aso_create_cq(struct mlx5_core_dev *mdev, int numa_node,
                return -ENOMEM;
 
        MLX5_SET(cqc, cqc_data, log_cq_size, 1);
-       MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.uar->index);
+       MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.bfreg.up->index);
        if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
                MLX5_SET(cqc, cqc_data, cqe_sz, CQE_STRIDE_128_PAD);
 
@@ -163,7 +163,7 @@ static int mlx5_aso_alloc_sq(struct mlx5_core_dev *mdev, int numa_node,
        struct mlx5_wq_param param;
        int err;
 
-       sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map;
+       sq->uar_map = mdev->priv.bfreg.map;
 
        param.db_numa_node = numa_node;
        param.buf_numa_node = numa_node;
@@ -203,7 +203,7 @@ static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn,
        MLX5_SET(sqc, sqc, ts_format, ts_format);
 
        MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
-       MLX5_SET(wq,   wq, uar_page,      mdev->mlx5e_res.hw_objs.bfreg.index);
+       MLX5_SET(wq,   wq, uar_page,      mdev->priv.bfreg.index);
        MLX5_SET(wq,   wq, log_wq_pg_sz,  sq->wq_ctrl.buf.page_shift -
                                          MLX5_ADAPTER_PAGE_SHIFT);
        MLX5_SET64(wq, wq, dbr_addr,      sq->wq_ctrl.db.dma);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index eb3ac98a262128f978bbb1c62103ea0d427f0359..6b6d6b05b8937ae9b9f19e30ff6cacf52c251a81 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -1316,10 +1316,9 @@ static int mlx5_load(struct mlx5_core_dev *dev)
 {
        int err;
 
-       dev->priv.uar = mlx5_get_uars_page(dev);
-       if (IS_ERR(dev->priv.uar)) {
-               mlx5_core_err(dev, "Failed allocating uar, aborting\n");
-               err = PTR_ERR(dev->priv.uar);
+       err = mlx5_alloc_bfreg(dev, &dev->priv.bfreg, false, false);
+       if (err) {
+               mlx5_core_err(dev, "Failed allocating bfreg, %d\n", err);
                return err;
        }
 
@@ -1430,7 +1429,7 @@ err_eq_table:
 err_irq_table:
        mlx5_pagealloc_stop(dev);
        mlx5_events_stop(dev);
-       mlx5_put_uars_page(dev, dev->priv.uar);
+       mlx5_free_bfreg(dev, &dev->priv.bfreg);
        return err;
 }
 
@@ -1455,7 +1454,7 @@ static void mlx5_unload(struct mlx5_core_dev *dev)
        mlx5_irq_table_destroy(dev);
        mlx5_pagealloc_stop(dev);
        mlx5_events_stop(dev);
-       mlx5_put_uars_page(dev, dev->priv.uar);
+       mlx5_free_bfreg(dev, &dev->priv.bfreg);
 }
 
 int mlx5_init_one_devl_locked(struct mlx5_core_dev *dev)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/send.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/send.c
index b0595c9b09e421996ec65d25e6a3a1b0373d9b57..24ef7d66fa8aa2c6aa298ee679f61d69a0e42de4 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/send.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/send.c
@@ -690,7 +690,7 @@ static int hws_send_ring_alloc_sq(struct mlx5_core_dev *mdev,
        size_t buf_sz;
        int err;
 
-       sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map;
+       sq->uar_map = mdev->priv.bfreg.map;
        sq->mdev = mdev;
 
        param.db_numa_node = numa_node;
@@ -764,7 +764,7 @@ static int hws_send_ring_create_sq(struct mlx5_core_dev *mdev, u32 pdn,
        MLX5_SET(sqc, sqc, ts_format, ts_format);
 
        MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
-       MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index);
+       MLX5_SET(wq, wq, uar_page, mdev->priv.bfreg.index);
        MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
        MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
 
@@ -940,7 +940,7 @@ static int hws_send_ring_create_cq(struct mlx5_core_dev *mdev,
                                  (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
 
        MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn);
-       MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
+       MLX5_SET(cqc, cqc, uar_page, mdev->priv.bfreg.up->index);
        MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
        MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
 
@@ -963,7 +963,7 @@ static int hws_send_ring_open_cq(struct mlx5_core_dev *mdev,
        if (!cqc_data)
                return -ENOMEM;
 
-       MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.uar->index);
+       MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.bfreg.up->index);
        MLX5_SET(cqc, cqc_data, log_cq_size, ilog2(queue->num_entries));
 
        err = hws_send_ring_alloc_cq(mdev, numa_node, queue, cqc_data, cq);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wc.c b/drivers/net/ethernet/mellanox/mlx5/core/wc.c
index 276594586404de30ecc6f3f0e37f69c52795d4b0..999d6216648a92c1c57bd98389e4ff5c3729622f 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/wc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/wc.c
@@ -94,7 +94,7 @@ static int create_wc_cq(struct mlx5_wc_cq *cq, void *cqc_data)
 
        MLX5_SET(cqc,   cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
        MLX5_SET(cqc,   cqc, c_eqn_or_apu_element, eqn);
-       MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);
+       MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.bfreg.up->index);
        MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
                                            MLX5_ADAPTER_PAGE_SHIFT);
        MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
@@ -116,7 +116,7 @@ static int mlx5_wc_create_cq(struct mlx5_core_dev *mdev, struct mlx5_wc_cq *cq)
                return -ENOMEM;
 
        MLX5_SET(cqc, cqc, log_cq_size, TEST_WC_LOG_CQ_SZ);
-       MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
+       MLX5_SET(cqc, cqc, uar_page, mdev->priv.bfreg.up->index);
        if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
                MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD);
 

diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index 5a85b6d91ba3373d4f422274e187c6ec088e8c25..15c434fedff770aae8fe663e7231e0f623399125 100644 (file)
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -612,7 +612,7 @@ struct mlx5_priv {
        struct mlx5_ft_pool             *ft_pool;
 
        struct mlx5_bfreg_data          bfregs;
-       struct mlx5_uars_page          *uar;
+       struct mlx5_sq_bfreg bfreg;
 #ifdef CONFIG_MLX5_SF
        struct mlx5_vhca_state_notifier *vhca_state_notifier;
        struct mlx5_sf_dev_table *sf_dev_table;
@@ -658,7 +658,6 @@ struct mlx5e_resources {
                u32                        pdn;
                struct mlx5_td             td;
                u32                        mkey;
-               struct mlx5_sq_bfreg       bfreg;
 #define MLX5_MAX_NUM_TC 8
                u32                        tisn[MLX5_MAX_PORTS][MLX5_MAX_NUM_TC];
                bool                       tisn_valid;