RDMA/irdma: Add missing read barriers

[ Upstream commit 4984eb51453ff7eddee9e5ce816145be39c0ec5c ]

On code inspection, there are many instances in the driver where
CEQE and AEQE fields written to by HW are read without guaranteeing
that the polarity bit has been read and checked first.

Add a read barrier to avoid reordering of loads on the CEQE/AEQE fields
prior to checking the polarity bit.

Fixes: 3f49d6842569 ("RDMA/irdma: Implement HW Admin Queue OPs")
Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com>
Link: https://lore.kernel.org/r/20230711175253.1289-2-shiraz.saleem@intel.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c
index 1ac7067e21be12d16a80e155f8734acad7482ff0..ab195ae089cfde7b6eb52e6bf45f49c25a3eacdd 100644 (file)
--- a/drivers/infiniband/hw/irdma/ctrl.c
+++ b/drivers/infiniband/hw/irdma/ctrl.c
@@ -3395,6 +3395,9 @@ enum irdma_status_code irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
        if (polarity != ccq->cq_uk.polarity)
                return IRDMA_ERR_Q_EMPTY;
 
+       /* Ensure CEQE contents are read after valid bit is checked */
+       dma_rmb();
+
        get_64bit_val(cqe, 8, &qp_ctx);
        cqp = (struct irdma_sc_cqp *)(unsigned long)qp_ctx;
        info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp);
@@ -4046,13 +4049,17 @@ enum irdma_status_code irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
        u8 polarity;
 
        aeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq);
-       get_64bit_val(aeqe, 0, &compl_ctx);
        get_64bit_val(aeqe, 8, &temp);
        polarity = (u8)FIELD_GET(IRDMA_AEQE_VALID, temp);
 
        if (aeq->polarity != polarity)
                return IRDMA_ERR_Q_EMPTY;
 
+       /* Ensure AEQE contents are read after valid bit is checked */
+       dma_rmb();
+
+       get_64bit_val(aeqe, 0, &compl_ctx);
+
        print_hex_dump_debug("WQE: AEQ_ENTRY WQE", DUMP_PREFIX_OFFSET, 16, 8,
                             aeqe, 16, false);
 

diff --git a/drivers/infiniband/hw/irdma/puda.c b/drivers/infiniband/hw/irdma/puda.c
index 58e7d875643b805af84d4054e5e5dcb12201c175..197eba5eb78fa65f248a0c718017869967ff3a6b 100644 (file)
--- a/drivers/infiniband/hw/irdma/puda.c
+++ b/drivers/infiniband/hw/irdma/puda.c
@@ -235,6 +235,9 @@ irdma_puda_poll_info(struct irdma_sc_cq *cq, struct irdma_puda_cmpl_info *info)
        if (valid_bit != cq_uk->polarity)
                return IRDMA_ERR_Q_EMPTY;
 
+       /* Ensure CQE contents are read after valid bit is checked */
+       dma_rmb();
+
        if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
                ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
 
@@ -248,6 +251,9 @@ irdma_puda_poll_info(struct irdma_sc_cq *cq, struct irdma_puda_cmpl_info *info)
                if (polarity != cq_uk->polarity)
                        return IRDMA_ERR_Q_EMPTY;
 
+               /* Ensure ext CQE contents are read after ext valid bit is checked */
+               dma_rmb();
+
                IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
                if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
                        cq_uk->polarity = !cq_uk->polarity;

diff --git a/drivers/infiniband/hw/irdma/uk.c b/drivers/infiniband/hw/irdma/uk.c
index a348f0c010ab396e5ea5e20f57812f906b9c1da8..4b00a9adbe3a52d7b393d5bd10bae4363d623bc0 100644 (file)
--- a/drivers/infiniband/hw/irdma/uk.c
+++ b/drivers/infiniband/hw/irdma/uk.c
@@ -1549,6 +1549,9 @@ void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq)
                if (polarity != temp)
                        break;
 
+               /* Ensure CQE contents are read after valid bit is checked */
+               dma_rmb();
+
                get_64bit_val(cqe, 8, &comp_ctx);
                if ((void *)(unsigned long)comp_ctx == q)
                        set_64bit_val(cqe, 8, 0);