RDMA/bnxt_re: Share a page to expose per SRQ info with userspace

Gen P7 adapters needs to share a toggle bits information received
in kernel driver with the user space. User space needs this
info to arm the SRQ.

User space application can get this page using the
UAPI routines. Library will mmap this page and get the
toggle bits to be used in the next ARM Doorbell.

Uses a hash list to map the SRQ structure from the SRQ ID.
SRQ structure is retrieved from the hash list while the
library calls the UAPI routine to get the toggle page
mapping. Currently the full page is mapped per SRQ. This
can be optimized to enable multiple SRQs from the same
application share the same page and different offsets
in the page

Signed-off-by: Chandramohan Akula <chandramohan.akula@broadcom.com>
Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
Link: https://patch.msgid.link/1724945645-14989-4-git-send-email-selvin.xavier@broadcom.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index 0912d2fa9634f246c21e8c134b7b9dbcff89f325..2be9a62d230fef0e41d33f94bbbf78faba3c1e66 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -141,6 +141,7 @@ struct bnxt_re_pacing {
 #define BNXT_RE_GRC_FIFO_REG_BASE 0x2000
 
 #define MAX_CQ_HASH_BITS               (16)
+#define MAX_SRQ_HASH_BITS              (16)
 struct bnxt_re_dev {
        struct ib_device                ibdev;
        struct list_head                list;
@@ -196,6 +197,7 @@ struct bnxt_re_dev {
        struct work_struct dbq_fifo_check_work;
        struct delayed_work dbq_pacing_work;
        DECLARE_HASHTABLE(cq_hash, MAX_CQ_HASH_BITS);
+       DECLARE_HASHTABLE(srq_hash, MAX_SRQ_HASH_BITS);
 };
 
 #define to_bnxt_re_dev(ptr, member)    \

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index eafbee4af0679192ee719cbe87c9cf575e33fc3b..f9f944a6094a4ec4fbacfc748a247c54ca7a179a 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -1707,6 +1707,10 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
 
        if (qplib_srq->cq)
                nq = qplib_srq->cq->nq;
+       if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
+               free_page((unsigned long)srq->uctx_srq_page);
+               hash_del(&srq->hash_entry);
+       }
        bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
        ib_umem_release(srq->umem);
        atomic_dec(&rdev->stats.res.srq_count);
@@ -1811,9 +1815,18 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,
        }
 
        if (udata) {
-               struct bnxt_re_srq_resp resp;
+               struct bnxt_re_srq_resp resp = {};
 
                resp.srqid = srq->qplib_srq.id;
+               if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
+                       hash_add(rdev->srq_hash, &srq->hash_entry, srq->qplib_srq.id);
+                       srq->uctx_srq_page = (void *)get_zeroed_page(GFP_KERNEL);
+                       if (!srq->uctx_srq_page) {
+                               rc = -ENOMEM;
+                               goto fail;
+                       }
+                       resp.comp_mask |= BNXT_RE_SRQ_TOGGLE_PAGE_SUPPORT;
+               }
                rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
                if (rc) {
                        ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!");
@@ -4291,6 +4304,19 @@ static struct bnxt_re_cq *bnxt_re_search_for_cq(struct bnxt_re_dev *rdev, u32 cq
        return cq;
 }
 
+static struct bnxt_re_srq *bnxt_re_search_for_srq(struct bnxt_re_dev *rdev, u32 srq_id)
+{
+       struct bnxt_re_srq *srq = NULL, *tmp_srq;
+
+       hash_for_each_possible(rdev->srq_hash, tmp_srq, hash_entry, srq_id) {
+               if (tmp_srq->qplib_srq.id == srq_id) {
+                       srq = tmp_srq;
+                       break;
+               }
+       }
+       return srq;
+}
+
 /* Helper function to mmap the virtual memory from user app */
 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
 {
@@ -4519,6 +4545,7 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund
        struct bnxt_re_ucontext *uctx;
        struct ib_ucontext *ib_uctx;
        struct bnxt_re_dev *rdev;
+       struct bnxt_re_srq *srq;
        u32 length = PAGE_SIZE;
        struct bnxt_re_cq *cq;
        u64 mem_offset;
@@ -4550,6 +4577,11 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund
                addr = (u64)cq->uctx_cq_page;
                break;
        case BNXT_RE_SRQ_TOGGLE_MEM:
+               srq = bnxt_re_search_for_srq(rdev, res_id);
+               if (!srq)
+                       return -EINVAL;
+
+               addr = (u64)srq->uctx_srq_page;
                break;
 
        default:

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
index 060bf62ecebab235123a3af7c7c6da47b572a2cd..b789e47ec97a854b3b7effacc3593cd9f5d8d1ee 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
@@ -78,6 +78,7 @@ struct bnxt_re_srq {
        struct ib_umem          *umem;
        spinlock_t              lock;           /* protect srq */
        void                    *uctx_srq_page;
+       struct hlist_node       hash_entry;
 };
 
 struct bnxt_re_qp {

diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index 31ba89cffe9d862734c9649965c290fae973c4cc..16a84ca1ce48bb6d34e210b44f9a70c5c42bc35f 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -139,8 +139,10 @@ static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev)
        if (bnxt_re_hwrm_qcaps(rdev))
                dev_err(rdev_to_dev(rdev),
                        "Failed to query hwrm qcaps\n");
-       if (bnxt_qplib_is_chip_gen_p7(rdev->chip_ctx))
+       if (bnxt_qplib_is_chip_gen_p7(rdev->chip_ctx)) {
                cctx->modes.toggle_bits |= BNXT_QPLIB_CQ_TOGGLE_BIT;
+               cctx->modes.toggle_bits |= BNXT_QPLIB_SRQ_TOGGLE_BIT;
+       }
 }
 
 static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)
@@ -1771,6 +1773,8 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev)
                bnxt_re_vf_res_config(rdev);
        }
        hash_init(rdev->cq_hash);
+       if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT)
+               hash_init(rdev->srq_hash);
 
        return 0;
 free_sctx:

diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h
index 6821002931c823a6709b577139ea5e2e41343119..faa9d62b3b3091ed0b8b1200a39bcab94797f45d 100644 (file)
--- a/include/uapi/rdma/bnxt_re-abi.h
+++ b/include/uapi/rdma/bnxt_re-abi.h
@@ -141,8 +141,14 @@ struct bnxt_re_srq_req {
        __aligned_u64 srq_handle;
 };
 
+enum bnxt_re_srq_mask {
+       BNXT_RE_SRQ_TOGGLE_PAGE_SUPPORT = 0x1,
+};
+
 struct bnxt_re_srq_resp {
        __u32 srqid;
+       __u32 rsvd; /* padding */
+       __aligned_u64 comp_mask;
 };
 
 enum bnxt_re_shpg_offt {