RDMA/bnxt_re: Support doorbells for app allocated QPs

App allocated QPs can use a separate doorbell for each QP.
This doorbell region can be passed through a new driver specific
DBR_HANDLE attribute, during QP creation. When this attribute
is set, associate the QP with the given doorbell region.

While the QP holds a reference to the dbr, the dbr itself
cannot be destroyed and is rejected with EBUSY error.

Link: https://patch.msgid.link/r/20260519150041.7251-9-sriharsha.basavapatna@broadcom.com
Signed-off-by: Sriharsha Basavapatna <sriharsha.basavapatna@broadcom.com>
Reviewed-by: Selvin Xavier <selvin.xavier@broadcom.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index c706ba19e7194c8f67b79ff8031d28b37bec845b..a5583f0125ed0f70b2aaa30711c5e647455d9ad1 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -1024,6 +1024,9 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
        if (rc)
                ibdev_err(&rdev->ibdev, "Failed to destroy HW QP");
 
+       if (qp->dbr_obj)
+               kref_put(&qp->dbr_obj->usecnt, bnxt_re_dbr_kref_release);
+
        if (rdma_is_kernel_res(&qp->ib_qp.res)) {
                flags = bnxt_re_lock_cqs(qp);
                bnxt_qplib_clean_qp(&qp->qplib_qp);
@@ -1191,7 +1194,8 @@ static int bnxt_re_get_psn_bytes(struct bnxt_re_dev *rdev,
 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
                                struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx,
                                struct bnxt_re_qp_req *ureq,
-                               bool fixed_que_attr)
+                               bool fixed_que_attr,
+                               struct bnxt_re_dbr_obj *dbr_obj)
 {
        struct bnxt_qplib_qp *qplib_qp;
        struct ib_umem *umem;
@@ -1234,8 +1238,11 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
                goto rqfail;
 
 done:
+       if (dbr_obj)
+               qplib_qp->dpi = &dbr_obj->dpi;
+       else
+               qplib_qp->dpi = &cntx->dpi;
        qplib_qp->qp_handle = ureq->qp_handle;
-       qplib_qp->dpi = &cntx->dpi;
        qplib_qp->is_user = true;
        return 0;
 
@@ -1709,7 +1716,8 @@ static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp,
 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
                                struct ib_qp_init_attr *init_attr,
                                struct bnxt_re_ucontext *uctx,
-                               struct bnxt_re_qp_req *ureq)
+                               struct bnxt_re_qp_req *ureq,
+                               struct bnxt_re_dbr_obj *dbr_obj)
 {
        struct bnxt_qplib_dev_attr *dev_attr;
        struct bnxt_qplib_qp *qplqp;
@@ -1776,7 +1784,8 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
                bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx);
 
        if (uctx) { /* This will update DPI and qp_handle */
-               rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq, fixed_que_attr);
+               rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq, fixed_que_attr,
+                                         dbr_obj);
                if (rc)
                        return rc;
        }
@@ -1912,7 +1921,9 @@ static int bnxt_re_add_unique_gid(struct bnxt_re_dev *rdev)
 int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
                      struct ib_udata *udata)
 {
+       struct bnxt_re_dbr_obj *dbr_obj = NULL;
        struct bnxt_qplib_dev_attr *dev_attr;
+       struct uverbs_attr_bundle *attrs;
        struct bnxt_re_ucontext *uctx;
        struct bnxt_re_qp_req ureq;
        struct bnxt_re_dev *rdev;
@@ -1933,6 +1944,17 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
                rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, 0);
                if (rc)
                        return rc;
+
+               attrs = rdma_udata_to_uverbs_attr_bundle(udata);
+               if (uverbs_attr_is_valid(attrs,
+                                        BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE)) {
+                       dbr_obj = uverbs_attr_get_obj(attrs,
+                                                     BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE);
+                       if (IS_ERR(dbr_obj))
+                               return PTR_ERR(dbr_obj);
+                       kref_get(&dbr_obj->usecnt);
+                       qp->dbr_obj = dbr_obj;
+               }
        }
 
        rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
@@ -1942,7 +1964,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
        }
 
        qp->rdev = rdev;
-       rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq);
+       rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq,
+                                 dbr_obj);
        if (rc)
                goto fail;
 
@@ -2012,6 +2035,8 @@ free_hwq:
        bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
        bnxt_re_qp_free_umem(qp);
 fail:
+       if (dbr_obj)
+               kref_put(&dbr_obj->usecnt, bnxt_re_dbr_kref_release);
        return rc;
 }
 

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
index 6a5bcc3fb289b24d3a966a6295db0353c29f6d4d..ebc393e6da4fd418dae5e94e91c3e0a9587d632b 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
@@ -96,6 +96,7 @@ struct bnxt_re_qp {
        struct bnxt_re_cq       *scq;
        struct bnxt_re_cq       *rcq;
        struct dentry           *dentry;
+       struct bnxt_re_dbr_obj *dbr_obj; /* doorbell region */
 };
 
 struct bnxt_re_cq {

diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c
index 1d44d6225da07aa3e3c77dde8da5a73d8f9bfc77..9e68b4a7e952d0e8e5ac7aea92cf3ca58930f8ca 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/uapi.c
+++ b/drivers/infiniband/hw/bnxt_re/uapi.c
@@ -418,6 +418,15 @@ static int bnxt_re_dbr_cleanup(struct ib_uobject *uobject,
 {
        struct bnxt_re_dbr_obj *obj = uobject->object;
 
+       /* If it is being destroyed explicitly while QPs still hold a
+        * reference (> 1), reject it with EBUSY. If no QP references
+        * or implicit teardown (process exit, driver removal), drop
+        * the uobject reference unconditionally. The object gets freed
+        * (bnxt_re_dbr_kref_release) when the usecnt goes to zero.
+        */
+       if (why == RDMA_REMOVE_DESTROY && kref_read(&obj->usecnt) > 1)
+               return -EBUSY;
+
        kref_put(&obj->usecnt, bnxt_re_dbr_kref_release);
        return 0;
 }
@@ -478,11 +487,26 @@ DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR,
 DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_DEFAULT_DBR,
                              &UVERBS_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR));
 
+ADD_UVERBS_ATTRIBUTES_SIMPLE(
+       bnxt_re_qp_create,
+       UVERBS_OBJECT_QP,
+       UVERBS_METHOD_QP_CREATE,
+       UVERBS_ATTR_IDR(BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE,
+                       BNXT_RE_OBJECT_DBR,
+                       UVERBS_ACCESS_READ,
+                       UA_OPTIONAL));
+
+const struct uapi_definition bnxt_re_create_qp_defs[] = {
+       UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_QP, &bnxt_re_qp_create),
+       {},
+};
+
 const struct uapi_definition bnxt_re_uapi_defs[] = {
        UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE),
        UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV),
        UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM),
        UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DBR),
        UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DEFAULT_DBR),
+       UAPI_DEF_CHAIN(bnxt_re_create_qp_defs),
        {}
 };

diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h
index db8400f2ce3bc542df8dd9008164e9bb384aabe4..4da8cda337dcbd1ea9781789c357f65a8d71dba5 100644 (file)
--- a/include/uapi/rdma/bnxt_re-abi.h
+++ b/include/uapi/rdma/bnxt_re-abi.h
@@ -138,6 +138,10 @@ struct bnxt_re_qp_req {
        __u32 sq_npsn;
 };
 
+enum bnxt_re_create_qp_attrs {
+       BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE = UVERBS_ID_DRIVER_NS_WITH_UHW,
+};
+
 struct bnxt_re_qp_resp {
        __u32 qpid;
        __u32 rsvd;