From: Sriharsha Basavapatna Date: Tue, 19 May 2026 15:00:40 +0000 (+0530) Subject: RDMA/bnxt_re: Support doorbells for app allocated QPs X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7e812673a948395675e9893e75b0fc60d19bbf27;p=thirdparty%2Fkernel%2Flinux.git RDMA/bnxt_re: Support doorbells for app allocated QPs App allocated QPs can use a separate doorbell for each QP. This doorbell region can be passed through a new driver specific DBR_HANDLE attribute, during QP creation. When this attribute is set, associate the QP with the given doorbell region. While the QP holds a reference to the dbr, the dbr itself cannot be destroyed and is rejected with EBUSY error. Link: https://patch.msgid.link/r/20260519150041.7251-9-sriharsha.basavapatna@broadcom.com Signed-off-by: Sriharsha Basavapatna Reviewed-by: Selvin Xavier Signed-off-by: Jason Gunthorpe --- diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index c706ba19e7194..a5583f0125ed0 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1024,6 +1024,9 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) if (rc) ibdev_err(&rdev->ibdev, "Failed to destroy HW QP"); + if (qp->dbr_obj) + kref_put(&qp->dbr_obj->usecnt, bnxt_re_dbr_kref_release); + if (rdma_is_kernel_res(&qp->ib_qp.res)) { flags = bnxt_re_lock_cqs(qp); bnxt_qplib_clean_qp(&qp->qplib_qp); @@ -1191,7 +1194,8 @@ static int bnxt_re_get_psn_bytes(struct bnxt_re_dev *rdev, static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx, struct bnxt_re_qp_req *ureq, - bool fixed_que_attr) + bool fixed_que_attr, + struct bnxt_re_dbr_obj *dbr_obj) { struct bnxt_qplib_qp *qplib_qp; struct ib_umem *umem; @@ -1234,8 +1238,11 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, goto rqfail; done: + if (dbr_obj) + qplib_qp->dpi = &dbr_obj->dpi; + else + qplib_qp->dpi = &cntx->dpi; qplib_qp->qp_handle = ureq->qp_handle; - qplib_qp->dpi = &cntx->dpi; qplib_qp->is_user = true; return 0; @@ -1709,7 +1716,8 @@ static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp, static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, struct ib_qp_init_attr *init_attr, struct bnxt_re_ucontext *uctx, - struct bnxt_re_qp_req *ureq) + struct bnxt_re_qp_req *ureq, + struct bnxt_re_dbr_obj *dbr_obj) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; @@ -1776,7 +1784,8 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx); if (uctx) { /* This will update DPI and qp_handle */ - rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq, fixed_que_attr); + rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq, fixed_que_attr, + dbr_obj); if (rc) return rc; } @@ -1912,7 +1921,9 @@ static int bnxt_re_add_unique_gid(struct bnxt_re_dev *rdev) int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, struct ib_udata *udata) { + struct bnxt_re_dbr_obj *dbr_obj = NULL; struct bnxt_qplib_dev_attr *dev_attr; + struct uverbs_attr_bundle *attrs; struct bnxt_re_ucontext *uctx; struct bnxt_re_qp_req ureq; struct bnxt_re_dev *rdev; @@ -1933,6 +1944,17 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, 0); if (rc) return rc; + + attrs = rdma_udata_to_uverbs_attr_bundle(udata); + if (uverbs_attr_is_valid(attrs, + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE)) { + dbr_obj = uverbs_attr_get_obj(attrs, + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE); + if (IS_ERR(dbr_obj)) + return PTR_ERR(dbr_obj); + kref_get(&dbr_obj->usecnt); + qp->dbr_obj = dbr_obj; + } } rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); @@ -1942,7 +1964,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, } qp->rdev = rdev; - rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq); + rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq, + dbr_obj); if (rc) goto fail; @@ -2012,6 +2035,8 @@ free_hwq: bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); bnxt_re_qp_free_umem(qp); fail: + if (dbr_obj) + kref_put(&dbr_obj->usecnt, bnxt_re_dbr_kref_release); return rc; } diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 6a5bcc3fb289b..ebc393e6da4fd 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -96,6 +96,7 @@ struct bnxt_re_qp { struct bnxt_re_cq *scq; struct bnxt_re_cq *rcq; struct dentry *dentry; + struct bnxt_re_dbr_obj *dbr_obj; /* doorbell region */ }; struct bnxt_re_cq { diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index 1d44d6225da07..9e68b4a7e952d 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -418,6 +418,15 @@ static int bnxt_re_dbr_cleanup(struct ib_uobject *uobject, { struct bnxt_re_dbr_obj *obj = uobject->object; + /* If it is being destroyed explicitly while QPs still hold a + * reference (> 1), reject it with EBUSY. If no QP references + * or implicit teardown (process exit, driver removal), drop + * the uobject reference unconditionally. The object gets freed + * (bnxt_re_dbr_kref_release) when the usecnt goes to zero. + */ + if (why == RDMA_REMOVE_DESTROY && kref_read(&obj->usecnt) > 1) + return -EBUSY; + kref_put(&obj->usecnt, bnxt_re_dbr_kref_release); return 0; } @@ -478,11 +487,26 @@ DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR, DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_DEFAULT_DBR, &UVERBS_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR)); +ADD_UVERBS_ATTRIBUTES_SIMPLE( + bnxt_re_qp_create, + UVERBS_OBJECT_QP, + UVERBS_METHOD_QP_CREATE, + UVERBS_ATTR_IDR(BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE, + BNXT_RE_OBJECT_DBR, + UVERBS_ACCESS_READ, + UA_OPTIONAL)); + +const struct uapi_definition bnxt_re_create_qp_defs[] = { + UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_QP, &bnxt_re_qp_create), + {}, +}; + const struct uapi_definition bnxt_re_uapi_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DBR), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DEFAULT_DBR), + UAPI_DEF_CHAIN(bnxt_re_create_qp_defs), {} }; diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h index db8400f2ce3bc..4da8cda337dcb 100644 --- a/include/uapi/rdma/bnxt_re-abi.h +++ b/include/uapi/rdma/bnxt_re-abi.h @@ -138,6 +138,10 @@ struct bnxt_re_qp_req { __u32 sq_npsn; }; +enum bnxt_re_create_qp_attrs { + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE = UVERBS_ID_DRIVER_NS_WITH_UHW, +}; + struct bnxt_re_qp_resp { __u32 qpid; __u32 rsvd;