RDMA/mlx5: Provide a modern CQ creation interface

The uverbs CQ creation UAPI allows users to supply their own umem for a CQ.
Update mlx5 to support this workflow while preserving support for creating
umem through the legacy interface.

Link: https://patch.msgid.link/20260213-refactor-umem-v1-11-f3be85847922@nvidia.com
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>

diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 839958d15a83328ab19bee30200460aaa71ead3b..43a7b5ca49dcc900472b24b28e7fbc1516dc4fdd 100644 (file)
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -749,16 +749,15 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 
        *cqe_size = ucmd.cqe_size;
 
-       cq->buf.umem =
-               ib_umem_get(&dev->ib_dev, ucmd.buf_addr,
-                           entries * ucmd.cqe_size, IB_ACCESS_LOCAL_WRITE);
-       if (IS_ERR(cq->buf.umem)) {
-               err = PTR_ERR(cq->buf.umem);
-               return err;
-       }
+       if (!cq->ibcq.umem)
+               cq->ibcq.umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr,
+                                           entries * ucmd.cqe_size,
+                                           IB_ACCESS_LOCAL_WRITE);
+       if (IS_ERR(cq->ibcq.umem))
+               return PTR_ERR(cq->ibcq.umem);
 
        page_size = mlx5_umem_find_best_cq_quantized_pgoff(
-               cq->buf.umem, cqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT,
+               cq->ibcq.umem, cqc, log_page_size, MLX5_ADAPTER_PAGE_SHIFT,
                page_offset, 64, &page_offset_quantized);
        if (!page_size) {
                err = -EINVAL;
@@ -769,12 +768,12 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
        if (err)
                goto err_umem;
 
-       ncont = ib_umem_num_dma_blocks(cq->buf.umem, page_size);
+       ncont = ib_umem_num_dma_blocks(cq->ibcq.umem, page_size);
        mlx5_ib_dbg(
                dev,
                "addr 0x%llx, size %u, npages %zu, page_size %lu, ncont %d\n",
                ucmd.buf_addr, entries * ucmd.cqe_size,
-               ib_umem_num_pages(cq->buf.umem), page_size, ncont);
+               ib_umem_num_pages(cq->ibcq.umem), page_size, ncont);
 
        *inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
                 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * ncont;
@@ -785,7 +784,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
        }
 
        pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas);
-       mlx5_ib_populate_pas(cq->buf.umem, page_size, pas, 0);
+       mlx5_ib_populate_pas(cq->ibcq.umem, page_size, pas, 0);
 
        cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context);
        MLX5_SET(cqc, cqc, log_page_size,
@@ -858,7 +857,7 @@ err_db:
        mlx5_ib_db_unmap_user(context, &cq->db);
 
 err_umem:
-       ib_umem_release(cq->buf.umem);
+       /* UMEM is released by ib_core */
        return err;
 }
 
@@ -868,7 +867,6 @@ static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_udata *udata)
                udata, struct mlx5_ib_ucontext, ibucontext);
 
        mlx5_ib_db_unmap_user(context, &cq->db);
-       ib_umem_release(cq->buf.umem);
 }
 
 static void init_cq_frag_buf(struct mlx5_ib_cq_buf *buf)
@@ -949,8 +947,9 @@ static void notify_soft_wc_handler(struct work_struct *work)
        cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
 
-int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
-                     struct uverbs_attr_bundle *attrs)
+int mlx5_ib_create_user_cq(struct ib_cq *ibcq,
+                          const struct ib_cq_init_attr *attr,
+                          struct uverbs_attr_bundle *attrs)
 {
        struct ib_udata *udata = &attrs->driver_udata;
        struct ib_device *ibdev = ibcq->device;
@@ -967,8 +966,7 @@ int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
        int eqn;
        int err;
 
-       if (entries < 0 ||
-           (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))))
+       if (attr->cqe > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))
                return -EINVAL;
 
        if (check_cq_create_flags(attr->flags))
@@ -981,27 +979,15 @@ int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
        cq->ibcq.cqe = entries - 1;
        mutex_init(&cq->resize_mutex);
        spin_lock_init(&cq->lock);
-       cq->resize_buf = NULL;
-       cq->resize_umem = NULL;
        if (attr->flags & IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION)
                cq->private_flags |= MLX5_IB_CQ_PR_TIMESTAMP_COMPLETION;
        INIT_LIST_HEAD(&cq->list_send_qp);
        INIT_LIST_HEAD(&cq->list_recv_qp);
 
-       if (udata) {
-               err = create_cq_user(dev, udata, cq, entries, &cqb, &cqe_size,
-                                    &index, &inlen, attrs);
-               if (err)
-                       return err;
-       } else {
-               cqe_size = cache_line_size() == 128 ? 128 : 64;
-               err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb,
-                                      &index, &inlen);
-               if (err)
-                       return err;
-
-               INIT_WORK(&cq->notify_work, notify_soft_wc_handler);
-       }
+       err = create_cq_user(dev, udata, cq, entries, &cqb, &cqe_size, &index,
+                            &inlen, attrs);
+       if (err)
+               return err;
 
        err = mlx5_comp_eqn_get(dev->mdev, vector, &eqn);
        if (err)
@@ -1021,12 +1007,8 @@ int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
        if (attr->flags & IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN)
                MLX5_SET(cqc, cqc, oi, 1);
 
-       if (udata) {
-               cq->mcq.comp = mlx5_add_cq_to_tasklet;
-               cq->mcq.tasklet_ctx.comp = mlx5_ib_cq_comp;
-       } else {
-               cq->mcq.comp  = mlx5_ib_cq_comp;
-       }
+       cq->mcq.comp = mlx5_add_cq_to_tasklet;
+       cq->mcq.tasklet_ctx.comp = mlx5_ib_cq_comp;
 
        err = mlx5_core_create_cq(dev->mdev, &cq->mcq, cqb, inlen, out, sizeof(out));
        if (err)
@@ -1037,12 +1019,10 @@ int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 
        INIT_LIST_HEAD(&cq->wc_list);
 
-       if (udata)
-               if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
-                       err = -EFAULT;
-                       goto err_cmd;
-               }
-
+       if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
+               err = -EFAULT;
+               goto err_cmd;
+       }
 
        kvfree(cqb);
        return 0;
@@ -1052,10 +1032,82 @@ err_cmd:
 
 err_cqb:
        kvfree(cqb);
-       if (udata)
-               destroy_cq_user(cq, udata);
-       else
-               destroy_cq_kernel(dev, cq);
+       destroy_cq_user(cq, udata);
+       return err;
+}
+
+
+int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+                     struct uverbs_attr_bundle *attrs)
+{
+       struct ib_device *ibdev = ibcq->device;
+       int entries = attr->cqe;
+       int vector = attr->comp_vector;
+       struct mlx5_ib_dev *dev = to_mdev(ibdev);
+       struct mlx5_ib_cq *cq = to_mcq(ibcq);
+       u32 out[MLX5_ST_SZ_DW(create_cq_out)];
+       int index;
+       int inlen;
+       u32 *cqb = NULL;
+       void *cqc;
+       int cqe_size;
+       int eqn;
+       int err;
+
+       if (attr->cqe > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))
+               return -EINVAL;
+
+       entries = roundup_pow_of_two(entries + 1);
+       if (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))
+               return -EINVAL;
+
+       cq->ibcq.cqe = entries - 1;
+       mutex_init(&cq->resize_mutex);
+       spin_lock_init(&cq->lock);
+       INIT_LIST_HEAD(&cq->list_send_qp);
+       INIT_LIST_HEAD(&cq->list_recv_qp);
+
+       cqe_size = cache_line_size() == 128 ? 128 : 64;
+       err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb, &index,
+                              &inlen);
+       if (err)
+               return err;
+
+       INIT_WORK(&cq->notify_work, notify_soft_wc_handler);
+
+       err = mlx5_comp_eqn_get(dev->mdev, vector, &eqn);
+       if (err)
+               goto err_cqb;
+
+       cq->cqe_size = cqe_size;
+
+       cqc = MLX5_ADDR_OF(create_cq_in, cqb, cq_context);
+       MLX5_SET(cqc, cqc, cqe_sz,
+                cqe_sz_to_mlx_sz(cqe_size,
+                                 cq->private_flags &
+                                 MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD));
+       MLX5_SET(cqc, cqc, log_cq_size, ilog2(entries));
+       MLX5_SET(cqc, cqc, uar_page, index);
+       MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn);
+       MLX5_SET64(cqc, cqc, dbr_addr, cq->db.dma);
+
+       cq->mcq.comp = mlx5_ib_cq_comp;
+
+       err = mlx5_core_create_cq(dev->mdev, &cq->mcq, cqb, inlen, out,
+                                 sizeof(out));
+       if (err)
+               goto err_cqb;
+
+       mlx5_ib_dbg(dev, "cqn 0x%x\n", cq->mcq.cqn);
+       cq->mcq.event = mlx5_ib_cq_event;
+
+       INIT_LIST_HEAD(&cq->wc_list);
+       kvfree(cqb);
+       return 0;
+
+err_cqb:
+       kvfree(cqb);
+       destroy_cq_kernel(dev, cq);
        return err;
 }
 
@@ -1390,8 +1442,8 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
 
        if (udata) {
                cq->ibcq.cqe = entries - 1;
-               ib_umem_release(cq->buf.umem);
-               cq->buf.umem = cq->resize_umem;
+               ib_umem_release(cq->ibcq.umem);
+               cq->ibcq.umem = cq->resize_umem;
                cq->resize_umem = NULL;
        } else {
                struct mlx5_ib_cq_buf tbuf;

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 635002e684a55fcc2f6c42127f67abba38809b08..26ee8e763d5e3d66c6e15ba6d6fa758fca7f93a3 100644 (file)
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -4517,6 +4517,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = {
        .check_mr_status = mlx5_ib_check_mr_status,
        .create_ah = mlx5_ib_create_ah,
        .create_cq = mlx5_ib_create_cq,
+       .create_user_cq = mlx5_ib_create_user_cq,
        .create_qp = mlx5_ib_create_qp,
        .create_srq = mlx5_ib_create_srq,
        .create_user_ah = mlx5_ib_create_ah,

diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index ce3372aea48b1715f9e3ba1b5d0c2c03850d9b3b..2556e326afde281026e2d7cb3214dddd3b4713e3 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1371,6 +1371,9 @@ int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer,
                         size_t buflen, size_t *bc);
 int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
                      struct uverbs_attr_bundle *attrs);
+int mlx5_ib_create_user_cq(struct ib_cq *ibcq,
+                          const struct ib_cq_init_attr *attr,
+                          struct uverbs_attr_bundle *attrs);
 int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
 int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
 int mlx5_ib_pre_destroy_cq(struct ib_cq *cq);