RDMA/mlx5: Enable Data-Direct with Relaxed Ordering

Relaxed Ordering can improve performance in certain scenarios.

Enable it in the Data-Direct use case as well.

Link: https://patch.msgid.link/r/1221dcdda8061ba5f6bc3519044083c7438b257e.1755088503.git.leon@kernel.org
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Reviewed-by: Gal Shalom <galshalom@Nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index d456e4fde3e1feaa7faf0a8cb611684ec5caa10a..505349c68e7938b46a8af2fc5874934944cb3e07 100644 (file)
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -3118,6 +3118,7 @@ mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev *dev)
 {
        int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
        struct mlx5_core_dev *mdev = dev->mdev;
+       bool ro_supp = false;
        void *mkc;
        u32 mkey;
        u32 pdn;
@@ -3146,14 +3147,37 @@ mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev *dev)
        MLX5_SET(mkc, mkc, length64, 1);
        MLX5_SET(mkc, mkc, qpn, 0xffffff);
        err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
-       kvfree(in);
        if (err)
-               goto err;
+               goto err_mkey;
 
        dev->ddr.mkey = mkey;
        dev->ddr.pdn = pdn;
+
+       /* create another mkey with RO support */
+       if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) {
+               MLX5_SET(mkc, mkc, relaxed_ordering_write, 1);
+               ro_supp = true;
+       }
+
+       if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) {
+               MLX5_SET(mkc, mkc, relaxed_ordering_read, 1);
+               ro_supp = true;
+       }
+
+       if (ro_supp) {
+               err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
+               /* RO is defined as best effort */
+               if (!err) {
+                       dev->ddr.mkey_ro = mkey;
+                       dev->ddr.mkey_ro_valid = true;
+               }
+       }
+
+       kvfree(in);
        return 0;
 
+err_mkey:
+       kvfree(in);
 err:
        mlx5_core_dealloc_pd(mdev, pdn);
        return err;
@@ -3162,6 +3186,10 @@ err:
 static void
 mlx5_ib_free_data_direct_resources(struct mlx5_ib_dev *dev)
 {
+
+       if (dev->ddr.mkey_ro_valid)
+               mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro);
+
        mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey);
        mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn);
 }

diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 7ffc7ee92cf03559c7eb546f1eb387e98d374a63..309f256fc27df23cc2b5360fc609b839195a97e4 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -854,6 +854,8 @@ struct mlx5_ib_port_resources {
 struct mlx5_data_direct_resources {
        u32 pdn;
        u32 mkey;
+       u32 mkey_ro;
+       u8 mkey_ro_valid :1;
 };
 
 struct mlx5_ib_resources {

diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 1317f2cb38a4419e59c091b62ad970994d45e558..d3c82ffa300e6ee652474735601a79889e71e1ac 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -1717,11 +1717,11 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
                goto end;
        }
 
-       /* The device's 'data direct mkey' was created without RO flags to
-        * simplify things and allow for a single mkey per device.
-        * Since RO is not a must, mask it out accordingly.
+       /* If no device's 'data direct mkey' with RO flags exists
+        * mask it out accordingly.
         */
-       access_flags &= ~IB_ACCESS_RELAXED_ORDERING;
+       if (!dev->ddr.mkey_ro_valid)
+               access_flags &= ~IB_ACCESS_RELAXED_ORDERING;
        crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev,
                                        offset, length, virt_addr, fd,
                                        access_flags, MLX5_MKC_ACCESS_MODE_KSM,

diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c
index 7ef35cddce81caaeea031ead77a105ff80f26b24..4e562e0dd9e125acb1bcd6558b68def07c271722 100644 (file)
--- a/drivers/infiniband/hw/mlx5/umr.c
+++ b/drivers/infiniband/hw/mlx5/umr.c
@@ -761,7 +761,11 @@ _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd,
 
                if (dd) {
                        cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter));
-                       cur_ksm->key = cpu_to_be32(dev->ddr.mkey);
+                       if (mr->access_flags & IB_ACCESS_RELAXED_ORDERING &&
+                           dev->ddr.mkey_ro_valid)
+                               cur_ksm->key = cpu_to_be32(dev->ddr.mkey_ro);
+                       else
+                               cur_ksm->key = cpu_to_be32(dev->ddr.mkey);
                        if (mr->umem->is_dmabuf &&
                            (flags & MLX5_IB_UPD_XLT_ZAP)) {
                                cur_ksm->va = 0;