RDMA/rxe: Add support for Send/Recv/Write/Read with ODP

rxe_mr_copy() is used widely to copy data to/from a user MR. requester uses
it to load payloads of requesting packets; responder uses it to process
Send, Write, and Read operaetions; completer uses it to copy data from
response packets of Read and Atomic operations to a user MR.

Allow these operations to be used with ODP by adding a subordinate function
rxe_odp_mr_copy(). It is comprised of the following steps:
 1. Check the driver page table(umem_odp->dma_list) to see if pages being
    accessed are present with appropriate permission.
 2. If necessary, trigger page fault to map the pages.
 3. Convert their user space addresses to kernel logical addresses using
    PFNs in the driver page table(umem_odp->pfn_list).
 4. Execute data copy to/from the pages.

Link: https://patch.msgid.link/r/20241220100936.2193541-5-matsuda-daisuke@fujitsu.com
Signed-off-by: Daisuke Matsuda <matsuda-daisuke@fujitsu.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index c08868c7bc55d624210801b6fa7ae4a5a0bfefab..d7daaafa8b3281162671c7fc6fa92e873b255aad 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -98,6 +98,16 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
 
                /* IB_ODP_SUPPORT_IMPLICIT is not supported right now. */
                rxe->attr.odp_caps.general_caps |= IB_ODP_SUPPORT;
+
+               rxe->attr.odp_caps.per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND;
+               rxe->attr.odp_caps.per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_RECV;
+               rxe->attr.odp_caps.per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
+
+               rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND;
+               rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV;
+               rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE;
+               rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ;
+               rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
        }
 }
 

diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index a63fa7ad583b1442ee416db0730d2ce3d3c5789f..ec53ba2bd7d8608f7e3b8de6fecc33ffd1113173 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -187,6 +187,8 @@ extern const struct mmu_interval_notifier_ops rxe_mn_ops;
 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
 int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length,
                         u64 iova, int access_flags, struct rxe_mr *mr);
+int rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
+                   enum rxe_mr_copy_dir dir);
 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 static inline int
 rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
@@ -194,6 +196,11 @@ rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
 {
        return -EOPNOTSUPP;
 }
+static inline int rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
+                                 int length, enum rxe_mr_copy_dir dir)
+{
+       return -EOPNOTSUPP;
+}
 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 
 #endif /* RXE_LOC_H */

diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index 190225a3413985b486f37a09c4859dda852505f7..6cd668a8dfb2fdeb166de204f87d33bcac4a4b5f 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -324,7 +324,7 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
        }
 
        if (mr->umem->is_odp)
-               return -EOPNOTSUPP;
+               return rxe_odp_mr_copy(mr, iova, addr, length, dir);
        else
                return rxe_mr_copy_xarray(mr, iova, addr, length, dir);
 }

diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c
index 1412900787542ef05d706869c15cadd211b26221..d3c67d18c1737adacb016159b94dd753dcee9e61 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_odp.c
+++ b/drivers/infiniband/sw/rxe/rxe_odp.c
@@ -122,3 +122,134 @@ int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length,
 
        return err;
 }
+
+static inline bool rxe_check_pagefault(struct ib_umem_odp *umem_odp,
+                                      u64 iova, int length, u32 perm)
+{
+       bool need_fault = false;
+       u64 addr;
+       int idx;
+
+       addr = iova & (~(BIT(umem_odp->page_shift) - 1));
+
+       /* Skim through all pages that are to be accessed. */
+       while (addr < iova + length) {
+               idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift;
+
+               if (!(umem_odp->dma_list[idx] & perm)) {
+                       need_fault = true;
+                       break;
+               }
+
+               addr += BIT(umem_odp->page_shift);
+       }
+       return need_fault;
+}
+
+static int rxe_odp_map_range_and_lock(struct rxe_mr *mr, u64 iova, int length, u32 flags)
+{
+       struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem);
+       bool need_fault;
+       u64 perm;
+       int err;
+
+       if (unlikely(length < 1))
+               return -EINVAL;
+
+       perm = ODP_READ_ALLOWED_BIT;
+       if (!(flags & RXE_PAGEFAULT_RDONLY))
+               perm |= ODP_WRITE_ALLOWED_BIT;
+
+       mutex_lock(&umem_odp->umem_mutex);
+
+       need_fault = rxe_check_pagefault(umem_odp, iova, length, perm);
+       if (need_fault) {
+               mutex_unlock(&umem_odp->umem_mutex);
+
+               /* umem_mutex is locked on success. */
+               err = rxe_odp_do_pagefault_and_lock(mr, iova, length,
+                                                   flags);
+               if (err < 0)
+                       return err;
+
+               need_fault = rxe_check_pagefault(umem_odp, iova, length, perm);
+               if (need_fault)
+                       return -EFAULT;
+       }
+
+       return 0;
+}
+
+static int __rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
+                            int length, enum rxe_mr_copy_dir dir)
+{
+       struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem);
+       struct page *page;
+       int idx, bytes;
+       size_t offset;
+       u8 *user_va;
+
+       idx = (iova - ib_umem_start(umem_odp)) >> umem_odp->page_shift;
+       offset = iova & (BIT(umem_odp->page_shift) - 1);
+
+       while (length > 0) {
+               u8 *src, *dest;
+
+               page = hmm_pfn_to_page(umem_odp->pfn_list[idx]);
+               user_va = kmap_local_page(page);
+               if (!user_va)
+                       return -EFAULT;
+
+               src = (dir == RXE_TO_MR_OBJ) ? addr : user_va;
+               dest = (dir == RXE_TO_MR_OBJ) ? user_va : addr;
+
+               bytes = BIT(umem_odp->page_shift) - offset;
+               if (bytes > length)
+                       bytes = length;
+
+               memcpy(dest, src, bytes);
+               kunmap_local(user_va);
+
+               length  -= bytes;
+               idx++;
+               offset = 0;
+       }
+
+       return 0;
+}
+
+int rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
+                   enum rxe_mr_copy_dir dir)
+{
+       struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem);
+       u32 flags = 0;
+       int err;
+
+       if (length == 0)
+               return 0;
+
+       if (unlikely(!mr->umem->is_odp))
+               return -EOPNOTSUPP;
+
+       switch (dir) {
+       case RXE_TO_MR_OBJ:
+               break;
+
+       case RXE_FROM_MR_OBJ:
+               flags = RXE_PAGEFAULT_RDONLY;
+               break;
+
+       default:
+               return -EINVAL;
+       }
+
+       err = rxe_odp_map_range_and_lock(mr, iova, length, flags);
+       if (err)
+               return err;
+
+       err =  __rxe_odp_mr_copy(mr, iova, addr, length, dir);
+
+       mutex_unlock(&umem_odp->umem_mutex);
+
+       return err;
+}