RDMA/rxe: Enable ODP in ATOMIC WRITE operation

Add rxe_odp_do_atomic_write() so that ODP specific steps are applied to
ATOMIC WRITE requests.

Signed-off-by: Daisuke Matsuda <matsuda-daisuke@fujitsu.com>
Link: https://patch.msgid.link/20250324075649.3313968-3-matsuda-daisuke@fujitsu.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index bf711eb9440cd038e981a2ae79f1fab3707716fb..3a77d6db17202d4ee561979b5b18c8d9ced7f819 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -102,6 +102,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe, struct net_device *ndev)
                rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC;
                rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
                rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_FLUSH;
+               rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC_WRITE;
        }
 }
 

diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index 0012bebe96ef3e7f10935543c2877f056b6e167f..6389d829b06a5387828ef7519b431a26c73b91b3 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -187,7 +187,7 @@ static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp)
 /* rxe_odp.c */
 extern const struct mmu_interval_notifier_ops rxe_mn_ops;
 
-#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
+#if defined CONFIG_INFINIBAND_ON_DEMAND_PAGING
 int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length,
                         u64 iova, int access_flags, struct rxe_mr *mr);
 int rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
@@ -221,4 +221,14 @@ static inline int rxe_odp_flush_pmem_iova(struct rxe_mr *mr, u64 iova,
 }
 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 
+#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
+enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value);
+#else
+static inline enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr,
+                                                      u64 iova, u64 value)
+{
+       return RESPST_ERR_UNSUPPORTED_OPCODE;
+}
+#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
+
 #endif /* RXE_LOC_H */

diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index 93e4b5acd3ac3af5855bf566b080ea2951ddee96..d40fbe10633f327c12696502c7b1cbca74fdf846 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -547,16 +547,6 @@ int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
        struct page *page;
        u64 *va;
 
-       /* ODP is not supported right now. WIP. */
-       if (mr->umem->is_odp)
-               return RESPST_ERR_UNSUPPORTED_OPCODE;
-
-       /* See IBA oA19-28 */
-       if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
-               rxe_dbg_mr(mr, "mr not in valid state\n");
-               return RESPST_ERR_RKEY_VIOLATION;
-       }
-
        if (mr->ibmr.type == IB_MR_TYPE_DMA) {
                page_offset = iova & (PAGE_SIZE - 1);
                page = ib_virt_dma_to_page(iova);
@@ -584,10 +574,8 @@ int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
        }
 
        va = kmap_local_page(page);
-
        /* Do atomic write after all prior operations have completed */
        smp_store_release(&va[page_offset >> 3], value);
-
        kunmap_local(va);
 
        return 0;

diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c
index 9a9aae967486409ebc4013857f44ca2d09970fc0..02de05d759c61fb95e14d72f53bd7221d24444d2 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_odp.c
+++ b/drivers/infiniband/sw/rxe/rxe_odp.c
@@ -378,3 +378,49 @@ int rxe_odp_flush_pmem_iova(struct rxe_mr *mr, u64 iova,
 
        return 0;
 }
+
+/* CONFIG_64BIT=y */
+enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
+{
+       struct ib_umem_odp *umem_odp = to_ib_umem_odp(mr->umem);
+       unsigned int page_offset;
+       unsigned long index;
+       struct page *page;
+       int err;
+       u64 *va;
+
+       /* See IBA oA19-28 */
+       err = mr_check_range(mr, iova, sizeof(value));
+       if (unlikely(err)) {
+               rxe_dbg_mr(mr, "iova out of range\n");
+               return RESPST_ERR_RKEY_VIOLATION;
+       }
+
+       err = rxe_odp_map_range_and_lock(mr, iova, sizeof(value),
+                                        RXE_PAGEFAULT_DEFAULT);
+       if (err)
+               return RESPST_ERR_RKEY_VIOLATION;
+
+       page_offset = rxe_odp_iova_to_page_offset(umem_odp, iova);
+       index = rxe_odp_iova_to_index(umem_odp, iova);
+       page = hmm_pfn_to_page(umem_odp->pfn_list[index]);
+       if (!page) {
+               mutex_unlock(&umem_odp->umem_mutex);
+               return RESPST_ERR_RKEY_VIOLATION;
+       }
+       /* See IBA A19.4.2 */
+       if (unlikely(page_offset & 0x7)) {
+               mutex_unlock(&umem_odp->umem_mutex);
+               rxe_dbg_mr(mr, "misaligned address\n");
+               return RESPST_ERR_MISALIGNED_ATOMIC;
+       }
+
+       va = kmap_local_page(page);
+       /* Do atomic write after all prior operations have completed */
+       smp_store_release(&va[page_offset >> 3], value);
+       kunmap_local(va);
+
+       mutex_unlock(&umem_odp->umem_mutex);
+
+       return RESPST_NONE;
+}

diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index 304e3de740ad3d12336ebc6bccb36bdf49256f32..fd7bac5bce1809e6527caa19ae79c29b1f8a0227 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -749,7 +749,16 @@ static enum resp_states atomic_write_reply(struct rxe_qp *qp,
        value = *(u64 *)payload_addr(pkt);
        iova = qp->resp.va + qp->resp.offset;
 
-       err = rxe_mr_do_atomic_write(mr, iova, value);
+       /* See IBA oA19-28 */
+       if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
+               rxe_dbg_mr(mr, "mr not in valid state\n");
+               return RESPST_ERR_RKEY_VIOLATION;
+       }
+
+       if (mr->umem->is_odp)
+               err = rxe_odp_do_atomic_write(mr, iova, value);
+       else
+               err = rxe_mr_do_atomic_write(mr, iova, value);
        if (err)
                return err;
 

diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 41bf98ccb27572e1423390e231c01bc6e5dc9f8d..0a7ccd08b3659a7d5a63a2cee1c60829cb68c211 100644 (file)
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -326,6 +326,7 @@ enum ib_odp_transport_cap_bits {
        IB_ODP_SUPPORT_ATOMIC   = 1 << 4,
        IB_ODP_SUPPORT_SRQ_RECV = 1 << 5,
        IB_ODP_SUPPORT_FLUSH    = 1 << 6,
+       IB_ODP_SUPPORT_ATOMIC_WRITE     = 1 << 7,
 };
 
 struct ib_odp_caps {