From: Chuck Lever Date: Wed, 28 Jan 2026 00:53:56 +0000 (-0500) Subject: RDMA/core: add bio_vec based RDMA read/write API X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5e541553588d493bd9317bc8a8c1ab85cbddc2c5;p=thirdparty%2Fkernel%2Flinux.git RDMA/core: add bio_vec based RDMA read/write API The existing rdma_rw_ctx_init() API requires callers to construct a scatterlist, which is then DMA-mapped page by page. Callers that already have data in bio_vec form (such as the NVMe-oF target) must first convert to scatterlist, adding overhead and complexity. Introduce rdma_rw_ctx_init_bvec() and rdma_rw_ctx_destroy_bvec() to accept bio_vec arrays directly. The new helpers use dma_map_phys() for hardware RDMA devices and virtual addressing for software RDMA devices (rxe, siw), avoiding intermediate scatterlist construction. Memory registration (MR) path support is deferred to a follow-up series; callers requiring MR-based transfers (iWARP devices or force_mr=1) receive -EOPNOTSUPP and should use the scatterlist API. Reviewed-by: Christoph Hellwig Signed-off-by: Chuck Lever Link: https://patch.msgid.link/20260128005400.25147-2-cel@kernel.org Signed-off-by: Leon Romanovsky --- diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c index 6354ddf2a274c..39ca21d18d7b1 100644 --- a/drivers/infiniband/core/rw.c +++ b/drivers/infiniband/core/rw.c @@ -274,6 +274,115 @@ static int rdma_rw_init_single_wr(struct rdma_rw_ctx *ctx, struct ib_qp *qp, return 1; } +static int rdma_rw_init_single_wr_bvec(struct rdma_rw_ctx *ctx, + struct ib_qp *qp, const struct bio_vec *bvecs, + struct bvec_iter *iter, u64 remote_addr, u32 rkey, + enum dma_data_direction dir) +{ + struct ib_device *dev = qp->pd->device; + struct ib_rdma_wr *rdma_wr = &ctx->single.wr; + struct bio_vec bv = mp_bvec_iter_bvec(bvecs, *iter); + u64 dma_addr; + + ctx->nr_ops = 1; + + dma_addr = ib_dma_map_bvec(dev, &bv, dir); + if (ib_dma_mapping_error(dev, dma_addr)) + return -ENOMEM; + + ctx->single.sge.lkey = qp->pd->local_dma_lkey; + ctx->single.sge.addr = dma_addr; + ctx->single.sge.length = bv.bv_len; + + memset(rdma_wr, 0, sizeof(*rdma_wr)); + if (dir == DMA_TO_DEVICE) + rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; + else + rdma_wr->wr.opcode = IB_WR_RDMA_READ; + rdma_wr->wr.sg_list = &ctx->single.sge; + rdma_wr->wr.num_sge = 1; + rdma_wr->remote_addr = remote_addr; + rdma_wr->rkey = rkey; + + ctx->type = RDMA_RW_SINGLE_WR; + return 1; +} + +static int rdma_rw_init_map_wrs_bvec(struct rdma_rw_ctx *ctx, struct ib_qp *qp, + const struct bio_vec *bvecs, u32 nr_bvec, struct bvec_iter *iter, + u64 remote_addr, u32 rkey, enum dma_data_direction dir) +{ + struct ib_device *dev = qp->pd->device; + u32 max_sge = dir == DMA_TO_DEVICE ? qp->max_write_sge : + qp->max_read_sge; + struct ib_sge *sge; + u32 total_len = 0, i, j; + u32 mapped_bvecs = 0; + u32 nr_ops = DIV_ROUND_UP(nr_bvec, max_sge); + size_t sges_size = array_size(nr_bvec, sizeof(*ctx->map.sges)); + size_t wrs_offset = ALIGN(sges_size, __alignof__(*ctx->map.wrs)); + size_t wrs_size = array_size(nr_ops, sizeof(*ctx->map.wrs)); + void *mem; + + if (sges_size == SIZE_MAX || wrs_size == SIZE_MAX || + check_add_overflow(wrs_offset, wrs_size, &wrs_size)) + return -ENOMEM; + + mem = kzalloc(wrs_size, GFP_KERNEL); + if (!mem) + return -ENOMEM; + + ctx->map.sges = sge = mem; + ctx->map.wrs = mem + wrs_offset; + + for (i = 0; i < nr_ops; i++) { + struct ib_rdma_wr *rdma_wr = &ctx->map.wrs[i]; + u32 nr_sge = min(nr_bvec - mapped_bvecs, max_sge); + + if (dir == DMA_TO_DEVICE) + rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; + else + rdma_wr->wr.opcode = IB_WR_RDMA_READ; + rdma_wr->remote_addr = remote_addr + total_len; + rdma_wr->rkey = rkey; + rdma_wr->wr.num_sge = nr_sge; + rdma_wr->wr.sg_list = sge; + + for (j = 0; j < nr_sge; j++) { + struct bio_vec bv = mp_bvec_iter_bvec(bvecs, *iter); + u64 dma_addr; + + dma_addr = ib_dma_map_bvec(dev, &bv, dir); + if (ib_dma_mapping_error(dev, dma_addr)) + goto out_unmap; + + mapped_bvecs++; + sge->addr = dma_addr; + sge->length = bv.bv_len; + sge->lkey = qp->pd->local_dma_lkey; + + total_len += bv.bv_len; + sge++; + + bvec_iter_advance_single(bvecs, iter, bv.bv_len); + } + + rdma_wr->wr.next = i + 1 < nr_ops ? + &ctx->map.wrs[i + 1].wr : NULL; + } + + ctx->nr_ops = nr_ops; + ctx->type = RDMA_RW_MULTI_WR; + return nr_ops; + +out_unmap: + for (i = 0; i < mapped_bvecs; i++) + ib_dma_unmap_bvec(dev, ctx->map.sges[i].addr, + ctx->map.sges[i].length, dir); + kfree(ctx->map.sges); + return -ENOMEM; +} + /** * rdma_rw_ctx_init - initialize a RDMA READ/WRITE context * @ctx: context to initialize @@ -344,6 +453,53 @@ out_unmap_sg: } EXPORT_SYMBOL(rdma_rw_ctx_init); +/** + * rdma_rw_ctx_init_bvec - initialize a RDMA READ/WRITE context from bio_vec + * @ctx: context to initialize + * @qp: queue pair to operate on + * @port_num: port num to which the connection is bound + * @bvecs: bio_vec array to READ/WRITE from/to + * @nr_bvec: number of entries in @bvecs + * @iter: bvec iterator describing offset and length + * @remote_addr: remote address to read/write (relative to @rkey) + * @rkey: remote key to operate on + * @dir: %DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ + * + * Accepts bio_vec arrays directly, avoiding scatterlist conversion for + * callers that already have data in bio_vec form. Prefer this over + * rdma_rw_ctx_init() when the source data is a bio_vec array. + * + * This function does not support devices requiring memory registration. + * iWARP devices and configurations with force_mr=1 should use + * rdma_rw_ctx_init() with a scatterlist instead. + * + * Returns the number of WQEs that will be needed on the workqueue if + * successful, or a negative error code: + * + * * -EINVAL - @nr_bvec is zero or @iter.bi_size is zero + * * -EOPNOTSUPP - device requires MR path (iWARP or force_mr=1) + * * -ENOMEM - DMA mapping or memory allocation failed + */ +int rdma_rw_ctx_init_bvec(struct rdma_rw_ctx *ctx, struct ib_qp *qp, + u32 port_num, const struct bio_vec *bvecs, u32 nr_bvec, + struct bvec_iter iter, u64 remote_addr, u32 rkey, + enum dma_data_direction dir) +{ + if (nr_bvec == 0 || iter.bi_size == 0) + return -EINVAL; + + /* MR path not supported for bvec - reject iWARP and force_mr */ + if (rdma_rw_io_needs_mr(qp->device, port_num, dir, nr_bvec)) + return -EOPNOTSUPP; + + if (nr_bvec == 1) + return rdma_rw_init_single_wr_bvec(ctx, qp, bvecs, &iter, + remote_addr, rkey, dir); + return rdma_rw_init_map_wrs_bvec(ctx, qp, bvecs, nr_bvec, &iter, + remote_addr, rkey, dir); +} +EXPORT_SYMBOL(rdma_rw_ctx_init_bvec); + /** * rdma_rw_ctx_signature_init - initialize a RW context with signature offload * @ctx: context to initialize @@ -598,6 +754,47 @@ void rdma_rw_ctx_destroy(struct rdma_rw_ctx *ctx, struct ib_qp *qp, } EXPORT_SYMBOL(rdma_rw_ctx_destroy); +/** + * rdma_rw_ctx_destroy_bvec - release resources from rdma_rw_ctx_init_bvec + * @ctx: context to release + * @qp: queue pair to operate on + * @port_num: port num to which the connection is bound (unused) + * @bvecs: bio_vec array that was used for the READ/WRITE (unused) + * @nr_bvec: number of entries in @bvecs + * @dir: %DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ + * + * Releases all resources allocated by a successful rdma_rw_ctx_init_bvec() + * call. Must not be called if rdma_rw_ctx_init_bvec() returned an error. + * + * The @port_num and @bvecs parameters are unused but present for API + * symmetry with rdma_rw_ctx_destroy(). + */ +void rdma_rw_ctx_destroy_bvec(struct rdma_rw_ctx *ctx, struct ib_qp *qp, + u32 __maybe_unused port_num, + const struct bio_vec __maybe_unused *bvecs, + u32 nr_bvec, enum dma_data_direction dir) +{ + struct ib_device *dev = qp->pd->device; + u32 i; + + switch (ctx->type) { + case RDMA_RW_MULTI_WR: + for (i = 0; i < nr_bvec; i++) + ib_dma_unmap_bvec(dev, ctx->map.sges[i].addr, + ctx->map.sges[i].length, dir); + kfree(ctx->map.sges); + break; + case RDMA_RW_SINGLE_WR: + ib_dma_unmap_bvec(dev, ctx->single.sge.addr, + ctx->single.sge.length, dir); + break; + default: + WARN_ON_ONCE(1); + return; + } +} +EXPORT_SYMBOL(rdma_rw_ctx_destroy_bvec); + /** * rdma_rw_ctx_destroy_signature - release all resources allocated by * rdma_rw_ctx_signature_init diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 6c372a37c4824..8bd020da77453 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -4266,6 +4267,47 @@ static inline void ib_dma_unmap_page(struct ib_device *dev, dma_unmap_page(dev->dma_device, addr, size, direction); } +/** + * ib_dma_map_bvec - Map a bio_vec to DMA address + * @dev: The device for which the dma_addr is to be created + * @bvec: The bio_vec to map + * @direction: The direction of the DMA + * + * Returns a DMA address for the bio_vec. The caller must check the + * result with ib_dma_mapping_error() before use; a failed mapping + * must not be passed to ib_dma_unmap_bvec(). + * + * For software RDMA devices (rxe, siw), returns a virtual address + * and no actual DMA mapping occurs. + */ +static inline u64 ib_dma_map_bvec(struct ib_device *dev, + struct bio_vec *bvec, + enum dma_data_direction direction) +{ + if (ib_uses_virt_dma(dev)) + return (uintptr_t)bvec_virt(bvec); + return dma_map_phys(dev->dma_device, bvec_phys(bvec), + bvec->bv_len, direction, 0); +} + +/** + * ib_dma_unmap_bvec - Unmap a bio_vec DMA mapping + * @dev: The device for which the DMA address was created + * @addr: The DMA address returned by ib_dma_map_bvec() + * @size: The size of the region in bytes + * @direction: The direction of the DMA + * + * Releases a DMA mapping created by ib_dma_map_bvec(). For software + * RDMA devices this is a no-op since no actual mapping occurred. + */ +static inline void ib_dma_unmap_bvec(struct ib_device *dev, + u64 addr, size_t size, + enum dma_data_direction direction) +{ + if (!ib_uses_virt_dma(dev)) + dma_unmap_phys(dev->dma_device, addr, size, direction, 0); +} + int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents); static inline int ib_dma_map_sg_attrs(struct ib_device *dev, struct scatterlist *sg, int nents, diff --git a/include/rdma/rw.h b/include/rdma/rw.h index d606cac482338..b2fc3e2373d70 100644 --- a/include/rdma/rw.h +++ b/include/rdma/rw.h @@ -5,6 +5,7 @@ #ifndef _RDMA_RW_H #define _RDMA_RW_H +#include #include #include #include @@ -49,6 +50,16 @@ void rdma_rw_ctx_destroy(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num, struct scatterlist *sg, u32 sg_cnt, enum dma_data_direction dir); +struct bio_vec; + +int rdma_rw_ctx_init_bvec(struct rdma_rw_ctx *ctx, struct ib_qp *qp, + u32 port_num, const struct bio_vec *bvecs, u32 nr_bvec, + struct bvec_iter iter, u64 remote_addr, u32 rkey, + enum dma_data_direction dir); +void rdma_rw_ctx_destroy_bvec(struct rdma_rw_ctx *ctx, struct ib_qp *qp, + u32 port_num, const struct bio_vec *bvecs, u32 nr_bvec, + enum dma_data_direction dir); + int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num, struct scatterlist *sg, u32 sg_cnt, struct scatterlist *prot_sg, u32 prot_sg_cnt,