io_uring/zcrx: implement device-less mode for zcrx

Allow creating a zcrx instance without attaching it to a net device.
All data will be copied through the fallback path. The user is also
expected to use ZCRX_CTRL_FLUSH_RQ to handle overflows as it normally
should even with a netdev, but it becomes even more relevant as there
will likely be no one to automatically pick up buffers.

Apart from that, it follows the zcrx uapi for the I/O path, and is
useful for testing, experimentation, and potentially for the copy
receive path in the future if improved.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://patch.msgid.link/674f8ad679c5a0bc79d538352b3042cf0999596e.1774261953.git.asml.silence@gmail.com
[axboe: fix spelling error in uapi header and commit message]
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/uapi/linux/io_uring/zcrx.h b/include/uapi/linux/io_uring/zcrx.h
index 3163a4b8aeb0221787bdeb1547b74717d38b0acf..5ce02c7a609690e6d81b31b72250a628c17ce3dc 100644 (file)
--- a/include/uapi/linux/io_uring/zcrx.h
+++ b/include/uapi/linux/io_uring/zcrx.h
@@ -49,7 +49,14 @@ struct io_uring_zcrx_area_reg {
 };
 
 enum zcrx_reg_flags {
-       ZCRX_REG_IMPORT = 1,
+       ZCRX_REG_IMPORT         = 1,
+
+       /*
+        * Register a zcrx instance without a net device. All data will be
+        * copied. The refill queue entries might not be automatically
+        * consumed and need to be flushed, see ZCRX_CTRL_FLUSH_RQ.
+        */
+       ZCRX_REG_NODEV          = 2,
 };
 
 enum zcrx_features {

diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index b0f889b11b7300a6b3d95dcbe2f2112c4138298b..c753f88b6575230c30465de88715c8392893f29d 100644 (file)
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -127,10 +127,10 @@ static int io_import_dmabuf(struct io_zcrx_ifq *ifq,
        int dmabuf_fd = area_reg->dmabuf_fd;
        int i, ret;
 
+       if (!ifq->dev)
+               return -EINVAL;
        if (off)
                return -EINVAL;
-       if (WARN_ON_ONCE(!ifq->dev))
-               return -EFAULT;
        if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
                return -EINVAL;
 
@@ -211,11 +211,13 @@ static int io_import_umem(struct io_zcrx_ifq *ifq,
        if (ret)
                goto out_err;
 
-       ret = dma_map_sgtable(ifq->dev, &mem->page_sg_table,
-                             DMA_FROM_DEVICE, IO_DMA_ATTR);
-       if (ret < 0)
-               goto out_err;
-       mapped = true;
+       if (ifq->dev) {
+               ret = dma_map_sgtable(ifq->dev, &mem->page_sg_table,
+                                     DMA_FROM_DEVICE, IO_DMA_ATTR);
+               if (ret < 0)
+                       goto out_err;
+               mapped = true;
+       }
 
        mem->account_pages = io_count_account_pages(pages, nr_pages);
        ret = io_account_mem(ifq->user, ifq->mm_account, mem->account_pages);
@@ -450,7 +452,8 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
        ret = io_import_area(ifq, &area->mem, area_reg);
        if (ret)
                goto err;
-       area->is_mapped = true;
+       if (ifq->dev)
+               area->is_mapped = true;
 
        if (buf_size_shift > io_area_max_shift(&area->mem)) {
                ret = -ERANGE;
@@ -486,9 +489,11 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
                niov->type = NET_IOV_IOURING;
        }
 
-       ret = io_populate_area_dma(ifq, area);
-       if (ret)
-               goto err;
+       if (ifq->dev) {
+               ret = io_populate_area_dma(ifq, area);
+               if (ret)
+                       goto err;
+       }
 
        area->free_count = nr_iovs;
        /* we're only supporting one area per ifq for now */
@@ -826,6 +831,8 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
                return -EFAULT;
        if (reg.if_rxq == -1 || !reg.rq_entries)
                return -EINVAL;
+       if ((reg.if_rxq || reg.if_idx) && (reg.flags & ZCRX_REG_NODEV))
+               return -EINVAL;
        if (reg.rq_entries > IO_RQ_MAX_ENTRIES) {
                if (!(ctx->flags & IORING_SETUP_CLAMP))
                        return -EINVAL;
@@ -861,9 +868,15 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
        if (ret)
                goto err;
 
-       ret = zcrx_register_netdev(ifq, &reg, &area);
-       if (ret)
-               goto err;
+       if (!(reg.flags & ZCRX_REG_NODEV)) {
+               ret = zcrx_register_netdev(ifq, &reg, &area);
+               if (ret)
+                       goto err;
+       } else {
+               ret = io_zcrx_create_area(ifq, &area, &reg);
+               if (ret)
+                       goto err;
+       }
 
        reg.zcrx_id = id;
 

diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h
index 0316a41a35617e95d2137a17a12089d3add3b1c3..f395656c3160847fd4d9c4d68850f030f4854921 100644 (file)
--- a/io_uring/zcrx.h
+++ b/io_uring/zcrx.h
@@ -8,7 +8,7 @@
 #include <net/page_pool/types.h>
 #include <net/net_trackers.h>
 
-#define ZCRX_SUPPORTED_REG_FLAGS       (ZCRX_REG_IMPORT)
+#define ZCRX_SUPPORTED_REG_FLAGS       (ZCRX_REG_IMPORT | ZCRX_REG_NODEV)
 #define ZCRX_FEATURES                  (ZCRX_FEATURE_RX_PAGE_SIZE)
 
 struct io_zcrx_mem {