RDMA/uverbs: Expose CoCo DMA bounce requirement to userspace

In CoCo guests, guest memory is encrypted and untrusted (T=0) devices
cannot DMA to it directly; such transfers must go through unencrypted
bounce buffers. RDMA registers user pages for direct device access,
bypassing the DMA layer and thus any bouncing, so registered memory does
not work in this configuration.

Until trusted (T=1) device detection is available, conservatively flag
every device attached to a CoCo guest. Expose the condition to userspace
as IB_UVERBS_DEVICE_CC_DMA_BOUNCE in device_cap_flags_ex so applications
can avoid memory registration and fall back to copying buffers through
send/recv.

Link: https://patch.msgid.link/r/20260517141311.2409230-2-jiri@resnulli.us
Signed-off-by: Jiri Pirko <jiri@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index b89efaaa81ec5874e2ff5a7f9970ab3e9db9e3a6..21ada0fe905994421dfa79e8a60bb71b2fc50a2a 100644 (file)
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -42,6 +42,7 @@
 #include <linux/security.h>
 #include <linux/notifier.h>
 #include <linux/hashtable.h>
+#include <linux/cc_platform.h>
 #include <rdma/rdma_netlink.h>
 #include <rdma/ib_addr.h>
 #include <rdma/ib_cache.h>
@@ -1419,6 +1420,14 @@ int ib_register_device(struct ib_device *device, const char *name,
         */
        WARN_ON(dma_device && !dma_device->dma_parms);
        device->dma_device = dma_device;
+       /*
+        * In a CoCo guest every device is currently assumed to be untrusted
+        * (T=0) and therefore subject to DMA bouncing. Once trusted (T=1)
+        * device detection is wired up, narrow this check to exclude such
+        * devices.
+        */
+       if (dma_device && cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
+               device->cc_dma_bounce = 1;
 
        ret = setup_device(device);
        if (ret)

diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 86df7ec83b3a7beca4feea1444f3b84e6bb718d5..10bd7cafd976f7a4e7d22dd04207e36bc982cb19 100644 (file)
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -3584,6 +3584,8 @@ static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)
        resp.timestamp_mask = attr.timestamp_mask;
        resp.hca_core_clock = attr.hca_core_clock;
        resp.device_cap_flags_ex = attr.device_cap_flags;
+       if (ib_dev->cc_dma_bounce)
+               resp.device_cap_flags_ex |= IB_UVERBS_DEVICE_CC_DMA_BOUNCE;
        resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
        resp.rss_caps.max_rwq_indirection_tables =
                attr.rss_caps.max_rwq_indirection_tables;

diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 893cb5b73951a53b1ebc3e42f3186c84ddf5afc3..0daa5089d539fdf0b299e15dd67d4b9e8e328545 100644 (file)
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -275,6 +275,7 @@ enum ib_device_cap_flags {
        IB_DEVICE_FLUSH_GLOBAL = IB_UVERBS_DEVICE_FLUSH_GLOBAL,
        IB_DEVICE_FLUSH_PERSISTENT = IB_UVERBS_DEVICE_FLUSH_PERSISTENT,
        IB_DEVICE_ATOMIC_WRITE = IB_UVERBS_DEVICE_ATOMIC_WRITE,
+       IB_DEVICE_CC_DMA_BOUNCE = IB_UVERBS_DEVICE_CC_DMA_BOUNCE,
 };
 
 enum ib_kernel_cap_flags {
@@ -2950,6 +2951,8 @@ struct ib_device {
        u16                          kverbs_provider:1;
        /* CQ adaptive moderation (RDMA DIM) */
        u16                          use_cq_dim:1;
+       /* CoCo guest with DMA bounce buffering required */
+       u16                          cc_dma_bounce:1;
        u8                           node_type;
        u32                          phys_port_cnt;
        struct ib_device_attr        attrs;

diff --git a/include/uapi/rdma/ib_user_verbs.h b/include/uapi/rdma/ib_user_verbs.h
index 3b7bd99813e9526ead3f9fb124835a0b17e4d2a6..d2aeadb6d2f995b138dd7fec39d92ea00615d0c6 100644 (file)
--- a/include/uapi/rdma/ib_user_verbs.h
+++ b/include/uapi/rdma/ib_user_verbs.h
@@ -1368,6 +1368,8 @@ enum ib_uverbs_device_cap_flags {
        IB_UVERBS_DEVICE_FLUSH_PERSISTENT = 1ULL << 39,
        /* Atomic write attributes */
        IB_UVERBS_DEVICE_ATOMIC_WRITE = 1ULL << 40,
+       /* CoCo guest with DMA bounce buffering required */
+       IB_UVERBS_DEVICE_CC_DMA_BOUNCE = 1ULL << 41,
 };
 
 enum ib_uverbs_raw_packet_caps {