qemu: Add support for HW-accelerated nested SMMUv3

Add support for enabling HW-accelerated nested SMMUv3 via <accel>
attribute and its additional attributes for ATS, SSID, RIL, and OAS
configuration.

Validate the domain iommu accel attribute in qemu_validate.c when
QEMU lacks this capability. Checking accel is sufficient because ats,
ril, ssidsize, and oas were introduced in the same QEMU release, and
it only makes sense to backport accel, ats, ril, ssidsize, and oas
all together.

Reviewed-by: Pavel Hrdina <phrdina@redhat.com>
Signed-off-by: Nathan Chen <nathanc@nvidia.com>

diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
index 50618bf05b2ecf22985c4b004de3f079f40a6e5f..0b346a0952d1be886442bf7cc2e5a5f226016e22 100644 (file)
--- a/docs/formatdomain.rst
+++ b/docs/formatdomain.rst
@@ -9430,6 +9430,44 @@ Examples:
       IOMMU device is attached to.
       :since:`Since 11.10.0` (QEMU/KVM and ``smmuv3`` model only)
 
+   ``accel``
+      The ``accel`` attribute with possible values ``on`` and ``off`` can
+      be used to enable hardware acceleration support for smmuv3 IOMMU
+      devices.
+      :since:`Since 12.7.0` (QEMU/KVM and ``smmuv3`` model only)
+
+   ``ats``
+      The ``ats`` attribute with possible values ``on`` and ``off`` can be
+      used to enable reporting Address Translation Services capability to
+      the guest for smmuv3 IOMMU devices with ``accel`` set to ``on``, if
+      the host SMMUv3 supports ATS and the associated passthrough device
+      supports ATS. If ``accel`` is enabled and ``ats`` is not configured,
+      the hypervisor default value is used.
+      :since:`Since 12.7.0` (QEMU/KVM and ``smmuv3`` model only)
+
+   ``ril``
+      The ``ril`` attribute with possible values ``on`` and ``off`` can be
+      used to report whether Range Invalidation for IOMMU devices with
+      ``accel`` set to ``on`` is compatible with host SMMUv3 support. If
+      ``accel`` is enabled and ``ril`` is not configured, the hypervisor
+      default value is used.
+      :since:`Since 12.7.0` (QEMU/KVM and ``smmuv3`` model only)
+
+   ``ssidsize``
+      The ``ssidsize`` attribute sets the number of bits used to
+      represent SubstreamIDs. A value of N allows SSIDs in the range
+      [0 .. 2^N - 1]. The valid range is 0-20, and a value greater than 0
+      is required for enabling PASID support, as doing so advertises PASID
+      capability to the vIOMMU. If ``accel`` is enabled and ``ssidsize`` is
+      not configured, the hypervisor default value is used.
+      :since:`Since 12.7.0` (QEMU/KVM and ``smmuv3`` model only)
+
+   ``oas``
+      The ``oas`` attribute sets the output address size in units of bits.
+      If ``accel`` is enabled and ``oas`` is not configured, the hypervisor
+      default value is used.
+      :since:`Since 12.7.0` (QEMU/KVM and ``smmuv3`` model only)
+
 In case of ``virtio`` IOMMU device, the ``driver`` element can optionally
 contain ``granule`` subelement that allows to choose which granule will be
 used by default. It is useful when running guests with different page size

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 0b48fe3010c556a21ddd580360e406cd8ca5cb2f..9880339e423d5b58a2df7f7efc9bcc4e4d715a94 100644 (file)
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -2864,6 +2864,10 @@ virDomainIOMMUDefNew(void)
 
     iommu->pci_bus = -1;
 
+    iommu->ssid_size = -1;
+
+    iommu->oas = -1;
+
     return g_steal_pointer(&iommu);
 }
 
@@ -14750,6 +14754,26 @@ virDomainIOMMUDefParseXML(virDomainXMLOption *xmlopt,
                           &iommu->pci_bus, -1) < 0)
             return NULL;
 
+        if (virXMLPropTristateSwitch(driver, "accel", VIR_XML_PROP_NONE,
+                                     &iommu->accel) < 0)
+            return NULL;
+
+        if (virXMLPropTristateSwitch(driver, "ats", VIR_XML_PROP_NONE,
+                                     &iommu->ats) < 0)
+            return NULL;
+
+        if (virXMLPropTristateSwitch(driver, "ril", VIR_XML_PROP_NONE,
+                                     &iommu->ril) < 0)
+            return NULL;
+
+        if (virXMLPropInt(driver, "ssidsize", 10, VIR_XML_PROP_NONE,
+                          &iommu->ssid_size, -1) < 0)
+            return NULL;
+
+        if (virXMLPropInt(driver, "oas", 10, VIR_XML_PROP_NONE,
+                          &iommu->oas, -1) < 0)
+            return NULL;
+
         if ((granule = virXPathNode("./driver/granule", ctxt))) {
             g_autofree char *mode = virXMLPropString(granule, "mode");
             unsigned long long size;
@@ -16854,6 +16878,11 @@ virDomainIOMMUDefEquals(const virDomainIOMMUDef *a,
         a->aw_bits != b->aw_bits ||
         a->dma_translation != b->dma_translation ||
         a->pci_bus != b->pci_bus ||
+        a->accel != b->accel ||
+        a->ats != b->ats ||
+        a->ril != b->ril ||
+        a->ssid_size != b->ssid_size ||
+        a->oas != b->oas ||
         a->xtsup != b->xtsup ||
         a->pt != b->pt ||
         a->granule != b->granule)
@@ -22738,6 +22767,36 @@ virDomainIOMMUDefCheckABIStability(virDomainIOMMUDef *src,
                        dst->pci_bus, src->pci_bus);
         return false;
     }
+    if (src->accel != dst->accel) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("Target domain IOMMU device accel value '%1$d' does not match source '%2$d'"),
+                       dst->accel, src->accel);
+        return false;
+    }
+    if (src->ats != dst->ats) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("Target domain IOMMU device ATS value '%1$d' does not match source '%2$d'"),
+                       dst->ats, src->ats);
+        return false;
+    }
+    if (src->ril != dst->ril) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("Target domain IOMMU device ril value '%1$d' does not match source '%2$d'"),
+                       dst->ril, src->ril);
+        return false;
+    }
+    if (src->ssid_size != dst->ssid_size) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("Target domain IOMMU device ssid_size value '%1$d' does not match source '%2$d'"),
+                       dst->ssid_size, src->ssid_size);
+        return false;
+    }
+    if (src->oas != dst->oas) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("Target domain IOMMU device oas value '%1$d' does not match source '%2$d'"),
+                       dst->oas, src->oas);
+        return false;
+    }
     if (src->dma_translation != dst->dma_translation) {
         virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                        _("Target domain IOMMU device dma translation '%1$s' does not match source '%2$s'"),
@@ -29208,6 +29267,26 @@ virDomainIOMMUDefFormat(virBuffer *buf,
         virBufferAsprintf(&driverAttrBuf, " pciBus='%d'",
                           iommu->pci_bus);
     }
+    if (iommu->accel != VIR_TRISTATE_SWITCH_ABSENT) {
+        virBufferAsprintf(&driverAttrBuf, " accel='%s'",
+                          virTristateSwitchTypeToString(iommu->accel));
+    }
+    if (iommu->ats != VIR_TRISTATE_SWITCH_ABSENT) {
+        virBufferAsprintf(&driverAttrBuf, " ats='%s'",
+                          virTristateSwitchTypeToString(iommu->ats));
+    }
+    if (iommu->ril != VIR_TRISTATE_SWITCH_ABSENT) {
+        virBufferAsprintf(&driverAttrBuf, " ril='%s'",
+                          virTristateSwitchTypeToString(iommu->ril));
+    }
+    if (iommu->ssid_size >= 0) {
+        virBufferAsprintf(&driverAttrBuf, " ssidsize='%d'",
+                          iommu->ssid_size);
+    }
+    if (iommu->oas >= 0) {
+        virBufferAsprintf(&driverAttrBuf, " oas='%d'",
+                          iommu->oas);
+    }
     if (iommu->granule != 0) {
         if (iommu->granule == -1) {
             virBufferAddLit(&driverChildBuf, "<granule mode='host'/>\n");

diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 91f57de0f13b17a295b75729d474af9644ec6cd6..2c017738e3e77e9514f148310c6ed597014dc0fa 100644 (file)
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -3105,6 +3105,11 @@ struct _virDomainIOMMUDef {
     virTristateSwitch xtsup;
     virTristateSwitch pt;
     int granule; /* -1 means 'host', 0 unset, page size in KiB otherwise */
+    virTristateSwitch accel;
+    virTristateSwitch ats;
+    virTristateSwitch ril;
+    int ssid_size;
+    int oas;
 };
 
 typedef enum {

diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
index 3946f921829808876f795600abb41660833355c1..710a8919042f81bb382076c7c226a1a864814744 100644 (file)
--- a/src/conf/domain_validate.c
+++ b/src/conf/domain_validate.c
@@ -3203,6 +3203,16 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
                            virDomainIOMMUModelTypeToString(iommu->model));
             return -1;
         }
+        if (iommu->accel != VIR_TRISTATE_SWITCH_ON &&
+            (iommu->ats != VIR_TRISTATE_SWITCH_ABSENT ||
+             iommu->ril != VIR_TRISTATE_SWITCH_ABSENT ||
+             iommu->ssid_size >= 0 ||
+             iommu->oas >= 0)) {
+            virReportError(VIR_ERR_XML_ERROR,
+                           _("accel must be enabled for iommu model '%1$s' when setting ats, ril, ssidsize, or oas"),
+                           virDomainIOMMUModelTypeToString(iommu->model));
+            return -1;
+        }
         break;
     case VIR_DOMAIN_IOMMU_MODEL_VIRTIO:
         if (iommu->intremap != VIR_TRISTATE_SWITCH_ABSENT ||
@@ -3210,7 +3220,12 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
             iommu->eim != VIR_TRISTATE_SWITCH_ABSENT ||
             iommu->iotlb != VIR_TRISTATE_SWITCH_ABSENT ||
             iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT ||
-            iommu->pci_bus >= 0) {
+            iommu->pci_bus >= 0 ||
+            iommu->accel != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ats != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ril != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ssid_size >= 0 ||
+            iommu->oas >= 0) {
             virReportError(VIR_ERR_XML_ERROR,
                            _("iommu model '%1$s' doesn't support some additional attributes"),
                            virDomainIOMMUModelTypeToString(iommu->model));
@@ -3233,6 +3248,11 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
             iommu->aw_bits != 0 ||
             iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT ||
             iommu->pci_bus >= 0 ||
+            iommu->accel != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ats != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ril != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ssid_size >= 0 ||
+            iommu->oas >= 0 ||
             iommu->granule != 0) {
             virReportError(VIR_ERR_XML_ERROR,
                            _("iommu model '%1$s' doesn't support some additional attributes"),
@@ -3245,6 +3265,11 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
         if (iommu->pt != VIR_TRISTATE_SWITCH_ABSENT ||
             iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT ||
             iommu->pci_bus >= 0 ||
+            iommu->accel != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ats != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ril != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ssid_size >= 0 ||
+            iommu->oas >= 0 ||
             iommu->granule != 0) {
             virReportError(VIR_ERR_XML_ERROR,
                            _("iommu model '%1$s' doesn't support some additional attributes"),

diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
index 81ffbfc2fa32e3b8f1825dbf1d34367bd2f72472..c8f1fe5547498ec3d97a53d705cbcff44693bb7f 100644 (file)
--- a/src/conf/schemas/domaincommon.rng
+++ b/src/conf/schemas/domaincommon.rng
@@ -6456,6 +6456,31 @@
                 <empty/>
               </element>
             </optional>
+            <optional>
+              <attribute name="accel">
+                <ref name="virOnOff"/>
+              </attribute>
+            </optional>
+            <optional>
+              <attribute name="ats">
+                <ref name="virOnOff"/>
+              </attribute>
+            </optional>
+            <optional>
+              <attribute name="ril">
+                <ref name="virOnOff"/>
+              </attribute>
+            </optional>
+            <optional>
+              <attribute name="ssidsize">
+                <data type="int"/>
+              </attribute>
+            </optional>
+            <optional>
+              <attribute name="oas">
+                <data type="int"/>
+              </attribute>
+            </optional>
           </element>
         </optional>
         <optional>

diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index a76a034158e819fa77e2f7a36f6000667c4d065e..980f0495b147996c4d1ea65d27cf85d05fe91bea 100644 (file)
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -6282,6 +6282,8 @@ qemuBuildPCINestedSmmuv3DevProps(const virDomainDef *def,
 {
     g_autoptr(virJSONValue) props = NULL;
     g_autofree char *bus = NULL;
+    g_autofree char *ssidsizeStr = NULL;
+    g_autofree char *oasStr = NULL;
     virPCIDeviceAddress addr = { .bus = iommu->pci_bus };
 
     bus = qemuBuildDeviceAddressPCIGetBus(def, &addr);
@@ -6292,10 +6294,23 @@ qemuBuildPCINestedSmmuv3DevProps(const virDomainDef *def,
         return NULL;
     }
 
+    if (iommu->ssid_size >= 0) {
+        ssidsizeStr = g_strdup_printf("%u", iommu->ssid_size);
+    }
+
+    if (iommu->oas >= 0) {
+        oasStr = g_strdup_printf("%u", iommu->oas);
+    }
+
     if (virJSONValueObjectAdd(&props,
                               "s:driver", "arm-smmuv3",
                               "s:primary-bus", bus,
                               "s:id", iommu->info.alias,
+                              "T:accel", iommu->accel,
+                              "S:ats", qemuOnOffAuto(iommu->ats),
+                              "S:ril", qemuOnOffAuto(iommu->ril),
+                              "S:ssidsize", ssidsizeStr,
+                              "S:oas", oasStr,
                               NULL) < 0)
         return NULL;
 

diff --git a/src/qemu/qemu_validate.c b/src/qemu/qemu_validate.c
index 3d2658c929d3b07d43dfe553ae48e324d44315de..61f3bd32782b2c27280852ab638a18ff8b5bfa3f 100644 (file)
--- a/src/qemu/qemu_validate.c
+++ b/src/qemu/qemu_validate.c
@@ -5817,6 +5817,18 @@ qemuValidateDomainDeviceDefIOMMU(const virDomainIOMMUDef *iommu,
         return -1;
     }
 
+    /* While QEMU_CAPS_ARM_SMMUV3_ACCEL tracks the .accel attribute of
+     * arm-smmuv3 it is also a good indicator of .ats, .ril, .ssidsize, and
+     * .oas attributes as all of them were introduced in the same release,
+     * and these features are meant to be backported all together. */
+    if (iommu->model == VIR_DOMAIN_IOMMU_MODEL_SMMUV3 &&
+        iommu->accel != VIR_TRISTATE_SWITCH_ABSENT &&
+        !virQEMUCapsGet(qemuCaps, QEMU_CAPS_ARM_SMMUV3_ACCEL)) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                       _("iommu: accel is not supported with this QEMU binary"));
+        return -1;
+    }
+
     if (iommu->granule > 0) {
         /* QEMU supports only 4KiB, 8KiB, 16KiB and 64KiB granule size */
         if (!(iommu->granule == 4 ||