qemu: open VFIO FDs from libvirt backend

Open VFIO FDs from libvirt backend without exposing
these FDs to XML users, i.e. one per iommufd hostdev
for /dev/vfio/devices/vfioX, and pass the FD to qemu
command line.

Suggested-by: Ján Tomko <jtomko@redhat.com>
Signed-off-by: Nathan Chen <nathanc@nvidia.com>
Reviewed-by: Pavel Hrdina <phrdina@redhat.com>

diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 7fa76a1ec36afedd45f03ea8e8f9900668865646..d81b30f0b69c4eef6a3e429dcd607492bbc3288d 100644 (file)
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -3156,6 +3156,7 @@ virPCIDeviceGetStubDriverName;
 virPCIDeviceGetStubDriverType;
 virPCIDeviceGetUnbindFromStub;
 virPCIDeviceGetUsedBy;
+virPCIDeviceGetVfioPath;
 virPCIDeviceGetVPD;
 virPCIDeviceHasPCIExpressLink;
 virPCIDeviceIsAssignable;

diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index 9066149df5793a2fe7f97f3502b0e93dd31e58d0..1bc2007cbd660e1740d1b529c831ca73bb240c0a 100644 (file)
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -4810,6 +4810,18 @@ qemuBuildPCIHostdevDevProps(const virDomainDef *def,
                               NULL) < 0)
         return NULL;
 
+    if (pcisrc->driver.name == VIR_DEVICE_HOSTDEV_PCI_DRIVER_NAME_VFIO &&
+        pcisrc->driver.iommufd == VIR_TRISTATE_BOOL_YES) {
+        qemuDomainHostdevPrivate *hostdevPriv = QEMU_DOMAIN_HOSTDEV_PRIVATE(dev);
+
+        if (hostdevPriv->vfioDeviceFd != -1) {
+            g_autofree char *fdstr = g_strdup_printf("%d", hostdevPriv->vfioDeviceFd);
+            if (virJSONValueObjectAdd(&props, "S:fd", fdstr, NULL) < 0)
+                return NULL;
+            hostdevPriv->vfioDeviceFd = -1;
+        }
+    }
+
     if (qemuBuildDeviceAddressProps(props, def, dev->info) < 0)
         return NULL;
 
@@ -5254,6 +5266,15 @@ qemuBuildHostdevCommandLine(virCommand *cmd,
             if (qemuCommandAddExtDevice(cmd, hostdev->info, def, qemuCaps) < 0)
                 return -1;
 
+            if (subsys->u.pci.driver.iommufd == VIR_TRISTATE_BOOL_YES) {
+                qemuDomainHostdevPrivate *hostdevPriv = QEMU_DOMAIN_HOSTDEV_PRIVATE(hostdev);
+
+                if (hostdevPriv->vfioDeviceFd != -1) {
+                    virCommandPassFD(cmd, hostdevPriv->vfioDeviceFd,
+                                     VIR_COMMAND_PASS_FD_CLOSE_PARENT);
+                }
+            }
+
             if (!(devprops = qemuBuildPCIHostdevDevProps(def, hostdev)))
                 return -1;
 

diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index a53bb40783f514d746cec90ba9a62662dc749d7d..2841856454accbefc1f411cc21d308a5df3fb2dd 100644 (file)
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -103,6 +103,7 @@
 #include "storage_source.h"
 #include "backup_conf.h"
 #include "storage_file_probe.h"
+#include "virpci.h"
 
 #include "logging/log_manager.h"
 #include "logging/log_protocol.h"
@@ -7670,6 +7671,81 @@ qemuProcessPrepareHostBackendChardevHotplug(virDomainObj *vm,
     return 0;
 }
 
+/**
+ * qemuProcessOpenVfioDeviceFd:
+ * @hostdev: host device definition
+ * @vfioFd: returned file descriptor
+ *
+ * Opens the VFIO device file descriptor for a hostdev.
+ *
+ * Returns: FD on success, -1 on failure
+ */
+static int
+qemuProcessOpenVfioDeviceFd(virDomainHostdevDef *hostdev)
+{
+    g_autofree char *vfioPath = NULL;
+    int fd = -1;
+
+    if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS ||
+        hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("VFIO FD only supported for PCI hostdevs"));
+        return -1;
+    }
+
+    if (virPCIDeviceGetVfioPath(&hostdev->source.subsys.u.pci.addr, &vfioPath) < 0)
+        return -1;
+
+    VIR_DEBUG("Opening VFIO device %s", vfioPath);
+
+    if ((fd = open(vfioPath, O_RDWR | O_CLOEXEC)) < 0) {
+        if (errno == ENOENT) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                           _("VFIO device %1$s not found - ensure device is bound to vfio-pci driver"),
+                           vfioPath);
+        } else {
+            virReportSystemError(errno,
+                                 _("cannot open VFIO device %1$s"), vfioPath);
+        }
+        return -1;
+    }
+
+    VIR_DEBUG("Opened VFIO device FD %d for %s", fd, vfioPath);
+    return fd;
+}
+
+/**
+ * qemuProcessOpenVfioFds:
+ * @vm: domain object
+ *
+ * Opens all necessary VFIO file descriptors for the domain.
+ *
+ * Returns: 0 on success, -1 on failure
+ */
+static int
+qemuProcessOpenVfioFds(virDomainObj *vm)
+{
+    size_t i;
+
+    /* Check if we have any hostdevs that need VFIO FDs */
+    for (i = 0; i < vm->def->nhostdevs; i++) {
+        virDomainHostdevDef *hostdev = vm->def->hostdevs[i];
+        qemuDomainHostdevPrivate *hostdevPriv = QEMU_DOMAIN_HOSTDEV_PRIVATE(hostdev);
+
+        if (hostdev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS &&
+            hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI &&
+            hostdev->source.subsys.u.pci.driver.name == VIR_DEVICE_HOSTDEV_PCI_DRIVER_NAME_VFIO &&
+            hostdev->source.subsys.u.pci.driver.iommufd == VIR_TRISTATE_BOOL_YES) {
+            /* Open VFIO device FD */
+            hostdevPriv->vfioDeviceFd = qemuProcessOpenVfioDeviceFd(hostdev);
+            if (hostdevPriv->vfioDeviceFd == -1)
+                 return -1;
+        }
+    }
+
+    return 0;
+}
+
 /**
  * qemuProcessPrepareHost:
  * @driver: qemu driver
@@ -7725,6 +7801,8 @@ qemuProcessPrepareHost(virQEMUDriver *driver,
         hostdev_flags |= VIR_HOSTDEV_COLD_BOOT;
     if (qemuHostdevPrepareDomainDevices(driver, vm->def, hostdev_flags) < 0)
         return -1;
+    if (qemuProcessOpenVfioFds(vm) < 0)
+        return -1;
 
     VIR_DEBUG("Preparing chr device backends");
     if (qemuProcessPrepareHostBackendChardev(vm) < 0)

diff --git a/src/util/virpci.c b/src/util/virpci.c
index 90617e69c6db611bfbe4b92a13940ae09c79dc39..2348a9800361f35ffa3800f740a8b93561b6b6b7 100644 (file)
--- a/src/util/virpci.c
+++ b/src/util/virpci.c
@@ -3320,3 +3320,42 @@ virPCIDeviceAddressFree(virPCIDeviceAddress *address)
 {
     g_free(address);
 }
+
+/**
+ * virPCIDeviceGetVfioPath:
+ * @addr: host device PCI address
+ * @vfioPath: returned VFIO device path
+ *
+ * Constructs the VFIO device path for a PCI hostdev.
+ *
+ * Returns: 0 on success, -1 on failure
+ */
+int
+virPCIDeviceGetVfioPath(virPCIDeviceAddress *addr,
+                        char **vfioPath)
+{
+    g_autofree char *addrStr = NULL;
+    g_autofree char *sysfsPath = NULL;
+    g_autoptr(DIR) dir = NULL;
+    struct dirent *entry = NULL;
+
+    *vfioPath = NULL;
+    addrStr = virPCIDeviceAddressAsString(addr);
+
+    /* Look in device's vfio-dev subdirectory */
+    sysfsPath = g_strdup_printf("/sys/bus/pci/devices/%s/vfio-dev/", addrStr);
+
+    if (virDirOpen(&dir, sysfsPath) == 1) {
+        while (virDirRead(dir, &entry, sysfsPath) > 0) {
+            if (STRPREFIX(entry->d_name, "vfio")) {
+                *vfioPath = g_strdup_printf("/dev/vfio/devices/%s", entry->d_name);
+                return 0;
+            }
+        }
+    }
+
+    virReportError(VIR_ERR_INTERNAL_ERROR,
+                   _("cannot find VFIO device for PCI device %1$s"),
+                   addrStr);
+    return -1;
+}

diff --git a/src/util/virpci.h b/src/util/virpci.h
index fc538566e10db2fe8888fcc8e1ea03d09528b542..24ede1075532da96fcd9a7392d4460478893de70 100644 (file)
--- a/src/util/virpci.h
+++ b/src/util/virpci.h
@@ -296,6 +296,8 @@ void virPCIEDeviceInfoFree(virPCIEDeviceInfo *dev);
 
 void virPCIDeviceAddressFree(virPCIDeviceAddress *address);
 
+int virPCIDeviceGetVfioPath(virPCIDeviceAddress *addr, char **vfioPath);
+
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(virPCIDevice, virPCIDeviceFree);
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(virPCIDeviceAddress, virPCIDeviceAddressFree);
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(virPCIEDeviceInfo, virPCIEDeviceInfoFree);