qemu: Support O_DIRECT with mapped-ram on save

When using the mapped-ram migration capability, direct IO is
enabled by setting the "direct-io" migration parameter to
"true" and passing QEMU an additional fd with O_DIRECT set.

Signed-off-by: Jim Fehlig <jfehlig@suse.com>
Reviewed-by: Daniel P. Berrangé <berrange@redhat.com>

diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index bb83948dd01241882d541c2d566895e5cb58f9a7..9c8541067c555219d736307e600445e7c2b7c7d8 100644 (file)
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -2668,7 +2668,8 @@ qemuDomainSaveInternal(virQEMUDriver *driver,
         goto endjob;
     xml = NULL;
 
-    if (!(saveParams = qemuMigrationParamsForSave(format == QEMU_SAVE_FORMAT_SPARSE)))
+    if (!(saveParams = qemuMigrationParamsForSave(format == QEMU_SAVE_FORMAT_SPARSE,
+                                                  flags)))
         goto endjob;
 
     ret = qemuSaveImageCreate(driver, vm, path, data, compressor,
@@ -3118,7 +3119,7 @@ doCoreDump(virQEMUDriver *driver,
         if (!(dump_params = qemuMigrationParamsNew()))
             goto cleanup;
 
-        if (qemuMigrationSrcToFile(driver, vm, &fd, compressor,
+        if (qemuMigrationSrcToFile(driver, vm, path, &fd, compressor,
                                    dump_params, dump_flags, VIR_ASYNC_JOB_DUMP) < 0)
             goto cleanup;
     }
@@ -5762,7 +5763,7 @@ qemuDomainRestoreInternal(virConnectPtr conn,
         goto cleanup;
 
     sparse = data->header.format == QEMU_SAVE_FORMAT_SPARSE;
-    if (!(restoreParams = qemuMigrationParamsForSave(sparse)))
+    if (!(restoreParams = qemuMigrationParamsForSave(sparse, flags)))
         goto cleanup;
 
     fd = qemuSaveImageOpen(driver, path,
@@ -6096,7 +6097,8 @@ qemuDomainObjRestore(virConnectPtr conn,
     }
 
     sparse = data->header.format == QEMU_SAVE_FORMAT_SPARSE;
-    if (!(restoreParams = qemuMigrationParamsForSave(sparse)))
+    if (!(restoreParams = qemuMigrationParamsForSave(sparse,
+                                                     bypass_cache ? VIR_DOMAIN_SAVE_BYPASS_CACHE : 0)))
         return -1;
 
     fd = qemuSaveImageOpen(driver, path, bypass_cache, sparse, &wrapperFd, false);

diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index 430608cb66f06e29120267f318eabcdedee888b0..d4e20dae460e459232843d602198d206bdc48b76 100644 (file)
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -7183,17 +7183,36 @@ qemuMigrationSrcToLegacyFile(virQEMUDriver *driver,
 static int
 qemuMigrationSrcToSparseFile(virQEMUDriver *driver,
                              virDomainObj *vm,
+                             const char *path,
                              int *fd,
                              unsigned int flags,
                              virDomainAsyncJob asyncJob)
 {
+    g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
+    VIR_AUTOCLOSE directFd = -1;
+    int directFlag = 0;
+    bool needUnlink = false;
     int ret;
 
-    /* mapped-ram does not support directIO */
+    /* When using directio with mapped-ram, qemu needs two fds. One with
+     * O_DIRECT set writing the memory, and another without it set for
+     * writing small bits of unaligned state. */
     if ((flags & VIR_DOMAIN_SAVE_BYPASS_CACHE)) {
-        virReportError(VIR_ERR_OPERATION_FAILED, "%s",
-                       _("bypass cache unsupported by this system"));
-        return -1;
+        directFlag = virFileDirectFdFlag();
+        if (directFlag < 0) {
+            virReportError(VIR_ERR_OPERATION_FAILED, "%s",
+                           _("bypass cache unsupported by this system"));
+            return -1;
+        }
+        directFd = virQEMUFileOpenAs(cfg->user, cfg->group, false, path,
+                           O_WRONLY | directFlag, &needUnlink);
+
+        if (directFd < 0)
+            return -1;
+
+        if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, directFd) < 0)
+            return -1;
+
     }
 
     if (qemuSecuritySetImageFDLabel(driver->securityManager, vm->def, *fd) < 0)
@@ -7202,7 +7221,7 @@ qemuMigrationSrcToSparseFile(virQEMUDriver *driver,
     if (qemuDomainObjEnterMonitorAsync(vm, asyncJob) < 0)
         return -1;
 
-    ret = qemuMonitorMigrateToFdSet(vm, 0, fd);
+    ret = qemuMonitorMigrateToFdSet(vm, 0, fd, &directFd);
     qemuDomainObjExitMonitor(vm);
     return ret;
 }
@@ -7211,6 +7230,7 @@ qemuMigrationSrcToSparseFile(virQEMUDriver *driver,
 /* Helper function called while vm is active.  */
 int
 qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm,
+                       const char *path,
                        int *fd,
                        virCommand *compressor,
                        qemuMigrationParams *migParams,
@@ -7248,7 +7268,7 @@ qemuMigrationSrcToFile(virQEMUDriver *driver, virDomainObj *vm,
 
     if (migParams &&
         qemuMigrationParamsCapEnabled(migParams, QEMU_MIGRATION_CAP_MAPPED_RAM))
-        rc = qemuMigrationSrcToSparseFile(driver, vm, fd, flags, asyncJob);
+        rc = qemuMigrationSrcToSparseFile(driver, vm, path, fd, flags, asyncJob);
     else
         rc = qemuMigrationSrcToLegacyFile(driver, vm, *fd, compressor, asyncJob);
 

diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h
index 71a99747536f98a58000c64ba8282aad5766beda..beb888160aaf0a56630fb510ff5eb42bbc92a51c 100644 (file)
--- a/src/qemu/qemu_migration.h
+++ b/src/qemu/qemu_migration.h
@@ -238,6 +238,7 @@ qemuMigrationSrcIsAllowed(virDomainObj *vm,
 int
 qemuMigrationSrcToFile(virQEMUDriver *driver,
                        virDomainObj *vm,
+                       const char *path,
                        int *fd,
                        virCommand *compressor,
                        qemuMigrationParams *migParams,

diff --git a/src/qemu/qemu_migration_params.c b/src/qemu/qemu_migration_params.c
index d48cdd5506fd0ddf27c635880598c8f592b14a22..16e93f27d62824e58be7a0be1afabd0c6974beb8 100644 (file)
--- a/src/qemu/qemu_migration_params.c
+++ b/src/qemu/qemu_migration_params.c
@@ -130,6 +130,7 @@ VIR_ENUM_IMPL(qemuMigrationParam,
               "multifd-zlib-level",
               "multifd-zstd-level",
               "avail-switchover-bandwidth",
+              "direct-io",
 );
 
 typedef struct _qemuMigrationParamsAlwaysOnItem qemuMigrationParamsAlwaysOnItem;
@@ -328,6 +329,9 @@ static const qemuMigrationParamInfoItem qemuMigrationParamInfo[] = {
     [QEMU_MIGRATION_PARAM_AVAIL_SWITCHOVER_BANDWIDTH] = {
         .type = QEMU_MIGRATION_PARAM_TYPE_ULL,
     },
+    [QEMU_MIGRATION_PARAM_DIRECT_IO] = {
+        .type = QEMU_MIGRATION_PARAM_TYPE_BOOL,
+    },
 };
 G_STATIC_ASSERT(G_N_ELEMENTS(qemuMigrationParamInfo) == QEMU_MIGRATION_PARAM_LAST);
 
@@ -793,7 +797,7 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params,
 
 
 qemuMigrationParams *
-qemuMigrationParamsForSave(bool sparse)
+qemuMigrationParamsForSave(bool sparse, unsigned int flags)
 {
     g_autoptr(qemuMigrationParams) saveParams = NULL;
 
@@ -807,6 +811,11 @@ qemuMigrationParamsForSave(bool sparse)
             return NULL;
         saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].value.i = 1;
         saveParams->params[QEMU_MIGRATION_PARAM_MULTIFD_CHANNELS].set = true;
+
+        if (flags & VIR_DOMAIN_SAVE_BYPASS_CACHE) {
+            saveParams->params[QEMU_MIGRATION_PARAM_DIRECT_IO].value.b = true;
+            saveParams->params[QEMU_MIGRATION_PARAM_DIRECT_IO].set = true;
+        }
     }
 
     return g_steal_pointer(&saveParams);

diff --git a/src/qemu/qemu_migration_params.h b/src/qemu/qemu_migration_params.h
index 88a1bc1a6683bb7c71297696e436507370ebc99d..87822332d3f0865866a8366f60611df1ee21ec3b 100644 (file)
--- a/src/qemu/qemu_migration_params.h
+++ b/src/qemu/qemu_migration_params.h
@@ -66,6 +66,7 @@ typedef enum {
     QEMU_MIGRATION_PARAM_MULTIFD_ZLIB_LEVEL,
     QEMU_MIGRATION_PARAM_MULTIFD_ZSTD_LEVEL,
     QEMU_MIGRATION_PARAM_AVAIL_SWITCHOVER_BANDWIDTH,
+    QEMU_MIGRATION_PARAM_DIRECT_IO,
 
     QEMU_MIGRATION_PARAM_LAST
 } qemuMigrationParam;
@@ -88,7 +89,7 @@ qemuMigrationParamsFromFlags(virTypedParameterPtr params,
                              qemuMigrationParty party);
 
 qemuMigrationParams *
-qemuMigrationParamsForSave(bool sparse);
+qemuMigrationParamsForSave(bool sparse, unsigned int flags);
 
 int
 qemuMigrationParamsDump(qemuMigrationParams *migParams,

diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
index ef86cbe9ff7ddebbc9bdd12c970260a7716f5a6d..7610d43e6c0ca971b5c001bc7e50eb3e8b044c62 100644 (file)
--- a/src/qemu/qemu_monitor.c
+++ b/src/qemu/qemu_monitor.c
@@ -2233,7 +2233,8 @@ qemuMonitorMigrateToFd(qemuMonitor *mon,
 int
 qemuMonitorMigrateToFdSet(virDomainObj *vm,
                           unsigned int flags,
-                          int *fd)
+                          int *fd,
+                          int *directFd)
 {
     qemuDomainObjPrivate *priv = vm->privateData;
     qemuMonitor *mon = priv->mon;
@@ -2242,7 +2243,7 @@ qemuMonitorMigrateToFdSet(virDomainObj *vm,
     g_autofree char *uri = NULL;
     int ret;
 
-    VIR_DEBUG("fd=%d flags=0x%x", *fd, flags);
+    VIR_DEBUG("fd=%d directFd=%d flags=0x%x", *fd, *directFd, flags);
 
     QEMU_CHECK_MONITOR(mon);
 
@@ -2254,6 +2255,8 @@ qemuMonitorMigrateToFdSet(virDomainObj *vm,
 
     fdPassMigrate = qemuFDPassNew("libvirt-outgoing-migrate", priv);
     qemuFDPassAddFD(fdPassMigrate, fd, "-fd");
+    if (*directFd != -1)
+        qemuFDPassAddFD(fdPassMigrate, directFd, "-directio-fd");
     qemuFDPassTransferMonitor(fdPassMigrate, mon);
 
     uri = g_strdup_printf("file:%s,offset=%#lx",

diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h
index 7026f74e2adc6aa162331ed2e37a4d2657ce9ba0..9a55b494720fe08c0018f4a43f81b0cd407981d9 100644 (file)
--- a/src/qemu/qemu_monitor.h
+++ b/src/qemu/qemu_monitor.h
@@ -861,7 +861,8 @@ int qemuMonitorMigrateToFd(qemuMonitor *mon,
 
 int qemuMonitorMigrateToFdSet(virDomainObj *vm,
                               unsigned int flags,
-                              int *fd);
+                              int *fd,
+                              int *directFd);
 
 int qemuMonitorMigrateToHost(qemuMonitor *mon,
                              unsigned int flags,

diff --git a/src/qemu/qemu_saveimage.c b/src/qemu/qemu_saveimage.c
index 0a634088faed5694d22b99a0c7016262626a57cd..a6a5fb486e595e48e0aeea87a74e9811f7bc2dba 100644 (file)
--- a/src/qemu/qemu_saveimage.c
+++ b/src/qemu/qemu_saveimage.c
@@ -501,7 +501,7 @@ qemuSaveImageCreate(virQEMUDriver *driver,
         goto cleanup;
 
     /* Perform the migration */
-    if (qemuMigrationSrcToFile(driver, vm, &fd, compressor, saveParams, flags, asyncJob) < 0)
+    if (qemuMigrationSrcToFile(driver, vm, path, &fd, compressor, saveParams, flags, asyncJob) < 0)
         goto cleanup;
 
     /* Touch up file header to mark image complete. */