Fix QEMU cpu affinity at startup to include all threads

The QEMU cpu affinity is used in NUMA scenarios to ensure that
guest memory is allocated from a specific node. Normally memory
is allocate on demand in vCPU threads, but when using hugepages
the initial thread leader allocates memory upfront. libvirt was
not setting affinity of the thread leader, or I/O threads. This
patch changes the code to set the process affinity in between
the fork()/exec() of QEMU. This ensures that every single QEMU
thread gets the affinity

* src/qemu/qemu_driver.c: Set affinity on entire QEMU process
  at startup

diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index b738be86e3e74dcc06bc8652efc6706dba195658..9ee5da718eab4538c8923b8e667b83387576651c 100644 (file)
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -1929,6 +1929,9 @@ qemuDetectVcpuPIDs(struct qemud_driver *driver,
     return 0;
 }
 
+/*
+ * To be run between fork/exec of QEMU only
+ */
 static int
 qemudInitCpuAffinity(virDomainObjPtr vm)
 {
@@ -1936,7 +1939,8 @@ qemudInitCpuAffinity(virDomainObjPtr vm)
     virNodeInfo nodeinfo;
     unsigned char *cpumap;
     int cpumaplen;
-    qemuDomainObjPrivatePtr priv = vm->privateData;
+
+    DEBUG0("Setting CPU affinity");
 
     if (nodeGetInfo(NULL, &nodeinfo) < 0)
         return -1;
@@ -1968,14 +1972,14 @@ qemudInitCpuAffinity(virDomainObjPtr vm)
             VIR_USE_CPU(cpumap, i);
     }
 
-    /* The XML config only gives a per-VM affinity, so we apply
-     * the same mapping to all vCPUs */
-    for (i = 0 ; i < priv->nvcpupids ; i++) {
-        if (virProcessInfoSetAffinity(priv->vcpupids[i],
-                                      cpumap, cpumaplen, maxcpu) < 0) {
-            VIR_FREE(cpumap);
-            return -1;
-        }
+    /* We are pressuming we are running between fork/exec of QEMU
+     * so use '0' to indicate our own process ID. No threads are
+     * running at this point
+     */
+    if (virProcessInfoSetAffinity(0, /* Self */
+                                  cpumap, cpumaplen, maxcpu) < 0) {
+        VIR_FREE(cpumap);
+        return -1;
     }
     VIR_FREE(cpumap);
 
@@ -2896,6 +2900,12 @@ struct qemudHookData {
 static int qemudSecurityHook(void *data) {
     struct qemudHookData *h = data;
 
+    /* This must take place before exec(), so that all QEMU
+     * memory allocation is on the correct NUMA node
+     */
+    if (qemudInitCpuAffinity(h->vm) < 0)
+        return -1;
+
     if (qemuAddToCgroup(h->driver, h->vm->def) < 0)
         return -1;
 
@@ -3202,10 +3212,6 @@ static int qemudStartVMDaemon(virConnectPtr conn,
     if (qemuDetectVcpuPIDs(driver, vm) < 0)
         goto abort;
 
-    DEBUG0("Setting CPU affinity");
-    if (qemudInitCpuAffinity(vm) < 0)
-        goto abort;
-
     DEBUG0("Setting any required VM passwords");
     if (qemuInitPasswords(conn, driver, vm, qemuCmdFlags) < 0)
         goto abort;