qemu: Add support for overriding max threads per process limit

Some VM configurations may result in a large number of threads created by
the associated qemu process which can exceed the system default limit. The
maximum number of threads allowed per process is controlled by the pids
cgroup controller and is set to 16k when creating VMs with systemd's
machined service. The maximum number of threads per process is recorded
in the pids.max file under the machine's pids controller cgroup hierarchy,
e.g.

$cgrp-mnt/pids/machine.slice/machine-qemu\\x2d1\\x2dtest.scope/pids.max

Maximum threads per process is controlled with the TasksMax property of
the systemd scope for the machine. This patch adds an option to qemu.conf
which can be used to override the maximum number of threads allowed per
qemu process. If the value of option is greater than zero, it will be set
in the TasksMax property of the machine's scope after creating the machine.

Signed-off-by: Jim Fehlig <jfehlig@suse.com>
Reviewed-by: Daniel P. Berrangé <berrange@redhat.com>

diff --git a/src/lxc/lxc_cgroup.c b/src/lxc/lxc_cgroup.c
index d93a19d684e605dfa5761ded2fa69ea95a58317d..76014f3bfd4e3c2fd6c63f6968defd103449c9d4 100644 (file)
--- a/src/lxc/lxc_cgroup.c
+++ b/src/lxc/lxc_cgroup.c
@@ -455,6 +455,7 @@ virCgroupPtr virLXCCgroupCreate(virDomainDefPtr def,
                             nnicindexes, nicindexes,
                             def->resource->partition,
                             -1,
+                            0,
                             &cgroup) < 0)
         goto cleanup;
 

diff --git a/src/qemu/libvirtd_qemu.aug b/src/qemu/libvirtd_qemu.aug
index eea9094d39406b8bc410ebdaf826df46a631678f..2a99a0c55fe7a116f6c1f6f955e3f14e9b10ba14 100644 (file)
--- a/src/qemu/libvirtd_qemu.aug
+++ b/src/qemu/libvirtd_qemu.aug
@@ -95,6 +95,7 @@ module Libvirtd_qemu =
                  | limits_entry "max_core"
                  | bool_entry "dump_guest_core"
                  | str_entry "stdio_handler"
+                 | int_entry "max_threads_per_process"
 
    let device_entry = bool_entry "mac_filter"
                  | bool_entry "relaxed_acs_check"

diff --git a/src/qemu/qemu.conf b/src/qemu/qemu.conf
index fd2ed9dc219b3a310dda530d29708eb45aaa0838..8cabeccacbe49223bac4ffc2b81c23fde6b49f06 100644 (file)
--- a/src/qemu/qemu.conf
+++ b/src/qemu/qemu.conf
@@ -613,6 +613,16 @@
 #max_processes = 0
 #max_files = 0
 
+# If max_threads_per_process is set to a positive integer, libvirt
+# will use it to set the maximum number of threads that can be
+# created by a qemu process. Some VM configurations can result in
+# qemu processes with tens of thousands of threads. systemd-based
+# systems typically limit the number of threads per process to
+# 16k. max_threads_per_process can be used to override default
+# limits in the host OS.
+#
+#max_threads_per_process = 0
+
 # If max_core is set to a non-zero integer, then QEMU will be
 # permitted to create core dumps when it crashes, provided its
 # RAM size is smaller than the limit set.

diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index 19ca60905a60673f6ff9a7d103aeebb6f94d5e84..ecd96efb0aaa48953fa6422ce3aa4a06f4b75a25 100644 (file)
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -929,6 +929,7 @@ qemuInitCgroup(virDomainObjPtr vm,
                             nnicindexes, nicindexes,
                             vm->def->resource->partition,
                             cfg->cgroupControllers,
+                            cfg->maxThreadsPerProc,
                             &priv->cgroup) < 0) {
         if (virCgroupNewIgnoreError())
             goto done;

diff --git a/src/qemu/qemu_conf.c b/src/qemu/qemu_conf.c
index e0195dac296541037bc77ea65c349a06cc5498bc..71d0464c0d548da6f9673a0b9adf1401ed13c066 100644 (file)
--- a/src/qemu/qemu_conf.c
+++ b/src/qemu/qemu_conf.c
@@ -670,6 +670,8 @@ virQEMUDriverConfigLoadProcessEntry(virQEMUDriverConfigPtr cfg,
         return -1;
     if (virConfGetValueUInt(conf, "max_files", &cfg->maxFiles) < 0)
         return -1;
+    if (virConfGetValueUInt(conf, "max_threads_per_process", &cfg->maxThreadsPerProc) < 0)
+        return -1;
 
     if (virConfGetValueType(conf, "max_core") == VIR_CONF_STRING) {
         if (virConfGetValueString(conf, "max_core", &corestr) < 0)

diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h
index 2229b76e89e3368ae72801dd6bf002c31ea7fdd8..d8e3bfe87c20d09f8455765f3e6b9f229834f40e 100644 (file)
--- a/src/qemu/qemu_conf.h
+++ b/src/qemu/qemu_conf.h
@@ -162,6 +162,7 @@ struct _virQEMUDriverConfig {
 
     unsigned int maxProcesses;
     unsigned int maxFiles;
+    unsigned int maxThreadsPerProc;
     unsigned long long maxCore;
     bool dumpGuestCore;
 

diff --git a/src/qemu/test_libvirtd_qemu.aug.in b/src/qemu/test_libvirtd_qemu.aug.in
index 388ba24b8bea6fd2b6cdfdd85928f6eaad4e298b..b3b44d42d95ed83047da9793df2f014daf191919 100644 (file)
--- a/src/qemu/test_libvirtd_qemu.aug.in
+++ b/src/qemu/test_libvirtd_qemu.aug.in
@@ -76,6 +76,7 @@ module Test_libvirtd_qemu =
 { "set_process_name" = "1" }
 { "max_processes" = "0" }
 { "max_files" = "0" }
+{ "max_threads_per_process" = "0" }
 { "max_core" = "unlimited" }
 { "dump_guest_core" = "1" }
 { "mac_filter" = "1" }

diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c
index f7afc2964d63c5ecf564399acabf9a195f73d830..9daf62795e908e9ca676e803e271870a05d00d9e 100644 (file)
--- a/src/util/vircgroup.c
+++ b/src/util/vircgroup.c
@@ -1118,6 +1118,7 @@ virCgroupNewMachineSystemd(const char *name,
                            int *nicindexes,
                            const char *partition,
                            int controllers,
+                           unsigned int maxthreads,
                            virCgroupPtr *group)
 {
     int rv;
@@ -1134,7 +1135,8 @@ virCgroupNewMachineSystemd(const char *name,
                                       isContainer,
                                       nnicindexes,
                                       nicindexes,
-                                      partition)) < 0)
+                                      partition,
+                                      maxthreads)) < 0)
         return rv;
 
     if (controllers != -1)
@@ -1246,6 +1248,7 @@ virCgroupNewMachine(const char *name,
                     int *nicindexes,
                     const char *partition,
                     int controllers,
+                    unsigned int maxthreads,
                     virCgroupPtr *group)
 {
     int rv;
@@ -1262,6 +1265,7 @@ virCgroupNewMachine(const char *name,
                                          nicindexes,
                                          partition,
                                          controllers,
+                                         maxthreads,
                                          group)) == 0)
         return 0;
 

diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h
index 2f68fdb685878eaaa9a86c7c8ae93522d863e1aa..3eefe7878769c1517001eb6ae147611ef5b8f2fb 100644 (file)
--- a/src/util/vircgroup.h
+++ b/src/util/vircgroup.h
@@ -98,6 +98,7 @@ int virCgroupNewMachine(const char *name,
                         int *nicindexes,
                         const char *partition,
                         int controllers,
+                        unsigned int maxthreads,
                         virCgroupPtr *group)
     ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2)
     ATTRIBUTE_NONNULL(3);

diff --git a/src/util/virsystemd.c b/src/util/virsystemd.c
index f6c5adc5eff88a1ccf1ebb7e8619e4be5673e2f7..1cb8874403aaf39b89c3d97b02aee5681bcb2e2c 100644 (file)
--- a/src/util/virsystemd.c
+++ b/src/util/virsystemd.c
@@ -252,12 +252,14 @@ int virSystemdCreateMachine(const char *name,
                             bool iscontainer,
                             size_t nnicindexes,
                             int *nicindexes,
-                            const char *partition)
+                            const char *partition,
+                            unsigned int maxthreads)
 {
     int ret;
     DBusConnection *conn;
     char *creatorname = NULL;
     char *slicename = NULL;
+    char *scopename = NULL;
     static int hasCreateWithNetwork = 1;
 
     if ((ret = virSystemdHasMachined()) < 0)
@@ -403,11 +405,31 @@ int virSystemdCreateMachine(const char *name,
             goto cleanup;
     }
 
+    if (maxthreads > 0) {
+        if (!(scopename = virSystemdMakeScopeName(name, drivername, false)))
+            goto cleanup;
+
+        if (virDBusCallMethod(conn,
+                              NULL,
+                              NULL,
+                              "org.freedesktop.systemd1",
+                              "/org/freedesktop/systemd1",
+                              "org.freedesktop.systemd1.Manager",
+                              "SetUnitProperties",
+                              "sba(sv)",
+                              scopename,
+                              true,
+                              1,
+                              "TasksMax", "t", (uint64_t)maxthreads) < 0)
+            goto cleanup;
+    }
+
     ret = 0;
 
  cleanup:
     VIR_FREE(creatorname);
     VIR_FREE(slicename);
+    VIR_FREE(scopename);
     return ret;
 }
 

diff --git a/src/util/virsystemd.h b/src/util/virsystemd.h
index 5d56c78835ee58975f87dc73cbb4b9638ec3a407..96626f8fff0d2fba1c32e69d8eb8573fd73e5606 100644 (file)
--- a/src/util/virsystemd.h
+++ b/src/util/virsystemd.h
@@ -50,7 +50,8 @@ int virSystemdCreateMachine(const char *name,
                             bool iscontainer,
                             size_t nnicindexes,
                             int *nicindexes,
-                            const char *partition);
+                            const char *partition,
+                            unsigned int maxthreads);
 
 int virSystemdTerminateMachine(const char *name);
 

diff --git a/tests/virsystemdtest.c b/tests/virsystemdtest.c
index 7aaa8f97fa85695d976a002852729b2d4af81ace..340b038095b60392913d74f458afd557c0d106e0 100644 (file)
--- a/tests/virsystemdtest.c
+++ b/tests/virsystemdtest.c
@@ -175,7 +175,7 @@ static int testCreateContainer(const void *opaque ATTRIBUTE_UNUSED)
                                 123,
                                 true,
                                 0, NULL,
-                                "highpriority.slice") < 0) {
+                                "highpriority.slice", 0) < 0) {
         fprintf(stderr, "%s", "Failed to create LXC machine\n");
         return -1;
     }
@@ -208,7 +208,7 @@ static int testCreateMachine(const void *opaque ATTRIBUTE_UNUSED)
                                 123,
                                 false,
                                 0, NULL,
-                                NULL) < 0) {
+                                NULL, 0) < 0) {
         fprintf(stderr, "%s", "Failed to create KVM machine\n");
         return -1;
     }
@@ -245,7 +245,7 @@ static int testCreateNoSystemd(const void *opaque ATTRIBUTE_UNUSED)
                                       123,
                                       false,
                                       0, NULL,
-                                      NULL)) == 0) {
+                                      NULL, 0)) == 0) {
         unsetenv("FAIL_NO_SERVICE");
         fprintf(stderr, "%s", "Unexpected create machine success\n");
         return -1;
@@ -279,7 +279,7 @@ static int testCreateSystemdNotRunning(const void *opaque ATTRIBUTE_UNUSED)
                                       123,
                                       false,
                                       0, NULL,
-                                      NULL)) == 0) {
+                                      NULL, 0)) == 0) {
         unsetenv("FAIL_NOT_REGISTERED");
         fprintf(stderr, "%s", "Unexpected create machine success\n");
         return -1;
@@ -313,7 +313,7 @@ static int testCreateBadSystemd(const void *opaque ATTRIBUTE_UNUSED)
                                       123,
                                       false,
                                       0, NULL,
-                                      NULL)) == 0) {
+                                      NULL, 0)) == 0) {
         unsetenv("FAIL_BAD_SERVICE");
         fprintf(stderr, "%s", "Unexpected create machine success\n");
         return -1;
@@ -348,7 +348,7 @@ static int testCreateNetwork(const void *opaque ATTRIBUTE_UNUSED)
                                 123,
                                 true,
                                 nnicindexes, nicindexes,
-                                "highpriority.slice") < 0) {
+                                "highpriority.slice", 0) < 0) {
         fprintf(stderr, "%s", "Failed to create LXC machine\n");
         return -1;
     }