MINOR: config: add thread-hard-limit to set an upper bound to nbthread

On todays large systems, it's not always desired to run on all threads
for light loads, and usually users enforce nbthread to a lower value
(e.g. 8). The problem is that this is a fixed value, and moving such
configs to smaller machines continues to enforce the value and this
becomes extremely unproductive due to having more threads than CPUs.
This also happens quite a bit in VMs, containers, or cloud instances
of various sizes.

This commit introduces the thread-hard-limit setting that allows to only
set an upper bound to the number of threads without raising a lower value.
This means that using "thread-hard-limit 8" will make sure that no more
than 8 threads will be used when available, but it will remain two when
run on a dual-core machine.

diff --git a/doc/configuration.txt b/doc/configuration.txt
index c0667af8f806e2b31c181b784ea0710fa8271107..3961a1bc7d4698ee80b19c5d0271a6464e6e3564 100644 (file)
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -2156,7 +2156,8 @@ nbthread <number>
   bound to upon startup. This means that the thread count can easily be
   adjusted from the calling process using commands like "taskset" or "cpuset".
   Otherwise, this value defaults to 1. The default value is reported in the
-  output of "haproxy -vv".
+  output of "haproxy -vv". Note that values set here or automatically detected
+  are subject to the limit set by "thread-hard-limit" (if set).
 
 no-quic
   Disable QUIC transport protocol. All the QUIC listeners will still be created.
@@ -2735,6 +2736,19 @@ thread-groups <number>
   since up to 64 threads per group may be configured. The maximum number of
   groups is configured at compile time and defaults to 16. See also "nbthread".
 
+thread-hard-limit <number>
+  This setting is used to enforce a limit to the number of threads, either
+  detected, or configured. This is particularly useful on operating systems
+  where the number of threads is automatically detected, where a number of
+  threads lower than the number of CPUs is desired in generic and portable
+  configurations. Indeed, while "nbthread" enforces a number of threads that
+  will result in a warning and bad performance if higher than CPUs available,
+  thread-hard-limit will only cap the maximum value and automatically limit
+  the number of threads to no higher than this value, but will not raise lower
+  values. If "nbthread" is forced to a higher value, thread-hard-limit wins,
+  and a warning is emitted in so that the configuration anomaly can be
+  fixed. By default there is no limit. See also "nbthread".
+
 trace <args...>
   This command configures one "trace" subsystem statement. Each of them can be
   found in the management manual, and follow the exact same syntax. Only one

diff --git a/include/haproxy/global-t.h b/include/haproxy/global-t.h
index a1357ac43d4c05e7af3d304045836beb23134223..f38c28eaeaceb88d90a1d45a3ebee4a47f18ff2c 100644 (file)
--- a/include/haproxy/global-t.h
+++ b/include/haproxy/global-t.h
@@ -214,6 +214,7 @@ struct global {
        } unix_bind;
        struct proxy *cli_fe;           /* the frontend holding the stats settings */
        int numa_cpu_mapping;
+       int thread_limit;               /* hard limit on the number of threads */
        int prealloc_fd;
        int cfg_curr_line;              /* line number currently being parsed */
        const char *cfg_curr_file;      /* config file currently being parsed or NULL */

diff --git a/src/cfgparse.c b/src/cfgparse.c
index c09d96435494d86286cfddf9f155ded9371f4d45..f5cde502cb269890e0adf56afc810d2688e93866 100644 (file)
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -2730,6 +2730,13 @@ int check_config_validity()
        if (!global.tune.requri_len)
                global.tune.requri_len = REQURI_LEN;
 
+       if (!global.thread_limit)
+               global.thread_limit = MAX_THREADS;
+
+#if defined(USE_THREAD)
+       if (thread_cpus_enabled_at_boot > global.thread_limit)
+               thread_cpus_enabled_at_boot = global.thread_limit;
+#endif
        if (!global.nbthread) {
                /* nbthread not set, thus automatic. In this case, and only if
                 * running on a single process, we enable the same number of
@@ -2753,13 +2760,24 @@ int check_config_validity()
                                global.nbtgroups = 1;
 
                        if (global.nbthread > MAX_THREADS_PER_GROUP * global.nbtgroups) {
-                               ha_diag_warning("nbthread not set, found %d CPUs, limiting to %d threads (maximum is %d per thread group). Please set nbthreads and/or increase thread-groups in the global section to silence this warning.\n",
-                                          global.nbthread, MAX_THREADS_PER_GROUP * global.nbtgroups, MAX_THREADS_PER_GROUP);
+                               if (global.nbthread <= global.thread_limit)
+                                       ha_diag_warning("nbthread not set, found %d CPUs, limiting to %d threads (maximum is %d per thread group). "
+                                                       "Please set nbthreads and/or increase thread-groups in the global section to silence this warning.\n",
+                                                       global.nbthread, MAX_THREADS_PER_GROUP * global.nbtgroups, MAX_THREADS_PER_GROUP);
                                global.nbthread = MAX_THREADS_PER_GROUP * global.nbtgroups;
                        }
+
+                       if (global.nbthread > global.thread_limit)
+                               global.nbthread = global.thread_limit;
                }
 #endif
        }
+       else if (global.nbthread > global.thread_limit) {
+               ha_warning("nbthread forced to a higher value (%d) than the configured thread-hard-limit (%d), enforcing the limit. "
+                          "Please fix either value to remove this warning.\n",
+                          global.nbthread, global.thread_limit);
+               global.nbthread = global.thread_limit;
+       }
 
        if (!global.nbtgroups)
                global.nbtgroups = 1;

diff --git a/src/thread.c b/src/thread.c
index ab4342dc74bd28e9edd6103d0b8c1aac615172d3..655e1998cf8d5ab135c7e1d0afd5871ff05e3485 100644 (file)
--- a/src/thread.c
+++ b/src/thread.c
@@ -1709,6 +1709,35 @@ static int cfg_parse_nbthread(char **args, int section_type, struct proxy *curpx
        return 0;
 }
 
+/* Parse the "thread-hard-limit" global directive, which takes an integer
+ * argument that contains the desired maximum number of threads that will
+ * not be crossed.
+ */
+static int cfg_parse_thread_hard_limit(char **args, int section_type, struct proxy *curpx,
+                              const struct proxy *defpx, const char *file, int line,
+                              char **err)
+{
+       long nbthread;
+       char *errptr;
+
+       if (too_many_args(1, args, err, NULL))
+               return -1;
+
+       nbthread = strtol(args[1], &errptr, 10);
+       if (!*args[1] || *errptr) {
+               memprintf(err, "'%s' passed a missing or unparsable integer value in '%s'", args[0], args[1]);
+               return -1;
+       }
+
+       if (nbthread < 1 || nbthread > MAX_THREADS) {
+               memprintf(err, "'%s' value must be at least 1 (was %ld)", args[0], nbthread);
+               return -1;
+       }
+
+       global.thread_limit = nbthread;
+       return 0;
+}
+
 /* Parse the "thread-group" global directive, which takes an integer argument
  * that designates a thread group, and a list of threads to put into that group.
  */
@@ -1855,6 +1884,7 @@ static int cfg_parse_thread_groups(char **args, int section_type, struct proxy *
 
 /* config keyword parsers */
 static struct cfg_kw_list cfg_kws = {ILH, {
+       { CFG_GLOBAL, "thread-hard-limit", cfg_parse_thread_hard_limit, 0 },
        { CFG_GLOBAL, "nbthread",       cfg_parse_nbthread, 0 },
        { CFG_GLOBAL, "thread-group",   cfg_parse_thread_group, 0 },
        { CFG_GLOBAL, "thread-groups",  cfg_parse_thread_groups, 0 },