MAJOR: threads: enable one thread per CPU by default

Threads have long matured by now, still for most users their usage is
not trivial. It's about time to enable them by default on platforms
where we know the number of CPUs bound. This patch does this, it counts
the number of CPUs the process is bound to upon startup, and enables as
many threads by default. Of course, "nbthread" still overrides this, but
if it's not set the default behaviour is to start one thread per CPU.

The default number of threads is reported in "haproxy -vv". Simply using
"taskset -c" is now enough to adjust this number of threads so that there
is no more need for playing with cpu-map. And thanks to the previous
patches on the listener, the vast majority of configurations will not
need to duplicate "bind" lines with the "process x/y" statement anymore
either, so a simple config will automatically adapt to the number of
processors available.

diff --git a/doc/configuration.txt b/doc/configuration.txt
index cd6ddf68bbbecd0663f8dc2b5ff7f402faf7ed12..dd8649309d3ad2f6e71f240d7a5897eac79cc756 100644 (file)
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -957,7 +957,8 @@ nbproc <number>
   Creates <number> processes when going daemon. This requires the "daemon"
   mode. By default, only one process is created, which is the recommended mode
   of operation. For systems limited to small sets of file descriptors per
-  process, it may be needed to fork multiple daemons. USING MULTIPLE PROCESSES
+  process, it may be needed to fork multiple daemons. When set to a value
+  larger than 1, threads are automatically disabled. USING MULTIPLE PROCESSES
   IS HARDER TO DEBUG AND IS REALLY DISCOURAGED. See also "daemon" and
   "nbthread".
 
@@ -968,7 +969,13 @@ nbthread <number>
   also involved a number of shortcomings related to the lack of synchronization
   between processes (health-checks, peers, stick-tables, stats, ...) which do
   not affect threads. As such, any modern configuration is strongly encouraged
-  to migrate away from "nbproc" to "nbthread". See also "nbproc".
+  to migrate away from "nbproc" to "nbthread". "nbthread" also works when
+  HAProxy is started in foreground. On some platforms supporting CPU affinity,
+  when nbproc is not used, the default "nbthread" value is automatically set to
+  the number of CPUs the process is bound to upon startup. This means that the
+  thread count can easily be adjusted from the calling process using commands
+  like "taskset" or "cpuset". Otherwise, this value defaults to 1. The default
+  value is reported in the output of "haproxy -vv". See also "nbproc".
 
 pidfile <pidfile>
   Writes PIDs of all daemons into file <pidfile>. This option is equivalent to

diff --git a/include/common/hathreads.h b/include/common/hathreads.h
index e4603a5f9b40b1a6a4781ae1f440a045fdae16e7..0cd00fb00f632cafc548341aac3d472f45cc0da6 100644 (file)
--- a/include/common/hathreads.h
+++ b/include/common/hathreads.h
@@ -984,11 +984,14 @@ void ha_rwlock_init(HA_RWLOCK_T *l);
 
 #endif /* USE_THREAD */
 
+extern int thread_cpus_enabled_at_boot;
+
 static inline void __ha_compiler_barrier(void)
 {
        __asm __volatile("" ::: "memory");
 }
 
 int parse_nbthread(const char *arg, char **err);
+int thread_get_default_count();
 
 #endif /* _COMMON_HATHREADS_H */

diff --git a/src/cfgparse.c b/src/cfgparse.c
index 0833574e9ad1b2e826df91ef7c4cd751b81f13d2..bf737c6e618c98cab1778b936df68d97e7095cad 100644 (file)
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -2200,6 +2200,20 @@ int check_config_validity()
        if (!global.tune.requri_len)
                global.tune.requri_len = REQURI_LEN;
 
+       if (!global.nbthread) {
+               /* nbthread not set, thus automatic. In this case, and only if
+                * running on a single process, we enable the same number of
+                * threads as the number of CPUs the process is bound to. This
+                * allows to easily control the number of threads using taskset.
+                */
+               global.nbthread = 1;
+#if defined(USE_THREAD)
+               if (global.nbproc == 1)
+                       global.nbthread = thread_cpus_enabled_at_boot;
+               all_threads_mask = nbits(global.nbthread);
+#endif
+       }
+
        if (global.nbproc > 1 && global.nbthread > 1) {
                ha_alert("config : cannot enable multiple processes if multiple threads are configured. Please use either nbproc or nbthread but not both.\n");
                err_code |= ERR_ALERT | ERR_FATAL;

diff --git a/src/haproxy.c b/src/haproxy.c
index c9aaa236f63fb41a9786ec66996ceedf2b9ee9da..942c075976b7e3b967e7525ab4b23a43749f50ac 100644 (file)
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -136,7 +136,7 @@ volatile unsigned long sleeping_thread_mask; /* Threads that are about to sleep
 struct global global = {
        .hard_stop_after = TICK_ETERNITY,
        .nbproc = 1,
-       .nbthread = 1,
+       .nbthread = 0,
        .req_count = 0,
        .logsrvs = LIST_HEAD_INIT(global.logsrvs),
        .maxzlibmem = 0,

diff --git a/src/hathreads.c b/src/hathreads.c
index 8a4085ee66e538d577b3e10e1db9adb5fa41228e..3077e491337690dcd3bc1222d1844ccdac59ff83 100644 (file)
--- a/src/hathreads.c
+++ b/src/hathreads.c
@@ -10,10 +10,15 @@
  *
  */
 
+#define _GNU_SOURCE
 #include <unistd.h>
 #include <stdlib.h>
 #include <fcntl.h>
 
+#ifdef USE_CPU_AFFINITY
+#include <sched.h>
+#endif
+
 #include <common/cfgparse.h>
 #include <common/hathreads.h>
 #include <common/standard.h>
@@ -28,6 +33,7 @@ volatile unsigned long threads_harmless_mask = 0;
 volatile unsigned long all_threads_mask  = 1; // nbthread 1 assumed by default
 THREAD_LOCAL unsigned int  tid           = 0;
 THREAD_LOCAL unsigned long tid_bit       = (1UL << 0);
+int thread_cpus_enabled_at_boot          = 1;
 
 
 #if defined(DEBUG_THREAD) || defined(DEBUG_FULL)
@@ -105,6 +111,24 @@ void ha_rwlock_init(HA_RWLOCK_T *l)
        HA_RWLOCK_INIT(l);
 }
 
+/* returns the number of CPUs the current process is enabled to run on */
+static int thread_cpus_enabled()
+{
+       int ret = 1;
+
+#ifdef USE_CPU_AFFINITY
+#if defined(__linux__) && defined(CPU_COUNT)
+       cpu_set_t mask;
+
+       if (sched_getaffinity(0, sizeof(mask), &mask) == 0)
+               ret = CPU_COUNT(&mask);
+#endif
+#endif
+       ret = MAX(ret, 1);
+       ret = MIN(ret, MAX_THREADS);
+       return ret;
+}
+
 __attribute__((constructor))
 static void __hathreads_init(void)
 {
@@ -116,7 +140,11 @@ static void __hathreads_init(void)
                         LONGBITS, MAX_THREADS);
                exit(1);
        }
-       memprintf(&ptr, "Built with multi-threading support (MAX_THREADS=%d).", MAX_THREADS);
+
+       thread_cpus_enabled_at_boot = thread_cpus_enabled();
+
+       memprintf(&ptr, "Built with multi-threading support (MAX_THREADS=%d, default=%d).",
+                 MAX_THREADS, thread_cpus_enabled_at_boot);
        hap_register_build_opts(ptr, 1);
 
 #if defined(DEBUG_THREAD) || defined(DEBUG_FULL)