From 149ab779cccb822fd93a7c59c3cf48aff35732f0 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Sat, 26 Jan 2019 14:27:06 +0100 Subject: [PATCH] MAJOR: threads: enable one thread per CPU by default Threads have long matured by now, still for most users their usage is not trivial. It's about time to enable them by default on platforms where we know the number of CPUs bound. This patch does this, it counts the number of CPUs the process is bound to upon startup, and enables as many threads by default. Of course, "nbthread" still overrides this, but if it's not set the default behaviour is to start one thread per CPU. The default number of threads is reported in "haproxy -vv". Simply using "taskset -c" is now enough to adjust this number of threads so that there is no more need for playing with cpu-map. And thanks to the previous patches on the listener, the vast majority of configurations will not need to duplicate "bind" lines with the "process x/y" statement anymore either, so a simple config will automatically adapt to the number of processors available. --- doc/configuration.txt | 11 +++++++++-- include/common/hathreads.h | 3 +++ src/cfgparse.c | 14 ++++++++++++++ src/haproxy.c | 2 +- src/hathreads.c | 30 +++++++++++++++++++++++++++++- 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/doc/configuration.txt b/doc/configuration.txt index cd6ddf68bb..dd8649309d 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -957,7 +957,8 @@ nbproc Creates processes when going daemon. This requires the "daemon" mode. By default, only one process is created, which is the recommended mode of operation. For systems limited to small sets of file descriptors per - process, it may be needed to fork multiple daemons. USING MULTIPLE PROCESSES + process, it may be needed to fork multiple daemons. When set to a value + larger than 1, threads are automatically disabled. USING MULTIPLE PROCESSES IS HARDER TO DEBUG AND IS REALLY DISCOURAGED. See also "daemon" and "nbthread". @@ -968,7 +969,13 @@ nbthread also involved a number of shortcomings related to the lack of synchronization between processes (health-checks, peers, stick-tables, stats, ...) which do not affect threads. As such, any modern configuration is strongly encouraged - to migrate away from "nbproc" to "nbthread". See also "nbproc". + to migrate away from "nbproc" to "nbthread". "nbthread" also works when + HAProxy is started in foreground. On some platforms supporting CPU affinity, + when nbproc is not used, the default "nbthread" value is automatically set to + the number of CPUs the process is bound to upon startup. This means that the + thread count can easily be adjusted from the calling process using commands + like "taskset" or "cpuset". Otherwise, this value defaults to 1. The default + value is reported in the output of "haproxy -vv". See also "nbproc". pidfile Writes PIDs of all daemons into file . This option is equivalent to diff --git a/include/common/hathreads.h b/include/common/hathreads.h index e4603a5f9b..0cd00fb00f 100644 --- a/include/common/hathreads.h +++ b/include/common/hathreads.h @@ -984,11 +984,14 @@ void ha_rwlock_init(HA_RWLOCK_T *l); #endif /* USE_THREAD */ +extern int thread_cpus_enabled_at_boot; + static inline void __ha_compiler_barrier(void) { __asm __volatile("" ::: "memory"); } int parse_nbthread(const char *arg, char **err); +int thread_get_default_count(); #endif /* _COMMON_HATHREADS_H */ diff --git a/src/cfgparse.c b/src/cfgparse.c index 0833574e9a..bf737c6e61 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -2200,6 +2200,20 @@ int check_config_validity() if (!global.tune.requri_len) global.tune.requri_len = REQURI_LEN; + if (!global.nbthread) { + /* nbthread not set, thus automatic. In this case, and only if + * running on a single process, we enable the same number of + * threads as the number of CPUs the process is bound to. This + * allows to easily control the number of threads using taskset. + */ + global.nbthread = 1; +#if defined(USE_THREAD) + if (global.nbproc == 1) + global.nbthread = thread_cpus_enabled_at_boot; + all_threads_mask = nbits(global.nbthread); +#endif + } + if (global.nbproc > 1 && global.nbthread > 1) { ha_alert("config : cannot enable multiple processes if multiple threads are configured. Please use either nbproc or nbthread but not both.\n"); err_code |= ERR_ALERT | ERR_FATAL; diff --git a/src/haproxy.c b/src/haproxy.c index c9aaa236f6..942c075976 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -136,7 +136,7 @@ volatile unsigned long sleeping_thread_mask; /* Threads that are about to sleep struct global global = { .hard_stop_after = TICK_ETERNITY, .nbproc = 1, - .nbthread = 1, + .nbthread = 0, .req_count = 0, .logsrvs = LIST_HEAD_INIT(global.logsrvs), .maxzlibmem = 0, diff --git a/src/hathreads.c b/src/hathreads.c index 8a4085ee66..3077e49133 100644 --- a/src/hathreads.c +++ b/src/hathreads.c @@ -10,10 +10,15 @@ * */ +#define _GNU_SOURCE #include #include #include +#ifdef USE_CPU_AFFINITY +#include +#endif + #include #include #include @@ -28,6 +33,7 @@ volatile unsigned long threads_harmless_mask = 0; volatile unsigned long all_threads_mask = 1; // nbthread 1 assumed by default THREAD_LOCAL unsigned int tid = 0; THREAD_LOCAL unsigned long tid_bit = (1UL << 0); +int thread_cpus_enabled_at_boot = 1; #if defined(DEBUG_THREAD) || defined(DEBUG_FULL) @@ -105,6 +111,24 @@ void ha_rwlock_init(HA_RWLOCK_T *l) HA_RWLOCK_INIT(l); } +/* returns the number of CPUs the current process is enabled to run on */ +static int thread_cpus_enabled() +{ + int ret = 1; + +#ifdef USE_CPU_AFFINITY +#if defined(__linux__) && defined(CPU_COUNT) + cpu_set_t mask; + + if (sched_getaffinity(0, sizeof(mask), &mask) == 0) + ret = CPU_COUNT(&mask); +#endif +#endif + ret = MAX(ret, 1); + ret = MIN(ret, MAX_THREADS); + return ret; +} + __attribute__((constructor)) static void __hathreads_init(void) { @@ -116,7 +140,11 @@ static void __hathreads_init(void) LONGBITS, MAX_THREADS); exit(1); } - memprintf(&ptr, "Built with multi-threading support (MAX_THREADS=%d).", MAX_THREADS); + + thread_cpus_enabled_at_boot = thread_cpus_enabled(); + + memprintf(&ptr, "Built with multi-threading support (MAX_THREADS=%d, default=%d).", + MAX_THREADS, thread_cpus_enabled_at_boot); hap_register_build_opts(ptr, 1); #if defined(DEBUG_THREAD) || defined(DEBUG_FULL) -- 2.39.5