From: Willy Tarreau Date: Tue, 18 Jul 2023 12:04:10 +0000 (+0200) Subject: MEDIUM: thread: start to detect thread groups and threads min/max X-Git-Tag: v3.2-dev8~72 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1af4942c95ce261a53db83d54565fa1d495ad424;p=thirdparty%2Fhaproxy.git MEDIUM: thread: start to detect thread groups and threads min/max By mutually refining the thread count and group count, we can try to detect the most suitable setup for the current machine. Taskset is implicitly handled correctly. tgroups automatically adapt to the configured number of threads. cpu-map manages to limit tgroups to the smallest supported value. The thread-limit is enforced. Just like in cfgparse, if the thread count was forced to a higher value, it's reduced and a warning is emitted. But if it was not set, the thr_max value is bound to this limit so that further calculations respect it. We continue to default to the max number of available threads and 1 tgroup by default, with the limit. This normally allows to get rid of that test in check_config_validity(). --- diff --git a/include/haproxy/thread.h b/include/haproxy/thread.h index 7233b6754..d99146766 100644 --- a/include/haproxy/thread.h +++ b/include/haproxy/thread.h @@ -47,6 +47,7 @@ int thread_detect_binding_discrepancies(void); int thread_detect_more_than_cpus(void); int thread_map_to_groups(); int thread_resolve_group_mask(struct thread_set *ts, int defgrp, char **err); +void thread_detect_count(void); int parse_thread_set(const char *arg, struct thread_set *ts, char **err); extern int thread_cpus_enabled_at_boot; diff --git a/src/haproxy.c b/src/haproxy.c index b6a72e3a2..3beba0461 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -2067,6 +2067,9 @@ static void step_init_2(int argc, char** argv) cpu_detect_topology(); #endif + /* detect the optimal thread-groups and nbthreads if not set */ + thread_detect_count(); + /* Note: global.nbthread will be initialized as part of this call */ err_code |= check_config_validity(); if (*initial_cwd && chdir(initial_cwd) == -1) { diff --git a/src/thread.c b/src/thread.c index 64d16acd3..deb4bc22d 100644 --- a/src/thread.c +++ b/src/thread.c @@ -1538,6 +1538,103 @@ int thread_resolve_group_mask(struct thread_set *ts, int defgrp, char **err) return 0; } +/* Tries to guess the best thread group count and thread count depending on + * (possibly) existing values, presence or not of cpu-map, of a forced + * taskset, etc. + */ +void thread_detect_count(void) +{ + int thr_min, thr_max; + int grp_min __maybe_unused; + int grp_max __maybe_unused; + int cpus_avail __maybe_unused; + int cpu __maybe_unused; + + thr_min = 1; thr_max = MAX_THREADS; + grp_min = 1; grp_max = MAX_TGROUPS; + + if (global.thread_limit && global.nbthread > global.thread_limit) { + ha_warning("nbthread forced to a higher value (%d) than the configured thread-hard-limit (%d), enforcing the limit. " + "Please fix either value to remove this warning.\n", + global.nbthread, global.thread_limit); + global.nbthread = global.thread_limit; + } + + /* config forces both values */ + if (global.nbthread) + thr_min = thr_max = global.nbthread; + + if (global.nbtgroups) + grp_min = grp_max = global.nbtgroups; + + /* Adjust to boot settings if not forced */ + if (thr_min <= thread_cpus_enabled_at_boot && thread_cpus_enabled_at_boot < thr_max) + thr_max = thread_cpus_enabled_at_boot; + + if (global.thread_limit && thr_max > global.thread_limit) + thr_max = global.thread_limit; + +#if defined(USE_THREAD) && defined(USE_CPU_AFFINITY) + /* consider the number of online CPUs as an upper limit if set */ + cpus_avail = 0; + for (cpu = 0; cpu <= cpu_topo_lastcpu; cpu++) + if (!(ha_cpu_topo[cpu].st & HA_CPU_F_OFFLINE)) + cpus_avail++; + + if (thr_min <= cpus_avail && cpus_avail < thr_max) + thr_max = cpus_avail; + + /* make sure values are consistent */ + if (thr_min < grp_min && thr_max >= grp_min) + thr_min = grp_min; + + if (thr_min <= MAX_THREADS_PER_GROUP * grp_max && + thr_max > MAX_THREADS_PER_GROUP * grp_max) + thr_max = MAX_THREADS_PER_GROUP * grp_max; + + if (grp_min < (thr_min + MAX_THREADS_PER_GROUP - 1) / MAX_THREADS_PER_GROUP && + grp_max >= (thr_min + MAX_THREADS_PER_GROUP - 1) / MAX_THREADS_PER_GROUP) + grp_min = (thr_min + MAX_THREADS_PER_GROUP - 1) / MAX_THREADS_PER_GROUP; + + if (grp_max > thr_max && grp_min <= thr_max) + grp_max = thr_max; + + if (grp_min < grp_max && cpu_map_configured()) { + /* if a cpu-map directive is set, we cannot reliably infer what + * CPUs will be used anymore, so we'll use the smallest permitted + * number of groups. + */ + grp_max = grp_min; + } + + /* now, if the thr_min < thr_max this means that we're supposed to + * figure the best set of CPUs to use. E.g. use a single cluster on + * a complex set. Thus we can try to select the best clusters in + * capacity order until we reach at least thr_min, then continue + * on the same cluster _capacity_ up to thr_max. + */ +#endif // USE_THREAD && USE_CPU_AFFINITY + + if (!global.nbthread) + global.nbthread = thr_max; + + if (!global.nbtgroups) + global.nbtgroups = 1; + + if (global.nbthread > MAX_THREADS_PER_GROUP * global.nbtgroups) { + ha_diag_warning("nbthread too large or not set, found %d CPUs, limiting to %d threads (maximum is %d per thread group and %d groups). Please set nbthreads and/or increase thread-groups in the global section to silence this warning.\n", + global.nbthread, MAX_THREADS_PER_GROUP * global.nbtgroups, MAX_THREADS_PER_GROUP, MAX_TGROUPS); + global.nbthread = MAX_THREADS_PER_GROUP * global.nbtgroups; + } + +#if defined(USE_THREAD) && defined(USE_CPU_AFFINITY) + if (global.tune.debug & GDBG_CPU_AFFINITY) + cpu_dump_topology(ha_cpu_topo); +#endif + return; +} + + /* Parse a string representing a thread set in one of the following forms: * * - { "all" | "odd" | "even" | [ "-" ] }[,...]