MINOR: threads: Add thread-map config parameter in the global section

By default, no affinity is set for threads. To bind threads on CPU, you must
define a "thread-map" in the global section. The format is the same than the
"cpu-map" parameter, with a small difference. The process number must be
defined, with the same format than cpu-map ("all", "even", "odd" or a number
between 1 and 31/63).

A thread will be bound on the intersection of its mapping and the one of the
process on which it is attached. If the intersection is null, no specific bind
will be set for the thread.

diff --git a/doc/configuration.txt b/doc/configuration.txt
index 6f7a99ff612b03906b3103d9be61167654c93048..786e5c3c1eb695de39c84430dc0e3428c3443c86 100644 (file)
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -558,6 +558,7 @@ The following keywords are supported in the "global" section :
    - ssl-default-server-options
    - ssl-dh-param-file
    - ssl-server-verify
+   - thread-map
    - unix-bind
    - unsetenv
    - 51degrees-data-file
@@ -943,6 +944,15 @@ ssl-server-verify [none|required]
   servers certificates are not verified. The default is 'required' except if
   forced using cmdline option '-dV'.
 
+
+thread-map <"all"|"odd"|"even"|process_num> <"all"|"odd"|"even"|thread_num> <cpu-set>...
+  This setting is only available when support for threads was built in. It
+  binds a thread to a specific CPU set. The process must be specified to allow
+  different mapping for different processes. For details about the arguments,
+  see "cpu-map" directive. A thread will be bound on the intersection of its
+  mapping and the one of the process on which it is attached. If the
+  intersection is null, no specific bind will be set for the thread.
+
 stats socket [<address:port>|<path>] [param*]
   Binds a UNIX socket to <path> or a TCPv4/v6 address to <address:port>.
   Connections to this socket will return various statistics outputs and even

diff --git a/include/types/global.h b/include/types/global.h
index 817c09ba69b5f229b2de64cd756724d58eb2a0b4..b3aa29cca46239d24c43d2feee0f952c2a2b47ba 100644 (file)
--- a/include/types/global.h
+++ b/include/types/global.h
@@ -162,7 +162,12 @@ struct global {
                } ux;
        } unix_bind;
 #ifdef USE_CPU_AFFINITY
-       unsigned long cpu_map[LONGBITS];  /* list of CPU masks for the 32/64 first processes */
+       unsigned long cpu_map[LONGBITS];              /* list of CPU masks for the 32/64 first processes */
+
+#ifdef USE_THREAD
+       unsigned long thread_map[LONGBITS][LONGBITS]; /* list of CPU masks for the 32/64 first threads per process */
+#endif
+
 #endif
        struct proxy *stats_fe;     /* the frontend holding the stats settings */
        struct vars   vars;         /* list of variables for the process scope. */

diff --git a/src/cfgparse.c b/src/cfgparse.c
index d693bfbf568c9b45de23a9b6d38f282480823bfc..4414f59c0632c71418fe53d94da7d8c3cf123569 100644 (file)
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -586,6 +586,72 @@ static int warnif_cond_conflicts(const struct acl_cond *cond, unsigned int where
        return ERR_WARN;
 }
 
+/* Parse a string representing a process number or a set of processes. It must
+ * be "all", "odd", "even" or a number between 1 and <LONGBITS>. It returns a
+ * mask where bits are set for corresponding processes or 0 if an error occured.
+ *
+ * Note: this function can also be used to parse a thread number or a set of
+ * threads.
+ */
+static unsigned long parse_process_number(const char *arg)
+{
+       unsigned long proc = 0;
+
+       if (strcmp(arg, "all") == 0)
+               proc = ~0UL;
+       else if (strcmp(arg, "odd") == 0)
+               proc = ~0UL/3UL; /* 0x555....555 */
+       else if (strcmp(arg, "even") == 0)
+               proc = (~0UL/3UL) << 1; /* 0xAAA...AAA */
+       else {
+               proc = atol(arg);
+               if (proc >= 1 && proc <= LONGBITS)
+                       proc = 1UL << (proc - 1);
+       }
+       return proc;
+}
+
+/* Parse cpu sets. Each CPU set is either a unique number between 0 and
+ * <LONGBITS> or a range with two such numbers delimited by a dash
+ * ('-'). Multiple CPU numbers or ranges may be specified. On success, it
+ * returns 0. otherwise it returns 1 with an error message in <err>.
+ */
+static unsigned long parse_cpu_set(const char **args, unsigned long *cpu_set, char **err)
+{
+       int cur_arg = 0;
+
+       *cpu_set = 0;
+       while (*args[cur_arg]) {
+               char        *dash;
+               unsigned int low, high;
+
+               if (!isdigit((int)*args[cur_arg])) {
+                       memprintf(err, "'%s' is not a CPU range.\n", args[cur_arg]);
+                       return -1;
+               }
+
+               low = high = str2uic(args[cur_arg]);
+               if ((dash = strchr(args[cur_arg], '-')) != NULL)
+                       high = str2uic(dash + 1);
+
+               if (high < low) {
+                       unsigned int swap = low;
+                       low = high;
+                       high = swap;
+               }
+
+               if (high >= LONGBITS) {
+                       memprintf(err, "supports CPU numbers from 0 to %d.\n", LONGBITS - 1);
+                       return 1;
+               }
+
+               while (low <= high)
+                       *cpu_set |= 1UL << low++;
+
+               cur_arg++;
+       }
+       return 0;
+}
 /*
  * parse a line in a <global> section. Returns the error code, 0 if OK, or
  * any combination of :
@@ -1602,74 +1668,77 @@ int cfg_parse_global(const char *file, int linenum, char **args, int kwm)
                        err_code |= ERR_ALERT | ERR_FATAL;
                }
        }
-       else if (strcmp(args[0], "cpu-map") == 0) {  /* map a process list to a CPU set */
+       else if (strcmp(args[0], "cpu-map") == 0) {
+               /* map a process list to a CPU set */
 #ifdef USE_CPU_AFFINITY
-               int cur_arg, i;
-               unsigned long proc = 0;
-               unsigned long cpus = 0;
-
-               if (strcmp(args[1], "all") == 0)
-                       proc = ~0UL;
-               else if (strcmp(args[1], "odd") == 0)
-                       proc = ~0UL/3UL; /* 0x555....555 */
-               else if (strcmp(args[1], "even") == 0)
-                       proc = (~0UL/3UL) << 1; /* 0xAAA...AAA */
-               else {
-                       proc = atol(args[1]);
-                       if (proc >= 1 && proc <= LONGBITS)
-                               proc = 1UL << (proc - 1);
-               }
+               unsigned long proc, cpus;
+               int i;
 
+               proc = parse_process_number(args[1]);
                if (!proc || !*args[2]) {
-                       Alert("parsing [%s:%d]: %s expects a process number including 'all', 'odd', 'even', or a number from 1 to %d, followed by a list of CPU ranges with numbers from 0 to %d.\n",
+                       Alert("parsing [%s:%d]: %s expects a process number "
+                             " ('all', 'odd', 'even', or a number from 1 to %d), "
+                             " followed by a list of CPU ranges with numbers from 0 to %d.\n",
                              file, linenum, args[0], LONGBITS, LONGBITS - 1);
                        err_code |= ERR_ALERT | ERR_FATAL;
                        goto out;
                }
-
-               cur_arg = 2;
-               while (*args[cur_arg]) {
-                       unsigned int low, high;
-
-                       if (isdigit((int)*args[cur_arg])) {
-                               char *dash = strchr(args[cur_arg], '-');
-
-                               low = high = str2uic(args[cur_arg]);
-                               if (dash)
-                                       high = str2uic(dash + 1);
-
-                               if (high < low) {
-                                       unsigned int swap = low;
-                                       low = high;
-                                       high = swap;
-                               }
-
-                               if (high >= LONGBITS) {
-                                       Alert("parsing [%s:%d]: %s supports CPU numbers from 0 to %d.\n",
-                                             file, linenum, args[0], LONGBITS - 1);
-                                       err_code |= ERR_ALERT | ERR_FATAL;
-                                       goto out;
-                               }
-
-                               while (low <= high)
-                                       cpus |= 1UL << low++;
-                       }
-                       else {
-                               Alert("parsing [%s:%d]: %s : '%s' is not a CPU range.\n",
-                                     file, linenum, args[0], args[cur_arg]);
-                               err_code |= ERR_ALERT | ERR_FATAL;
-                               goto out;
-                       }
-                       cur_arg++;
+               if (parse_cpu_set((const char **)args+2, &cpus, &errmsg)) {
+                       Alert("parsing [%s:%d] : %s : %s\n", file, linenum, args[0], errmsg);
+                       err_code |= ERR_ALERT | ERR_FATAL;
+                       goto out;
                }
                for (i = 0; i < LONGBITS; i++)
                        if (proc & (1UL << i))
                                global.cpu_map[i] = cpus;
 #else
-               Alert("parsing [%s:%d] : '%s' is not enabled, please check build options for USE_CPU_AFFINITY.\n", file, linenum, args[0]);
+               Alert("parsing [%s:%d] : '%s' is not enabled, please check build options for USE_CPU_AFFINITY.\n",
+                     file, linenum, args[0]);
                err_code |= ERR_ALERT | ERR_FATAL;
                goto out;
-#endif
+#endif /* ! USE_CPU_AFFINITY */
+       }
+       else if (strcmp(args[0], "thread-map") == 0) {
+               /* map a thread list to a CPU set */
+#ifdef USE_CPU_AFFINITY
+#ifdef USE_THREAD
+               unsigned long proc, thread, cpus;
+               int i, j;
+
+               proc    = parse_process_number(args[1]);
+               thread  = parse_process_number(args[2]);
+               if (!proc || !thread || !*args[3]) {
+                       Alert("parsing [%s:%d]: %s expects a process number "
+                             "('all', 'odd', 'even', or a number from 1 to %d), "
+                             " followed by a thread number using the same format, "
+                             " followed by a list of CPU ranges with numbers from 0 to %d.\n",
+                             file, linenum, args[0], LONGBITS, LONGBITS - 1);
+                       err_code |= ERR_ALERT | ERR_FATAL;
+                       goto out;
+               }
+               if (parse_cpu_set((const char **)args+3, &cpus, &errmsg)) {
+                       Alert("parsing [%s:%d] : %s : %s\n", file, linenum, args[0], errmsg);
+                       err_code |= ERR_ALERT | ERR_FATAL;
+                       goto out;
+               }
+               for (i = 0; i < LONGBITS; i++)
+                       if (proc & (1UL << i)) {
+                               for (j = 0; j < LONGBITS; j++)
+                                       if (thread & (1UL << j))
+                                               global.thread_map[i][j] = cpus;
+                       }
+#else
+               Alert("parsing [%s:%d] : '%s' is not enabled, please check build options for USE_THREAD.\n",
+                     file, linenum, args[0]);
+               err_code |= ERR_ALERT | ERR_FATAL;
+               goto out;
+#endif /* ! USE_THREAD*/
+#else
+               Alert("parsing [%s:%d] : '%s' is not enabled, please check build options for USE_CPU_AFFINITY.\n",
+                     file, linenum, args[0]);
+               err_code |= ERR_ALERT | ERR_FATAL;
+               goto out;
+#endif /* ! USE_CPU_AFFINITY */
        }
        else if (strcmp(args[0], "setenv") == 0 || strcmp(args[0], "presetenv") == 0) {
                if (alertif_too_many_args(3, file, linenum, args, &err_code))

diff --git a/src/haproxy.c b/src/haproxy.c
index 30ac15737e53795537ac260b154bf8ed5bbdf664..f4353e16549b9e39386813fdee24851995a9ce5c 100644 (file)
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -2814,6 +2814,15 @@ int main(int argc, char **argv)
                for (i = 0; i < global.nbthread; i++) {
                        tids[i] = i;
                        pthread_create(&threads[i], NULL, &run_thread_poll_loop, &tids[i]);
+#ifdef USE_CPU_AFFINITY
+                       if (global.cpu_map[relative_pid-1])
+                               global.thread_map[relative_pid-1][i] &= global.cpu_map[relative_pid-1];
+
+                       if (i < LONGBITS &&       /* only the first 32/64 threads may be pinned */
+                           global.thread_map[relative_pid-1][i]) /* only do this if the thread has a THREAD map */
+                               pthread_setaffinity_np(threads[i],
+                                                      sizeof(unsigned long), (void *)&global.thread_map[relative_pid-1][i]);
+#endif
                }
                for (i = 0; i < global.nbthread; i++)
                        pthread_join(threads[i], NULL);
@@ -2830,6 +2839,17 @@ int main(int argc, char **argv)
        else {
                tid = 0;
 
+#ifdef USE_THREAD
+#ifdef USE_CPU_AFFINITY
+               if (global.cpu_map[relative_pid-1])
+                       global.thread_map[relative_pid-1][tid] &= global.cpu_map[relative_pid-1];
+
+               if (global.thread_map[relative_pid-1][tid]) /* only do this if the thread has a THREAD map */
+                       pthread_setaffinity_np(pthread_self(),
+                                              sizeof(unsigned long), (void *)&global.thread_map[relative_pid-1][tid]);
+#endif
+#endif
+
                if (global.mode & MODE_MWORKER)
                        mworker_pipe_register(mworker_pipe);