MEDIUM: global: add support for CPU binding on Linux ("cpu-map")

The new "cpu-map" directive allows one to assign the CPU sets that
a process is allowed to bind to. This is useful in combination with
the "nbproc" and "bind-process" directives.

The support is implicit on Linux 2.6.28 and above.

diff --git a/Makefile b/Makefile
index f13848c470f8c1ff0e3a298728340a83fee357b4..9c7a75f23292a095541319a49791070b3ee3a32b 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -30,6 +30,7 @@
 #   USE_ACCEPT4          : enable use of accept4() on linux. Automatic.
 #   USE_MY_ACCEPT4       : use own implemention of accept4() if glibc < 2.10.
 #   USE_ZLIB             : enable zlib library support.
+#   USE_CPU_AFFINITY     : enable pinning processes to CPU on Linux. Automatic.
 #
 # Options can be forced by specifying "USE_xxx=1" or can be disabled by using
 # "USE_xxx=" (empty string).
@@ -241,6 +242,7 @@ ifeq ($(TARGET),linux2628)
   USE_LINUX_TPROXY= implicit
   USE_ACCEPT4     = implicit
   USE_FUTEX       = implicit
+  USE_CPU_AFFINITY= implicit
 else
 ifeq ($(TARGET),solaris)
   # This is for Solaris 8
@@ -437,6 +439,11 @@ OPTIONS_CFLAGS += -DCONFIG_HAP_LINUX_VSYSCALL
 BUILD_OPTIONS  += $(call ignore_implicit,USE_VSYSCALL)
 endif
 
+ifneq ($(USE_CPU_AFFINITY),)
+OPTIONS_CFLAGS += -DUSE_CPU_AFFINITY
+BUILD_OPTIONS  += $(call ignore_implicit,USE_CPU_AFFINITY)
+endif
+
 ifneq ($(USE_MY_SPLICE),)
 OPTIONS_CFLAGS += -DUSE_MY_SPLICE
 BUILD_OPTIONS  += $(call ignore_implicit,USE_MY_SPLICE)

diff --git a/doc/configuration.txt b/doc/configuration.txt
index 4d250155dc3834b8aab4040d5fdff32580b4387f..84a3f9a4836f49b58f3ee7141a80124a1ac86efa 100644 (file)
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -499,6 +499,21 @@ chroot <jail dir>
   with superuser privileges. It is important to ensure that <jail_dir> is both
   empty and unwritable to anyone.
 
+cpu-map <"all"|"odd"|"even"|process_num> <cpu-set>...
+  On Linux 2.6 and above, it is possible to bind a process to a specific CPU
+  set. This means that the process will never run on other CPUs. The "cpu-map"
+  directive specifies CPU sets for process sets. The first argument is the
+  process number to bind. This process must have a number between 1 and 32,
+  and any process IDs above nbproc are ignored. It is possible to specify all
+  processes at once using "all", only odd numbers using "odd" or even numbers
+  using "even", just like with the "bind-process" directive. The second and
+  forthcoming arguments are CPU sets. Each CPU set is either a unique number
+  between 0 and 31 or a range with two such numbers delimited by a dash ('-').
+  Multiple CPU numbers or ranges may be specified, and the processes will be
+  allowed to bind to all of them. Obviously, multiple "cpu-map" directives may
+  be specified. Each "cpu-map" directive will replace the previous ones when
+  they overlap.
+
 crt-base <dir>
   Assigns a default directory to fetch SSL certificates from when a relative
   path is used with "crtfile" directives. Absolute locations specified after

diff --git a/include/types/global.h b/include/types/global.h
index 9d2eedf4d37216e6f323605fa04ce77e9a20cf68..3cd0772453021e60f4cfcda35295e1fe72a2c31c 100644 (file)
--- a/include/types/global.h
+++ b/include/types/global.h
@@ -130,6 +130,9 @@ struct global {
                        int level;      /* access level (ACCESS_LVL_*) */
                } ux;
        } unix_bind;
+#ifdef USE_CPU_AFFINITY
+       unsigned long cpu_map[32];  /* list of CPU masks for the 32 first processes */
+#endif
        struct proxy *stats_fe;     /* the frontend holding the stats settings */
 };
 

diff --git a/src/cfgparse.c b/src/cfgparse.c
index 13363dbccccb0212310343d1a5dbc2308aef28da..569fcd354e5720adee562f60bf2f21aa7f6cdb49 100644 (file)
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -1160,6 +1160,75 @@ int cfg_parse_global(const char *file, int linenum, char **args, int kwm)
                        err_code |= ERR_ALERT | ERR_FATAL;
                }
        }
+       else if (strcmp(args[0], "cpu-map") == 0) {  /* map a process list to a CPU set */
+#ifdef USE_CPU_AFFINITY
+               int cur_arg, i;
+               unsigned int proc = 0;
+               unsigned long cpus = 0;
+
+               if (strcmp(args[1], "all") == 0)
+                       proc = 0xFFFFFFFF;
+               else if (strcmp(args[1], "odd") == 0)
+                       proc = 0x55555555;
+               else if (strcmp(args[1], "even") == 0)
+                       proc = 0xAAAAAAAA;
+               else {
+                       proc = atoi(args[1]);
+                       if (proc >= 1 && proc <= 32)
+                               proc = 1 << (proc - 1);
+               }
+
+               if (!proc || !*args[2]) {
+                       Alert("parsing [%s:%d]: %s expects a process number including 'all', 'odd', 'even', or a number from 1 to 32, followed by a list of CPU ranges with numbers from 0 to 31.\n",
+                             file, linenum, args[0]);
+                       err_code |= ERR_ALERT | ERR_FATAL;
+                       goto out;
+               }
+
+               cur_arg = 2;
+               while (*args[cur_arg]) {
+                       unsigned int low, high;
+
+                       if (isdigit(*args[cur_arg])) {
+                               char *dash = strchr(args[cur_arg], '-');
+
+                               low = high = str2uic(args[cur_arg]);
+                               if (dash)
+                                       high = str2uic(dash + 1);
+
+                               if (high < low) {
+                                       unsigned int swap = low;
+                                       low = high;
+                                       high = swap;
+                               }
+
+                               if (low < 0 || high >= sizeof(long) * 8) {
+                                       Alert("parsing [%s:%d]: %s supports CPU numbers from 0 to %d.\n",
+                                             file, linenum, args[0], (int)(sizeof(long) * 8 - 1));
+                                       err_code |= ERR_ALERT | ERR_FATAL;
+                                       goto out;
+                               }
+
+                               while (low <= high)
+                                       cpus |= 1UL << low++;
+                       }
+                       else {
+                               Alert("parsing [%s:%d]: %s : '%s' is not a CPU range.\n",
+                                     file, linenum, args[0], args[cur_arg]);
+                               err_code |= ERR_ALERT | ERR_FATAL;
+                               goto out;
+                       }
+                       cur_arg++;
+               }
+               for (i = 0; i < 32; i++)
+                       if (proc & (1 << i))
+                               global.cpu_map[i] = cpus;
+#else
+               Alert("parsing [%s:%d] : '%s' is not enabled, please check build options for USE_CPU_AFFINITY.\n", file, linenum, args[0]);
+               err_code |= ERR_ALERT | ERR_FATAL;
+               goto out;
+#endif
+       }
        else {
                struct cfg_kw_list *kwl;
                int index;

diff --git a/src/haproxy.c b/src/haproxy.c
index c4122e2d7ca41ff11c47575acb8eb678e4f18cee..20fb56d75ca08af7258528285202d3c1ada09971 100644 (file)
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -1,6 +1,6 @@
 /*
  * HA-Proxy : High Availability-enabled HTTP/TCP proxy
- * Copyright 2000-2011  Willy Tarreau <w@1wt.eu>.
+ * Copyright 2000-2012  Willy Tarreau <w@1wt.eu>.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -44,6 +44,11 @@
 #include <sys/resource.h>
 #include <time.h>
 #include <syslog.h>
+#ifdef USE_CPU_AFFINITY
+#define __USE_GNU
+#include <sched.h>
+#undef __USE_GNU
+#endif
 
 #ifdef DEBUG_FULL
 #include <assert.h>
@@ -1467,6 +1472,13 @@ int main(int argc, char **argv)
                        }
                        relative_pid++; /* each child will get a different one */
                }
+
+#ifdef USE_CPU_AFFINITY
+               if (proc < global.nbproc &&  /* child */
+                   proc < 32 &&             /* only the first 32 processes may be pinned */
+                   global.cpu_map[proc])    /* only do this if the process has a CPU map */
+                       sched_setaffinity(0, sizeof(unsigned long), (void *)&global.cpu_map[proc]);
+#endif
                /* close the pidfile both in children and father */
                if (pidfd >= 0) {
                        //lseek(pidfd, 0, SEEK_SET);  /* debug: emulate eglibc bug */