MEDIUM: global: add support for CPU binding on Linux ("cpu-map")
The new "cpu-map" directive allows one to assign the CPU sets that
a process is allowed to bind to. This is useful in combination with
the "nbproc" and "bind-process" directives.
The support is implicit on Linux 2.6.28 and above.
diff --git a/Makefile b/Makefile
index f13848c..9c7a75f 100644
--- a/Makefile
+++ b/Makefile
@@ -30,6 +30,7 @@
# USE_ACCEPT4 : enable use of accept4() on linux. Automatic.
# USE_MY_ACCEPT4 : use own implemention of accept4() if glibc < 2.10.
# USE_ZLIB : enable zlib library support.
+# USE_CPU_AFFINITY : enable pinning processes to CPU on Linux. Automatic.
#
# Options can be forced by specifying "USE_xxx=1" or can be disabled by using
# "USE_xxx=" (empty string).
@@ -241,6 +242,7 @@
USE_LINUX_TPROXY= implicit
USE_ACCEPT4 = implicit
USE_FUTEX = implicit
+ USE_CPU_AFFINITY= implicit
else
ifeq ($(TARGET),solaris)
# This is for Solaris 8
@@ -437,6 +439,11 @@
BUILD_OPTIONS += $(call ignore_implicit,USE_VSYSCALL)
endif
+ifneq ($(USE_CPU_AFFINITY),)
+OPTIONS_CFLAGS += -DUSE_CPU_AFFINITY
+BUILD_OPTIONS += $(call ignore_implicit,USE_CPU_AFFINITY)
+endif
+
ifneq ($(USE_MY_SPLICE),)
OPTIONS_CFLAGS += -DUSE_MY_SPLICE
BUILD_OPTIONS += $(call ignore_implicit,USE_MY_SPLICE)
diff --git a/doc/configuration.txt b/doc/configuration.txt
index 4d25015..84a3f9a 100644
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -499,6 +499,21 @@
with superuser privileges. It is important to ensure that <jail_dir> is both
empty and unwritable to anyone.
+cpu-map <"all"|"odd"|"even"|process_num> <cpu-set>...
+ On Linux 2.6 and above, it is possible to bind a process to a specific CPU
+ set. This means that the process will never run on other CPUs. The "cpu-map"
+ directive specifies CPU sets for process sets. The first argument is the
+ process number to bind. This process must have a number between 1 and 32,
+ and any process IDs above nbproc are ignored. It is possible to specify all
+ processes at once using "all", only odd numbers using "odd" or even numbers
+ using "even", just like with the "bind-process" directive. The second and
+ forthcoming arguments are CPU sets. Each CPU set is either a unique number
+ between 0 and 31 or a range with two such numbers delimited by a dash ('-').
+ Multiple CPU numbers or ranges may be specified, and the processes will be
+ allowed to bind to all of them. Obviously, multiple "cpu-map" directives may
+ be specified. Each "cpu-map" directive will replace the previous ones when
+ they overlap.
+
crt-base <dir>
Assigns a default directory to fetch SSL certificates from when a relative
path is used with "crtfile" directives. Absolute locations specified after
diff --git a/include/types/global.h b/include/types/global.h
index 9d2eedf..3cd0772 100644
--- a/include/types/global.h
+++ b/include/types/global.h
@@ -130,6 +130,9 @@
int level; /* access level (ACCESS_LVL_*) */
} ux;
} unix_bind;
+#ifdef USE_CPU_AFFINITY
+ unsigned long cpu_map[32]; /* list of CPU masks for the 32 first processes */
+#endif
struct proxy *stats_fe; /* the frontend holding the stats settings */
};
diff --git a/src/cfgparse.c b/src/cfgparse.c
index 13363db..569fcd3 100644
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -1160,6 +1160,75 @@
err_code |= ERR_ALERT | ERR_FATAL;
}
}
+ else if (strcmp(args[0], "cpu-map") == 0) { /* map a process list to a CPU set */
+#ifdef USE_CPU_AFFINITY
+ int cur_arg, i;
+ unsigned int proc = 0;
+ unsigned long cpus = 0;
+
+ if (strcmp(args[1], "all") == 0)
+ proc = 0xFFFFFFFF;
+ else if (strcmp(args[1], "odd") == 0)
+ proc = 0x55555555;
+ else if (strcmp(args[1], "even") == 0)
+ proc = 0xAAAAAAAA;
+ else {
+ proc = atoi(args[1]);
+ if (proc >= 1 && proc <= 32)
+ proc = 1 << (proc - 1);
+ }
+
+ if (!proc || !*args[2]) {
+ Alert("parsing [%s:%d]: %s expects a process number including 'all', 'odd', 'even', or a number from 1 to 32, followed by a list of CPU ranges with numbers from 0 to 31.\n",
+ file, linenum, args[0]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+
+ cur_arg = 2;
+ while (*args[cur_arg]) {
+ unsigned int low, high;
+
+ if (isdigit(*args[cur_arg])) {
+ char *dash = strchr(args[cur_arg], '-');
+
+ low = high = str2uic(args[cur_arg]);
+ if (dash)
+ high = str2uic(dash + 1);
+
+ if (high < low) {
+ unsigned int swap = low;
+ low = high;
+ high = swap;
+ }
+
+ if (low < 0 || high >= sizeof(long) * 8) {
+ Alert("parsing [%s:%d]: %s supports CPU numbers from 0 to %d.\n",
+ file, linenum, args[0], (int)(sizeof(long) * 8 - 1));
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+
+ while (low <= high)
+ cpus |= 1UL << low++;
+ }
+ else {
+ Alert("parsing [%s:%d]: %s : '%s' is not a CPU range.\n",
+ file, linenum, args[0], args[cur_arg]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+ cur_arg++;
+ }
+ for (i = 0; i < 32; i++)
+ if (proc & (1 << i))
+ global.cpu_map[i] = cpus;
+#else
+ Alert("parsing [%s:%d] : '%s' is not enabled, please check build options for USE_CPU_AFFINITY.\n", file, linenum, args[0]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+#endif
+ }
else {
struct cfg_kw_list *kwl;
int index;
diff --git a/src/haproxy.c b/src/haproxy.c
index c4122e2..20fb56d 100644
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -1,6 +1,6 @@
/*
* HA-Proxy : High Availability-enabled HTTP/TCP proxy
- * Copyright 2000-2011 Willy Tarreau <w@1wt.eu>.
+ * Copyright 2000-2012 Willy Tarreau <w@1wt.eu>.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
@@ -44,6 +44,11 @@
#include <sys/resource.h>
#include <time.h>
#include <syslog.h>
+#ifdef USE_CPU_AFFINITY
+#define __USE_GNU
+#include <sched.h>
+#undef __USE_GNU
+#endif
#ifdef DEBUG_FULL
#include <assert.h>
@@ -1467,6 +1472,13 @@
}
relative_pid++; /* each child will get a different one */
}
+
+#ifdef USE_CPU_AFFINITY
+ if (proc < global.nbproc && /* child */
+ proc < 32 && /* only the first 32 processes may be pinned */
+ global.cpu_map[proc]) /* only do this if the process has a CPU map */
+ sched_setaffinity(0, sizeof(unsigned long), (void *)&global.cpu_map[proc]);
+#endif
/* close the pidfile both in children and father */
if (pidfd >= 0) {
//lseek(pidfd, 0, SEEK_SET); /* debug: emulate eglibc bug */