From: Willy Tarreau <w@1wt.eu>
Date: Tue, 7 Sep 2021 08:49:45 +0000 (+0200)
Subject: MINOR: proxy: add a global "grace" directive to postpone soft-stop
X-Git-Tag: v2.5-dev7~35
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=10080716bfa76380a7b93c4d0ff4081301fada0c;p=thirdparty%2Fhaproxy.git

MINOR: proxy: add a global "grace" directive to postpone soft-stop

In ticket #1348 some users expressed some concerns regarding the removal
of the "grace" directive from the proxies. Their use case very closely
mimmicks the original intent of the grace keyword, which is, let haproxy
accept traffic for some time when stopping, while indicating an external
LB that it's stopping.

This is implemented here by starting a task whose expiration triggers
the soft-stop for real. The global "stopping" variable is immediately
set however. For example, this below will be sufficient to instantly
notify an external check on port 9999 that the service is going down,
while other services remain active for 10s:

    global
      grace 10s

    frontend ext-check
      bind :9999
      monitor-uri /ext-check
      monitor fail if { stopping }
---

diff --git a/doc/configuration.txt b/doc/configuration.txt
index 8981a6c77f..056bc1a385 100644
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -994,6 +994,7 @@ The following keywords are supported in the "global" section :
    - expose-experimental-directives
    - external-check
    - gid
+   - grace
    - group
    - hard-stop-after
    - h1-case-adjust
@@ -1309,6 +1310,52 @@ gid <number>
   will only be able to drop these groups if started with superuser privileges.
   See also "group" and "uid".
 
+grace <time>
+  Defines a delay between SIGUSR1 and real soft-stop.
+
+  Arguments :
+    <time>  is an extra delay (by default in milliseconds) after receipt of the
+            SIGUSR1 signal that will be waited for before proceeding with the
+            soft-stop operation.
+
+  This is used for compatibility with legacy environments where the haproxy
+  process needs to be stopped but some external components need to detect the
+  status before listeners are unbound. The principle is that the internal
+  "stopping" variable (which is reported by the "stopping" sample fetch
+  function) will be turned to true, but listeners will continue to accept
+  connections undisturbed, until the delay expires, after what the regular
+  soft-stop will proceed. This must not be used with processes that are
+  reloaded, or this will prevent the old process from unbinding, and may
+  prevent the new one from starting, or simply cause trouble.
+
+  Example:
+
+    global
+      grace 10s
+
+    # Returns 200 OK until stopping is set via SIGUSR1
+    frontend ext-check
+      bind :9999
+      monitor-uri /ext-check
+      monitor fail if { stopping }
+
+  Please note that a more flexible and durable approach would instead consist
+  for an orchestration system in setting a global variable from the CLI, use
+  that variable to respond to external checks, then after a delay send the
+  SIGUSR1 signal.
+
+  Example:
+
+    # Returns 200 OK until proc.stopping is set to non-zero. May be done
+    # from HTTP using set-var(proc.stopping) or from the CLI using:
+    # > set var proc.stopping int(1)
+    frontend ext-check
+      bind :9999
+      monitor-uri /ext-check
+      monitor fail if { var(proc.stopping) -m int gt 0 }
+
+  See also: hard-stop-after, monitor
+
 group <group name>
   Similar to "gid" but uses the GID of group name <group name> from /etc/group.
   See also "gid" and "user".
@@ -1329,6 +1376,8 @@ hard-stop-after <time>
     global
       hard-stop-after 30s
 
+  See also: grace
+
 h1-case-adjust <from> <to>
   Defines the case adjustment to apply, when enabled, to the header name
   <from>, to change it to <to> before sending it to HTTP/1 clients or
diff --git a/include/haproxy/global-t.h b/include/haproxy/global-t.h
index 9c892a7df5..d7c9481323 100644
--- a/include/haproxy/global-t.h
+++ b/include/haproxy/global-t.h
@@ -95,6 +95,7 @@ struct global {
 	int nbthread;
 	int mode;
 	unsigned int hard_stop_after;	/* maximum time allowed to perform a soft-stop */
+	unsigned int grace_delay;       /* grace delay between SIGUSR1 and soft-stop */
 	int maxconn, hardmaxconn;
 	int maxsslconn;
 	int ssl_session_max_cost;   /* how many bytes an SSL session may cost */
diff --git a/src/proxy.c b/src/proxy.c
index e67ba39581..ad5120acf1 100644
--- a/src/proxy.c
+++ b/src/proxy.c
@@ -1928,6 +1928,34 @@ struct task *manage_proxy(struct task *t, void *context, unsigned int state)
 }
 
 
+static int proxy_parse_grace(char **args, int section_type, struct proxy *curpx,
+                             const struct proxy *defpx, const char *file, int line,
+                             char **err)
+{
+	const char *res;
+
+	if (!*args[1]) {
+		memprintf(err, "'%s' expects <time> as argument.\n", args[0]);
+		return -1;
+	}
+	res = parse_time_err(args[1], &global.grace_delay, TIME_UNIT_MS);
+	if (res == PARSE_TIME_OVER) {
+		memprintf(err, "timer overflow in argument '%s' to '%s' (maximum value is 2147483647 ms or ~24.8 days)",
+			  args[1], args[0]);
+		return -1;
+	}
+	else if (res == PARSE_TIME_UNDER) {
+		memprintf(err, "timer underflow in argument '%s' to '%s' (minimum non-null value is 1 ms)",
+			  args[1], args[0]);
+		return -1;
+	}
+	else if (res) {
+		memprintf(err, "unexpected character '%c' in argument to <%s>.\n", *res, args[0]);
+		return -1;
+	}
+	return 0;
+}
+
 static int proxy_parse_hard_stop_after(char **args, int section_type, struct proxy *curpx,
                                 const struct proxy *defpx, const char *file, int line,
                                 char **err)
@@ -2001,17 +2029,15 @@ struct task *hard_stop(struct task *t, void *context, unsigned int state)
 	return t;
 }
 
-/*
- * this function disables health-check servers so that the process will quickly be ignored
- * by load balancers.
- */
-void soft_stop(void)
+/* perform the soft-stop right now (i.e. unbind listeners) */
+static void do_soft_stop_now()
 {
 	struct task *task;
 
-	stopping = 1;
 	/* disable busy polling to avoid cpu eating for the new process */
 	global.tune.options &= ~GTUNE_BUSY_POLLING;
+
+	/* schedule a hard-stop after a delay if needed */
 	if (tick_isset(global.hard_stop_after)) {
 		task = task_new(MAX_THREADS_MASK);
 		if (task) {
@@ -2030,6 +2056,44 @@ void soft_stop(void)
 	signal_handler(0);
 }
 
+/* triggered by a soft-stop delayed with `grace` */
+static struct task *grace_expired(struct task *t, void *context, unsigned int state)
+{
+	ha_notice("Grace period expired, proceeding with soft-stop now.\n");
+	send_log(NULL, LOG_NOTICE, "Grace period expired, proceeding with soft-stop now.\n");
+	do_soft_stop_now();
+	task_destroy(t);
+	return NULL;
+}
+
+/*
+ * this function disables health-check servers so that the process will quickly be ignored
+ * by load balancers.
+ */
+void soft_stop(void)
+{
+	struct task *task;
+
+	stopping = 1;
+
+	if (tick_isset(global.grace_delay)) {
+		task = task_new(MAX_THREADS_MASK);
+		if (task) {
+			ha_notice("Scheduling a soft-stop in %u ms.\n", global.grace_delay);
+			send_log(NULL, LOG_WARNING, "Scheduling a soft-stop in %u ms.\n", global.grace_delay);
+			task->process = grace_expired;
+			task_schedule(task, tick_add(now_ms, global.grace_delay));
+			return;
+		}
+		else {
+			ha_alert("out of memory trying to allocate the stop-stop task, stopping now.\n");
+		}
+	}
+
+	/* no grace (or failure to enforce it): stop now */
+	do_soft_stop_now();
+}
+
 
 /* Temporarily disables listening on all of the proxy's listeners. Upon
  * success, the proxy enters the PR_PAUSED state. The function returns 0
@@ -2373,6 +2437,7 @@ void proxy_adjust_all_maxconn()
 /* Config keywords below */
 
 static struct cfg_kw_list cfg_kws = {ILH, {
+	{ CFG_GLOBAL, "grace", proxy_parse_grace },
 	{ CFG_GLOBAL, "hard-stop-after", proxy_parse_hard_stop_after },
 	{ CFG_LISTEN, "timeout", proxy_parse_timeout },
 	{ CFG_LISTEN, "clitimeout", proxy_parse_timeout }, /* This keyword actually fails to parse, this line remains for better error messages. */