login proxy: Added delayed disconnection of clients on server mass-disconnect.

login_proxy_max_disconnect_delay setting (default 0 = disabled) controls for
how long time period the disconnections are spread to. The idea behind this
is to avoid load spikes due to client reconnections when a backend server
dies or is restarted.

diff --git a/src/login-common/login-proxy-state.h b/src/login-common/login-proxy-state.h
index 799b00c669da0d67610a33bc67ae39e029a71e1f..405cd0d71d565c7a370b05f1efa2b887c0e59559 100644 (file)
--- a/src/login-common/login-proxy-state.h
+++ b/src/login-common/login-proxy-state.h
@@ -7,8 +7,22 @@ struct login_proxy_record {
        struct ip_addr ip;
        in_port_t port;
 
+       /* These are used to spread client-visible disconnects over longer
+          periods of time to avoid reconnect spikes when a server dies.
+
+          If num_disconnects_since_ts=0 when server disconnects us, it's
+          increased and disconnect_timestamp is updated. Afterwards it's
+          increased for each new disconnection. num_disconnects_since_ts gets
+          reset back to zero whenever a) last_success gets updated or b)
+          num_delayed_client_disconnects drops to 0. */
+       struct timeval disconnect_timestamp;
+       unsigned int num_disconnects_since_ts;
+       unsigned int num_delayed_client_disconnects;
+
        /* these are tracking connect()s, not necessarily logins: */
        unsigned int num_waiting_connections;
+       /* number of connections we're proxying now (post-login) */
+       unsigned int num_proxying_connections;
        struct timeval last_failure;
        struct timeval last_success;
 };

diff --git a/src/login-common/login-proxy.c b/src/login-common/login-proxy.c
index 03147c6f5ba744c807c094b62722224a359b24ba..2f55f738e55ef56ec46bc77ec7ad619817aac961 100644 (file)
--- a/src/login-common/login-proxy.c
+++ b/src/login-common/login-proxy.c
@@ -16,6 +16,8 @@
 #include "login-proxy-state.h"
 #include "login-proxy.h"
 
+#include <stdlib.h>
+
 #define MAX_PROXY_INPUT_SIZE 4096
 #define OUTBUF_THRESHOLD 1024
 #define LOGIN_PROXY_DIE_IDLE_SECS 2
@@ -24,6 +26,7 @@
 #define KILLED_BY_ADMIN_REASON "Killed by admin"
 #define PROXY_IMMEDIATE_FAILURE_SECS 30
 #define PROXY_CONNECT_RETRY_MSECS 1000
+#define PROXY_DISCONNECT_INTERVAL_MSECS 100
 
 struct login_proxy {
        struct login_proxy *prev, *next;
@@ -53,21 +56,27 @@ struct login_proxy {
        unsigned int connected:1;
        unsigned int destroying:1;
        unsigned int disconnecting:1;
+       unsigned int delayed_disconnect:1;
        unsigned int num_waiting_connections_updated:1;
 };
 
 static struct login_proxy_state *proxy_state;
 static struct login_proxy *login_proxies = NULL;
 static struct login_proxy *login_proxies_pending = NULL;
+static struct login_proxy *login_proxies_disconnecting = NULL;
 static struct ipc_server *login_proxy_ipc_server;
 
 static int login_proxy_connect(struct login_proxy *proxy);
 static void login_proxy_disconnect(struct login_proxy *proxy);
 static void login_proxy_ipc_cmd(struct ipc_cmd *cmd, const char *line);
+static void login_proxy_free_final(struct login_proxy *proxy);
 
 static void
 login_proxy_free_reason(struct login_proxy **_proxy, const char *reason)
        ATTR_NULL(2);
+static void
+login_proxy_free_delayed(struct login_proxy **_proxy, const char *reason)
+       ATTR_NULL(2);
 
 static void login_proxy_free_errno(struct login_proxy **proxy,
                                   int err, bool server)
@@ -77,7 +86,10 @@ static void login_proxy_free_errno(struct login_proxy **proxy,
        reason = err == 0 || err == EPIPE ?
                t_strdup_printf("Disconnected by %s", who) :
                t_strdup_printf("Disconnected by %s: %s", who, strerror(errno));
-       login_proxy_free_reason(proxy, reason);
+       if (server)
+               login_proxy_free_delayed(proxy, reason);
+       else
+               login_proxy_free_reason(proxy, reason);
 }
 
 static void server_input(struct login_proxy *proxy)
@@ -136,6 +148,16 @@ static void proxy_client_input(struct login_proxy *proxy)
        }
 }
 
+static void proxy_client_disconnected_input(struct login_proxy *proxy)
+{
+       unsigned char buf[OUTBUF_THRESHOLD];
+
+       /* we're already disconnected from server. either wait for
+          disconnection timeout or for client to disconnect itself. */
+       if (net_receive(proxy->client_fd, buf, sizeof(buf)) < 0)
+               login_proxy_free_final(proxy);
+}
+
 static int server_output(struct login_proxy *proxy)
 {
        proxy->last_io = ioloop_time;
@@ -283,6 +305,8 @@ static void proxy_wait_connect(struct login_proxy *proxy)
        proxy->state_rec->last_success = ioloop_timeval;
        i_assert(proxy->state_rec->num_waiting_connections > 0);
        proxy->state_rec->num_waiting_connections--;
+       proxy->state_rec->num_proxying_connections++;
+       proxy->state_rec->num_disconnects_since_ts = 0;
 
        if ((proxy->ssl_flags & PROXY_SSL_FLAG_YES) != 0 &&
            (proxy->ssl_flags & PROXY_SSL_FLAG_STARTTLS) == 0) {
@@ -408,6 +432,10 @@ static void login_proxy_disconnect(struct login_proxy *proxy)
                i_assert(proxy->state_rec->num_waiting_connections > 0);
                proxy->state_rec->num_waiting_connections--;
        }
+       if (proxy->connected) {
+               i_assert(proxy->state_rec->num_proxying_connections > 0);
+               proxy->state_rec->num_proxying_connections--;
+       }
 
        if (proxy->server_io != NULL)
                io_remove(&proxy->server_io);
@@ -421,6 +449,17 @@ static void login_proxy_disconnect(struct login_proxy *proxy)
 
 static void login_proxy_free_final(struct login_proxy *proxy)
 {
+       if (proxy->delayed_disconnect) {
+               DLLIST_REMOVE(&login_proxies_disconnecting, proxy);
+
+               i_assert(proxy->state_rec->num_delayed_client_disconnects > 0);
+               if (--proxy->state_rec->num_delayed_client_disconnects == 0)
+                       proxy->state_rec->num_disconnects_since_ts = 0;
+               timeout_remove(&proxy->to);
+       }
+
+       if (proxy->client_io != NULL)
+               io_remove(&proxy->client_io);
        if (proxy->client_output != NULL)
                o_stream_destroy(&proxy->client_output);
        if (proxy->client_fd != -1)
@@ -431,12 +470,69 @@ static void login_proxy_free_final(struct login_proxy *proxy)
        i_free(proxy);
 }
 
+static unsigned int login_proxy_delay_disconnect(struct login_proxy *proxy)
+{
+       struct login_proxy_record *rec = proxy->state_rec;
+       const unsigned int max_delay =
+               proxy->client->set->login_proxy_max_disconnect_delay;
+       struct timeval disconnect_time_offset;
+       unsigned int max_disconnects_per_sec, delay_msecs_since_ts, max_conns;
+       int delay_msecs;
+
+       if (rec->num_disconnects_since_ts == 0) {
+               rec->disconnect_timestamp = ioloop_timeval;
+               /* start from a slightly random timestamp. this way all proxy
+                  processes will disconnect at slightly different times to
+                  spread the load. */
+               timeval_add_msecs(&rec->disconnect_timestamp,
+                                 rand() % PROXY_DISCONNECT_INTERVAL_MSECS);
+       }
+       rec->num_disconnects_since_ts++;
+       if (proxy->to != NULL) {
+               /* we were already lazily disconnecting this */
+               return 0;
+       }
+       if (max_delay == 0) {
+               /* delaying is disabled */
+               return 0;
+       }
+       max_conns = rec->num_proxying_connections + rec->num_disconnects_since_ts;
+       max_disconnects_per_sec = (max_conns + max_delay-1) / max_delay;
+       if (rec->num_disconnects_since_ts <= max_disconnects_per_sec &&
+           rec->num_delayed_client_disconnects == 0) {
+               /* wait delaying until we have 1 second's worth of clients
+                  disconnected */
+               return 0;
+       }
+
+       /* see at which time we should be disconnecting the client.
+          do it in 100ms intervals so the timeouts are triggered together. */
+       disconnect_time_offset = rec->disconnect_timestamp;
+       delay_msecs_since_ts = PROXY_DISCONNECT_INTERVAL_MSECS *
+               (max_delay * rec->num_disconnects_since_ts *
+                (1000/PROXY_DISCONNECT_INTERVAL_MSECS) / max_conns);
+       timeval_add_msecs(&disconnect_time_offset, delay_msecs_since_ts);
+       delay_msecs = timeval_diff_msecs(&disconnect_time_offset, &ioloop_timeval);
+       if (delay_msecs <= 0) {
+               /* we already reached the time */
+               return 0;
+       }
+
+       rec->num_delayed_client_disconnects++;
+       proxy->delayed_disconnect = TRUE;
+       proxy->to = timeout_add(delay_msecs, login_proxy_free_final, proxy);
+       DLLIST_PREPEND(&login_proxies_disconnecting, proxy);
+       return delay_msecs;
+}
+
 static void ATTR_NULL(2)
-login_proxy_free_reason(struct login_proxy **_proxy, const char *reason)
+login_proxy_free_full(struct login_proxy **_proxy, const char *reason,
+                     bool delayed)
 {
        struct login_proxy *proxy = *_proxy;
        struct client *client = proxy->client;
        const char *ipstr;
+       unsigned int delay_ms = 0;
 
        *_proxy = NULL;
 
@@ -444,18 +540,23 @@ login_proxy_free_reason(struct login_proxy **_proxy, const char *reason)
                return;
        proxy->destroying = TRUE;
 
+       /* we'll disconnect server side in any case. */
        login_proxy_disconnect(proxy);
 
        if (proxy->client_fd != -1) {
                /* detached proxy */
                DLLIST_REMOVE(&login_proxies, proxy);
 
+               if (delayed)
+                       delay_ms = login_proxy_delay_disconnect(proxy);
+
                ipstr = net_ip2addr(&proxy->client->ip);
                client_log(proxy->client, t_strdup_printf(
-                       "proxy(%s): disconnecting %s%s",
+                       "proxy(%s): disconnecting %s%s%s",
                        proxy->client->virtual_user,
                        ipstr != NULL ? ipstr : "",
-                       reason == NULL ? "" : t_strdup_printf(" (%s)", reason)));
+                       reason == NULL ? "" : t_strdup_printf(" (%s)", reason),
+                       delay_ms == 0 ? "" : t_strdup_printf(" - disconnecting client in %ums", delay_ms)));
 
                if (proxy->client_io != NULL)
                        io_remove(&proxy->client_io);
@@ -469,12 +570,29 @@ login_proxy_free_reason(struct login_proxy **_proxy, const char *reason)
                if (proxy->callback != NULL)
                        proxy->callback(proxy->client);
        }
-       login_proxy_free_final(proxy);
+       if (delay_ms == 0)
+               login_proxy_free_final(proxy);
+       else {
+               proxy->client_io = io_add(proxy->client_fd, IO_READ,
+                       proxy_client_disconnected_input, proxy);
+       }
 
        client->login_proxy = NULL;
        client_unref(&client);
 }
 
+static void ATTR_NULL(2)
+login_proxy_free_reason(struct login_proxy **_proxy, const char *reason)
+{
+       login_proxy_free_full(_proxy, reason, FALSE);
+}
+
+static void ATTR_NULL(2)
+login_proxy_free_delayed(struct login_proxy **_proxy, const char *reason)
+{
+       login_proxy_free_full(_proxy, reason, TRUE);
+}
+
 void login_proxy_free(struct login_proxy **_proxy)
 {
        login_proxy_free_reason(_proxy, NULL);
@@ -680,7 +798,7 @@ login_proxy_cmd_kick(struct ipc_cmd *cmd, const char *const *args)
                next = proxy->next;
 
                if (strcmp(proxy->client->virtual_user, args[0]) == 0) {
-                       login_proxy_free_reason(&proxy, KILLED_BY_ADMIN_REASON);
+                       login_proxy_free_delayed(&proxy, KILLED_BY_ADMIN_REASON);
                        count++;
                }
        }
@@ -726,7 +844,7 @@ login_proxy_cmd_kick_director_hash(struct ipc_cmd *cmd, const char *const *args)
 
                if (director_username_hash(proxy->client) == hash &&
                    !net_ip_compare(&proxy->ip, &except_ip)) {
-                       login_proxy_free_reason(&proxy, KILLED_BY_ADMIN_REASON);
+                       login_proxy_free_delayed(&proxy, KILLED_BY_ADMIN_REASON);
                        count++;
                }
        }
@@ -799,6 +917,8 @@ void login_proxy_deinit(void)
                proxy = login_proxies;
                login_proxy_free_reason(&proxy, KILLED_BY_ADMIN_REASON);
        }
+       while (login_proxies_disconnecting != NULL)
+               login_proxy_free_final(login_proxies_disconnecting);
        if (login_proxy_ipc_server != NULL)
                ipc_server_deinit(&login_proxy_ipc_server);
        login_proxy_state_deinit(&proxy_state);

diff --git a/src/login-common/login-settings.c b/src/login-common/login-settings.c
index 24b2deedd8d65cd1d9ae1172a71ce8107bd78643..96eee102fab68f2c6f6583256eda0ff1346a4d04 100644 (file)
--- a/src/login-common/login-settings.c
+++ b/src/login-common/login-settings.c
@@ -26,6 +26,7 @@ static const struct setting_define login_setting_defines[] = {
        DEF(SET_STR, login_log_format_elements),
        DEF(SET_STR, login_log_format),
        DEF(SET_STR, login_access_sockets),
+       DEF(SET_TIME, login_proxy_max_disconnect_delay),
        DEF(SET_STR, director_username_hash),
 
        DEF(SET_STR, ssl_client_cert),
@@ -51,6 +52,7 @@ static const struct login_settings login_default_settings = {
        .login_log_format_elements = "user=<%u> method=%m rip=%r lip=%l mpid=%e %c session=<%{session}>",
        .login_log_format = "%$: %s",
        .login_access_sockets = "",
+       .login_proxy_max_disconnect_delay = 0,
        .director_username_hash = "%u",
 
        .ssl_client_cert = "",

diff --git a/src/login-common/login-settings.h b/src/login-common/login-settings.h
index 7a5c60c759e13d7f25c35d5cb2887a9f31b3d79f..cc8e9280977537d421d84827ddd97d76365c3152 100644 (file)
--- a/src/login-common/login-settings.h
+++ b/src/login-common/login-settings.h
@@ -9,6 +9,7 @@ struct login_settings {
        const char *login_greeting;
        const char *login_log_format_elements, *login_log_format;
        const char *login_access_sockets;
+       unsigned int login_proxy_max_disconnect_delay;
        const char *director_username_hash;
 
        const char *ssl_client_cert;