master: When process_limit fills up, wait 10s before closing pending connections.

It might have only been a temporary burst that gets resolved quickly enough.

diff --git a/src/master/service-monitor.c b/src/master/service-monitor.c
index c22bdbb75d5bf5a7dcf47dd519b61a19e9af882e..3fbad236cb6488f38b2e2a979282bc7a28a189ec 100644 (file)
--- a/src/master/service-monitor.c
+++ b/src/master/service-monitor.c
@@ -22,10 +22,12 @@
 
 #define SERVICE_STARTUP_FAILURE_THROTTLE_SECS 60
 #define SERVICE_DROP_WARN_INTERVAL_SECS 60
+#define SERVICE_DROP_TIMEOUT_MSECS (10*1000)
 
 static void service_monitor_start_extra_avail(struct service *service);
 static void service_status_more(struct service_process *process,
                                const struct master_status *status);
+static void service_monitor_listen_start_force(struct service *service);
 
 static void service_process_kill_idle(struct service_process *process)
 {
@@ -195,6 +197,32 @@ static void service_monitor_throttle(struct service *service)
        service_throttle(service, SERVICE_STARTUP_FAILURE_THROTTLE_SECS);
 }
 
+static void service_drop_timeout(struct service *service)
+{
+       struct service_listener *const *lp;
+       int fd;
+
+       i_assert(service->process_avail == 0);
+
+       /* drop all pending connections */
+       array_foreach(&service->listeners, lp) {
+               while ((fd = net_accept((*lp)->fd, NULL, NULL)) > 0)
+                       net_disconnect(fd);
+       }
+
+       service_monitor_listen_start_force(service);
+       service->listen_pending = TRUE;
+}
+
+static void service_monitor_listen_pending(struct service *service)
+{
+       service_monitor_listen_stop(service);
+       service->listen_pending = TRUE;
+
+       service->to_drop = timeout_add(SERVICE_DROP_TIMEOUT_MSECS,
+                                      service_drop_timeout, service);
+}
+
 static void service_drop_connections(struct service_listener *l)
 {
        struct service *service = l->service;
@@ -216,12 +244,16 @@ static void service_drop_connections(struct service_listener *l)
                   reach connection limit */
                service_login_notify(service, TRUE);
 
-               service_monitor_listen_stop(service);
-               service->listen_pending = TRUE;
+               service_monitor_listen_pending(service);
+       } else if (!service->listen_pending) {
+               /* maybe this is a temporary peak, stop for a while and
+                  see if it goes away */
+               service_monitor_listen_pending(service);
        } else {
-               /* just accept and close the connection, so it's clear that
-                  this is happening because of the limit, rather than because
-                  the service processes aren't answering fast enough */
+               /* this has been happening for a while now. just accept and
+                  close the connection, so it's clear that this is happening
+                  because of the limit, rather than because the service
+                  processes aren't answering fast enough */
                fd = net_accept(l->fd, NULL, NULL);
                if (fd > 0)
                        net_disconnect(fd);
@@ -271,17 +303,14 @@ static void service_monitor_start_extra_avail(struct service *service)
        }
 }
 
-void service_monitor_listen_start(struct service *service)
+static void service_monitor_listen_start_force(struct service *service)
 {
        struct service_listener *const *listeners;
 
-       if (service->process_avail > 0 ||
-           (service->process_count == service->process_limit &&
-            service->listen_pending))
-               return;
-
        service->listening = TRUE;
        service->listen_pending = FALSE;
+       if (service->to_drop != NULL)
+               timeout_remove(&service->to_drop);
 
        array_foreach(&service->listeners, listeners) {
                struct service_listener *l = *listeners;
@@ -291,6 +320,16 @@ void service_monitor_listen_start(struct service *service)
        }
 }
 
+void service_monitor_listen_start(struct service *service)
+{
+       if (service->process_avail > 0 ||
+           (service->process_count == service->process_limit &&
+            service->listen_pending))
+               return;
+
+       service_monitor_listen_start_force(service);
+}
+
 void service_monitor_listen_stop(struct service *service)
 {
        struct service_listener *const *listeners;
@@ -303,6 +342,8 @@ void service_monitor_listen_stop(struct service *service)
        }
        service->listening = FALSE;
        service->listen_pending = FALSE;
+       if (service->to_drop != NULL)
+               timeout_remove(&service->to_drop);
 }
 
 static int service_login_create_notify_fd(struct service *service)

diff --git a/src/master/service.h b/src/master/service.h
index e652da0aaa785bd5b430a011773071bc36940df4..14ebc07618ae21d22144ac89a77557b0119f5b13 100644 (file)
--- a/src/master/service.h
+++ b/src/master/service.h
@@ -88,6 +88,9 @@ struct service {
        /* if a process fails before servicing its first request, assume it's
           broken and start throtting new process creations */
        struct timeout *to_throttle;
+       /* when process_limit is reached, wait for a while until we actually
+          start dropping pending connections */
+       struct timeout *to_drop;
 
        /* Last time a "dropping client connections" warning was logged */
        time_t last_drop_warning;