master: Added reuse_port setting to inet_listeners, which enables SO_REUSEPORT if...

After forking a new service process, a new listener socket is created for
each such inet_listener. Linux v3.9+ added SO_REUSEPORT feature, which
should distribute clients more uniformly to the processes. I'm not sure if
this makes any difference in BSDs.

At least in Linux v3.9 there was still a bug that if the number of listening
processes changed, some TCP handshakes might not finish. I don't see if this
has already been fixed, so this is probably safe to use only for services
whose process count doesn't change (e.g. process_min_avail is set high
enough).

diff --git a/src/imap-login/imap-login-settings.c b/src/imap-login/imap-login-settings.c
index fe3c7122e26d494c5c871c240e1a5b9285f20867..0b1ffc2f3d579efe0288b3d97ed639ce26c33649 100644 (file)
--- a/src/imap-login/imap-login-settings.c
+++ b/src/imap-login/imap-login-settings.c
@@ -11,8 +11,8 @@
 
 /* <settings checks> */
 static struct inet_listener_settings imap_login_inet_listeners_array[] = {
-       { "imap", "", 143, FALSE },
-       { "imaps", "", 993, TRUE }
+       { .name = "imap", .address = "", .port = 143 },
+       { .name = "imaps", .address = "", .port = 993, .ssl = TRUE }
 };
 static struct inet_listener_settings *imap_login_inet_listeners[] = {
        &imap_login_inet_listeners_array[0],

diff --git a/src/lib-master/service-settings.h b/src/lib-master/service-settings.h
index 2dcdc7dbb664efa8e3da2356c3e79edc5b7a08b7..b97aacceb8c03cc03c02bbfef2f3c80891d7ba90 100644 (file)
--- a/src/lib-master/service-settings.h
+++ b/src/lib-master/service-settings.h
@@ -31,6 +31,7 @@ struct inet_listener_settings {
        const char *address;
        unsigned int port;
        bool ssl;
+       bool reuse_port;
 };
 ARRAY_DEFINE_TYPE(inet_listener_settings, struct inet_listener_settings *);
 

diff --git a/src/master/master-settings.c b/src/master/master-settings.c
index 45e56ac126aa82b52a69e3366062f56473db0c1d..2ea707abee445be2f91ef656d7cb2d4f3fd34378 100644 (file)
--- a/src/master/master-settings.c
+++ b/src/master/master-settings.c
@@ -64,6 +64,7 @@ static const struct setting_define inet_listener_setting_defines[] = {
        DEF(SET_STR, address),
        DEF(SET_UINT, port),
        DEF(SET_BOOL, ssl),
+       DEF(SET_BOOL, reuse_port),
 
        SETTING_DEFINE_LIST_END
 };
@@ -72,7 +73,8 @@ static const struct inet_listener_settings inet_listener_default_settings = {
        .name = "",
        .address = "",
        .port = 0,
-       .ssl = FALSE
+       .ssl = FALSE,
+       .reuse_port = FALSE
 };
 
 static const struct setting_parser_info inet_listener_setting_parser_info = {

diff --git a/src/master/service-listen.c b/src/master/service-listen.c
index 4f149c81bf905baee67a2d4d85ee17701cbfc7e4..37cdc917c85673eb0803d379b478c589f03837c2 100644 (file)
--- a/src/master/service-listen.c
+++ b/src/master/service-listen.c
@@ -189,9 +189,11 @@ systemd_listen_fd(const struct ip_addr *ip, unsigned int port, int *fd_r)
 static int service_inet_listener_listen(struct service_listener *l)
 {
         struct service *service = l->service;
+       enum net_listen_flags flags = 0;
        const struct inet_listener_settings *set = l->set.inetset.set;
        unsigned int port = set->port;
        int fd;
+
 #ifdef HAVE_SYSTEMD
        if (systemd_listen_fd(&l->set.inetset.ip, port, &fd) < 0)
                return -1;
@@ -199,13 +201,16 @@ static int service_inet_listener_listen(struct service_listener *l)
        if (fd == -1)
 #endif
        {
-               fd = net_listen(&l->set.inetset.ip, &port,
-                               service_get_backlog(service));
+               if (set->reuse_port)
+                       flags |= NET_LISTEN_FLAG_REUSEPORT;
+               fd = net_listen_full(&l->set.inetset.ip, &port, &flags,
+                                    service_get_backlog(service));
                if (fd < 0) {
                        service_error(service, "listen(%s, %u) failed: %m",
                                      l->inet_address, set->port);
                        return errno == EADDRINUSE ? 0 : -1;
                }
+               l->reuse_port = (flags & NET_LISTEN_FLAG_REUSEPORT) != 0;
        }
        net_set_nonblock(fd, TRUE);
        fd_close_on_exec(fd, TRUE);
@@ -214,6 +219,19 @@ static int service_inet_listener_listen(struct service_listener *l)
        return 1;
 }
 
+int service_listener_listen(struct service_listener *l)
+{
+       switch (l->type) {
+       case SERVICE_LISTENER_UNIX:
+               return service_unix_listener_listen(l);
+       case SERVICE_LISTENER_FIFO:
+               return service_fifo_listener_listen(l);
+       case SERVICE_LISTENER_INET:
+               return service_inet_listener_listen(l);
+       }
+       i_unreached();
+}
+
 static int service_listen(struct service *service)
 {
        struct service_listener *const *listeners;
@@ -225,18 +243,7 @@ static int service_listen(struct service *service)
                if (l->fd != -1)
                        continue;
 
-               switch (l->type) {
-               case SERVICE_LISTENER_UNIX:
-                       ret2 = service_unix_listener_listen(l);
-                       break;
-               case SERVICE_LISTENER_FIFO:
-                       ret2 = service_fifo_listener_listen(l);
-                       break;
-               case SERVICE_LISTENER_INET:
-                       ret2 = service_inet_listener_listen(l);
-                       break;
-               }
-
+               ret2 = service_listener_listen(l);
                if (ret2 < ret)
                        ret = ret2;
        }

diff --git a/src/master/service-listen.h b/src/master/service-listen.h
index dc792faa8b8b131d68d26e4efe1e883438c9685d..ffd88bd5fa031c280d7fe94b38d409c3c11063a5 100644 (file)
--- a/src/master/service-listen.h
+++ b/src/master/service-listen.h
@@ -13,4 +13,6 @@ int services_listen(struct service_list *service_list);
 int services_listen_using(struct service_list *new_service_list,
                          struct service_list *old_service_list);
 
+int service_listener_listen(struct service_listener *l);
+
 #endif

diff --git a/src/master/service-process.c b/src/master/service-process.c
index e737d4ed966a2f8d2a75ef0784b837c17fbfc3cb..6d2d22246e47afa58183e2dfa723a65483e63d92 100644 (file)
--- a/src/master/service-process.c
+++ b/src/master/service-process.c
@@ -23,6 +23,7 @@
 #include "dup2-array.h"
 #include "service.h"
 #include "service-anvil.h"
+#include "service-listen.h"
 #include "service-log.h"
 #include "service-process-notify.h"
 #include "service-process.h"
@@ -34,6 +35,24 @@
 #include <signal.h>
 #include <sys/wait.h>
 
+static void service_reopen_inet_listeners(struct service *service)
+{
+       struct service_listener *const *listeners;
+       unsigned int i, count;
+       int old_fd;
+
+       listeners = array_get(&service->listeners, &count);
+       for (i = 0; i < count; i++) {
+               if (!listeners[i]->reuse_port || listeners[i]->fd == -1)
+                       continue;
+
+               old_fd = listeners[i]->fd;
+               listeners[i]->fd = -1;
+               if (service_listener_listen(listeners[i]) < 0)
+                       listeners[i]->fd = old_fd;
+       }
+}
+
 static void
 service_dup_fds(struct service *service)
 {
@@ -305,6 +324,7 @@ struct service_process *service_process_create(struct service *service)
        if (pid == 0) {
                /* child */
                service_process_setup_environment(service, uid);
+               service_reopen_inet_listeners(service);
                service_dup_fds(service);
                drop_privileges(service);
                process_exec(service->executable, NULL);

diff --git a/src/master/service.h b/src/master/service.h
index 3ed10b4e3e225d39f503d5d76e697680d49285bf..79aa65f9ef03b189442484f1772983093ab245f5 100644 (file)
--- a/src/master/service.h
+++ b/src/master/service.h
@@ -38,6 +38,8 @@ struct service_listener {
                        struct ip_addr ip;
                } inetset;
        } set;
+
+       bool reuse_port;
 };
 
 struct service {

diff --git a/src/pop3-login/pop3-login-settings.c b/src/pop3-login/pop3-login-settings.c
index f69f2ec1ccab523e5b57bca0d22465a0f2e2c469..584e508559ddfb3d4ccb63074ec7d18f9300dc72 100644 (file)
--- a/src/pop3-login/pop3-login-settings.c
+++ b/src/pop3-login/pop3-login-settings.c
@@ -11,8 +11,8 @@
 
 /* <settings checks> */
 static struct inet_listener_settings pop3_login_inet_listeners_array[] = {
-       { "pop3", "", 110, FALSE },
-       { "pop3s", "", 995, TRUE }
+       { .name = "pop3", .address = "", .port = 110 },
+       { .name = "pop3s", .address = "", .port = 995, .ssl = TRUE }
 };
 static struct inet_listener_settings *pop3_login_inet_listeners[] = {
        &pop3_login_inet_listeners_array[0],