MINOR: proxy: Don't close FDs if not our proxy.

When running with multiple process, if some proxies are just assigned
to some processes, the other processes will just close the file descriptors
for the listening sockets. However, we may still have to provide those
sockets when reloading, so instead we just try hard to pretend those proxies
are dead, while keeping the sockets opened.
A new global option, no-reused-socket", has been added, to restore the old
behavior of closing the sockets not bound to this process.

diff --git a/doc/configuration.txt b/doc/configuration.txt
index 05f07019f989d484b7c8388343fce2ffe69ac386..e6ea2cfc8945e2b17b1e4cf27a115cf274140f85 100644 (file)
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -587,6 +587,7 @@ The following keywords are supported in the "global" section :
    - nosplice
    - nogetaddrinfo
    - noreuseport
+   - no-unused-socket
    - spread-checks
    - server-state-base
    - server-state-file
@@ -1250,6 +1251,12 @@ noreuseport
   Disables the use of SO_REUSEPORT - see socket(7). It is equivalent to the
   command line argument "-dR".
 
+no-unused-socket
+  By default, each haproxy process keeps all sockets opened, event those that
+  are only used by another processes, so that any process can provide all the
+  sockets, to make reloads seamless. This option disables this, and close all
+  unused sockets, to save some file descriptors.
+
 spread-checks <0..50, in percent>
   Sometimes it is desirable to avoid sending agent and health checks to
   servers at exact intervals, for instance when many logical servers are

diff --git a/include/proto/fd.h b/include/proto/fd.h
index 87309bf0a38a24790c9a3237bf2a21905f4cd9eb..1efe32399aa6bcc65ff7fe28e4ce968f76efe783 100644 (file)
--- a/include/proto/fd.h
+++ b/include/proto/fd.h
@@ -41,6 +41,11 @@ extern int fd_nbupdt;               // number of updates in the list
  */
 void fd_delete(int fd);
 
+/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
+ * The file descriptor is kept open.
+ */
+void fd_remove(int fd);
+
 /* disable the specified poller */
 void disable_poller(const char *poller_name);
 

diff --git a/include/proto/listener.h b/include/proto/listener.h
index 552d695954cd427e50ea2c820cae1c040d55162f..079d976a30716e6317d83fad0c1de32c95b49aab 100644 (file)
--- a/include/proto/listener.h
+++ b/include/proto/listener.h
@@ -88,6 +88,11 @@ void dequeue_all_listeners(struct list *list);
  */
 int unbind_listener(struct listener *listener);
 
+/* This function pretends the listener is dead, but keeps the FD opened, so
+ * that we can provide it, for conf reloading.
+ */
+int unbind_listener_no_close(struct listener *listener);
+
 /* This function closes all listening sockets bound to the protocol <proto>,
  * and the listeners end in LI_ASSIGNED state if they were higher. It does not
  * detach them from the protocol. It always returns ERR_NONE.

diff --git a/include/proto/proxy.h b/include/proto/proxy.h
index 72f1e1d33258c7376584dd25a451593f0e46c167..e5d20c4f5d1eeacc125ef0491b87692c8d5cfefb 100644 (file)
--- a/include/proto/proxy.h
+++ b/include/proto/proxy.h
@@ -42,6 +42,7 @@ void soft_stop(void);
 int pause_proxy(struct proxy *p);
 int resume_proxy(struct proxy *p);
 void stop_proxy(struct proxy *p);
+void zombify_proxy(struct proxy *p);
 void pause_proxies(void);
 void resume_proxies(void);
 int  stream_set_backend(struct stream *s, struct proxy *be);

diff --git a/include/types/global.h b/include/types/global.h
index df8e2c69aa97133a928e4ba54334b390010dc745..57b969dd14675ba9e8ee6e59bcf9fe28d712f8ca 100644 (file)
--- a/include/types/global.h
+++ b/include/types/global.h
@@ -62,6 +62,8 @@
 #define GTUNE_USE_REUSEPORT      (1<<6)
 #define GTUNE_RESOLVE_DONTFAIL   (1<<7)
 
+#define GTUNE_SOCKET_TRANSFER   (1<<8)
+
 /* Access level for a stats socket */
 #define ACCESS_LVL_NONE     0
 #define ACCESS_LVL_USER     1

diff --git a/include/types/listener.h b/include/types/listener.h
index 227cc284d4ca3f055aefd6c0734ec41c683052e7..2b8f5feb64fe173984224542bed3aad36f299691 100644 (file)
--- a/include/types/listener.h
+++ b/include/types/listener.h
@@ -47,6 +47,7 @@ enum li_state {
        LI_INIT,        /* all parameters filled in, but not assigned yet */
        LI_ASSIGNED,    /* assigned to the protocol, but not listening yet */
        LI_PAUSED,      /* listener was paused, it's bound but not listening  */
+       LI_ZOMBIE,      /* The listener doesn't belong to the process, but is kept opened */
        LI_LISTEN,      /* started, listening but not enabled */
        LI_READY,       /* started, listening and enabled */
        LI_FULL,        /* reached its connection limit */

diff --git a/src/cfgparse.c b/src/cfgparse.c
index 47d33cf1b05ff72b42382e90bf8d6bdb3f41899f..348b9e886667e80157b6b6a2a8f6dcec6ef7f0e4 100644 (file)
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -659,6 +659,11 @@ int cfg_parse_global(const char *file, int linenum, char **args, int kwm)
                        goto out;
                global.tune.options &= ~GTUNE_USE_REUSEPORT;
        }
+       else if (!strcmp(args[0], "no-unused-socket")) {
+               if (alertif_too_many_args(0, file, linenum, args, &err_code))
+                       goto out;
+               global.tune.options &= ~GTUNE_SOCKET_TRANSFER;
+       }
        else if (!strcmp(args[0], "quiet")) {
                if (alertif_too_many_args(0, file, linenum, args, &err_code))
                        goto out;

diff --git a/src/cli.c b/src/cli.c
index 54fb43892e21294ded2c2c7695b4e3a5cdd2e3b4..20e143b965a0dbf1e1a41c76fabd3a283b5dd5c1 100644 (file)
--- a/src/cli.c
+++ b/src/cli.c
@@ -1063,10 +1063,11 @@ static int _getsocks(char **args, struct appctx *appctx, void *private)
                struct listener *l;
 
                list_for_each_entry(l, &px->conf.listeners, by_fe) {
-                       /* Only transfer IPv4/IPv6 sockets */
-                       if (l->proto->sock_family == AF_INET ||
+                       /* Only transfer IPv4/IPv6/UNIX sockets */
+                       if (l->state >= LI_ZOMBIE &&
+                           (l->proto->sock_family == AF_INET ||
                            l->proto->sock_family == AF_INET6 ||
-                           l->proto->sock_family == AF_UNIX)
+                           l->proto->sock_family == AF_UNIX))
                                tot_fd_nb++;
                }
                px = px->next;
@@ -1117,7 +1118,7 @@ static int _getsocks(char **args, struct appctx *appctx, void *private)
                list_for_each_entry(l, &px->conf.listeners, by_fe) {
                        int ret;
                        /* Only transfer IPv4/IPv6 sockets */
-                       if (l->state >= LI_LISTEN &&
+                       if (l->state >= LI_ZOMBIE &&
                            (l->proto->sock_family == AF_INET ||
                            l->proto->sock_family == AF_INET6 ||
                            l->proto->sock_family == AF_UNIX)) {

diff --git a/src/fd.c b/src/fd.c
index aeee6026100f8bbe86f82e4e0357b4432c68980b..1a62f9a66905a9ddf34e2a1a9f10d167f501f7ab 100644 (file)
--- a/src/fd.c
+++ b/src/fd.c
@@ -175,7 +175,7 @@ int fd_nbupdt = 0;             // number of updates in the list
 /* Deletes an FD from the fdsets, and recomputes the maxfd limit.
  * The file descriptor is also closed.
  */
-void fd_delete(int fd)
+static void fd_dodelete(int fd, int do_close)
 {
        if (fdtab[fd].linger_risk) {
                /* this is generally set when connecting to servers */
@@ -190,7 +190,8 @@ void fd_delete(int fd)
 
        port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
        fdinfo[fd].port_range = NULL;
-       close(fd);
+       if (do_close)
+               close(fd);
        fdtab[fd].owner = NULL;
        fdtab[fd].new = 0;
 
@@ -198,6 +199,22 @@ void fd_delete(int fd)
                maxfd--;
 }
 
+/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
+ * The file descriptor is also closed.
+ */
+void fd_delete(int fd)
+{
+       fd_dodelete(fd, 1);
+}
+
+/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
+ * The file descriptor is kept open.
+ */
+void fd_remove(int fd)
+{
+       fd_dodelete(fd, 0);
+}
+
 /* Scan and process the cached events. This should be called right after
  * the poller. The loop may cause new entries to be created, for example
  * if a listener causes an accept() to initiate a new incoming connection

diff --git a/src/haproxy.c b/src/haproxy.c
index bf176df7891217d1e499f05e85dc656077330307..01969c90d1778f3214b2a30259541b3407c3f303 100644 (file)
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -857,6 +857,7 @@ static void init(int argc, char **argv)
 #if defined(SO_REUSEPORT)
        global.tune.options |= GTUNE_USE_REUSEPORT;
 #endif
+       global.tune.options |= GTUNE_SOCKET_TRANSFER;
 
        pid = getpid();
        progname = *argv;
@@ -1668,6 +1669,15 @@ void deinit(void)
                }/* end while(s) */
 
                list_for_each_entry_safe(l, l_next, &p->conf.listeners, by_fe) {
+                       /*
+                        * Zombie proxy, the listener just pretend to be up
+                        * because they still hold an opened fd.
+                        * Close it and give the listener its real state.
+                        */
+                       if (p->state == PR_STSTOPPED && l->state >= LI_ZOMBIE) {
+                               close(l->fd);
+                               l->state = LI_INIT;
+                       }
                        unbind_listener(l);
                        delete_listener(l);
                        LIST_DEL(&l->by_fe);
@@ -2148,8 +2158,12 @@ int main(int argc, char **argv)
                px = proxy;
                while (px != NULL) {
                        if (px->bind_proc && px->state != PR_STSTOPPED) {
-                               if (!(px->bind_proc & (1UL << proc)))
-                                       stop_proxy(px);
+                               if (!(px->bind_proc & (1UL << proc))) {
+                                       if (global.tune.options & GTUNE_SOCKET_TRANSFER)
+                                               zombify_proxy(px);
+                                       else
+                                               stop_proxy(px);
+                               }
                        }
                        px = px->next;
                }

diff --git a/src/listener.c b/src/listener.c
index 7a1df0e412db7f1bf1ead7ce067cadf27ab2392f..a99e4c0d3573f00bea3fed0883b6fea61445bf42 100644 (file)
--- a/src/listener.c
+++ b/src/listener.c
@@ -59,7 +59,12 @@ void enable_listener(struct listener *listener)
                        /* we don't want to enable this listener and don't
                         * want any fd event to reach it.
                         */
-                       unbind_listener(listener);
+                       if (!(global.tune.options & GTUNE_SOCKET_TRANSFER))
+                               unbind_listener(listener);
+                       else {
+                               unbind_listener_no_close(listener);
+                               listener->state = LI_LISTEN;
+                       }
                }
                else if (listener->nbconn < listener->maxconn) {
                        fd_want_recv(listener->fd);
@@ -95,7 +100,7 @@ void disable_listener(struct listener *listener)
  */
 int pause_listener(struct listener *l)
 {
-       if (l->state <= LI_PAUSED)
+       if (l->state <= LI_ZOMBIE)
                return 1;
 
        if (l->proto->pause) {
@@ -149,7 +154,7 @@ int resume_listener(struct listener *l)
                        return 0;
        }
 
-       if (l->state < LI_PAUSED)
+       if (l->state < LI_PAUSED || l->state == LI_ZOMBIE)
                return 0;
 
        if (l->proto->sock_prot == IPPROTO_TCP &&
@@ -242,12 +247,7 @@ void dequeue_all_listeners(struct list *list)
        }
 }
 
-/* This function closes the listening socket for the specified listener,
- * provided that it's already in a listening state. The listener enters the
- * LI_ASSIGNED state. It always returns ERR_NONE. This function is intended
- * to be used as a generic function for standard protocols.
- */
-int unbind_listener(struct listener *listener)
+static int do_unbind_listener(struct listener *listener, int do_close)
 {
        if (listener->state == LI_READY)
                fd_stop_recv(listener->fd);
@@ -256,13 +256,35 @@ int unbind_listener(struct listener *listener)
                LIST_DEL(&listener->wait_queue);
 
        if (listener->state >= LI_PAUSED) {
-               fd_delete(listener->fd);
-               listener->fd = -1;
+               if (do_close) {
+                       fd_delete(listener->fd);
+                       listener->fd = -1;
+               }
+               else
+                       fd_remove(listener->fd);
                listener->state = LI_ASSIGNED;
        }
        return ERR_NONE;
 }
 
+/* This function closes the listening socket for the specified listener,
+ * provided that it's already in a listening state. The listener enters the
+ * LI_ASSIGNED state. It always returns ERR_NONE. This function is intended
+ * to be used as a generic function for standard protocols.
+ */
+int unbind_listener(struct listener *listener)
+{
+       return do_unbind_listener(listener, 1);
+}
+
+/* This function pretends the listener is dead, but keeps the FD opened, so
+ * that we can provide it, for conf reloading.
+ */
+int unbind_listener_no_close(struct listener *listener)
+{
+       return do_unbind_listener(listener, 0);
+}
+
 /* This function closes all listening sockets bound to the protocol <proto>,
  * and the listeners end in LI_ASSIGNED state if they were higher. It does not
  * detach them from the protocol. It always returns ERR_NONE.

diff --git a/src/proxy.c b/src/proxy.c
index d158fac03f0ac1a59d4fd1e2c042f3901b08ef65..dc702139adde1da7f429e0d6c5266cf6801293c8 100644 (file)
--- a/src/proxy.c
+++ b/src/proxy.c
@@ -991,6 +991,19 @@ void soft_stop(void)
        p = proxy;
        tv_update_date(0,1); /* else, the old time before select will be used */
        while (p) {
+               /* Zombie proxy, let's close the file descriptors */
+               if (p->state == PR_STSTOPPED &&
+                   !LIST_ISEMPTY(&p->conf.listeners) &&
+                   LIST_ELEM(p->conf.listeners.n,
+                   struct listener *, by_fe)->state >= LI_ZOMBIE) {
+                       struct listener *l;
+                       list_for_each_entry(l, &p->conf.listeners, by_fe) {
+                               if (l->state >= LI_ZOMBIE)
+                                       close(l->fd);
+                               l->state = LI_INIT;
+                       }
+               }
+
                if (p->state != PR_STSTOPPED) {
                        Warning("Stopping %s %s in %d ms.\n", proxy_cap_str(p->cap), p->id, p->grace);
                        send_log(p, LOG_WARNING, "Stopping %s %s in %d ms.\n", proxy_cap_str(p->cap), p->id, p->grace);
@@ -1051,6 +1064,45 @@ int pause_proxy(struct proxy *p)
        return 1;
 }
 
+/* This function makes the proxy unusable, but keeps the listening sockets
+ * opened, so that if any process requests them, we are able to serve them.
+ * This should only be called early, before we started accepting requests.
+ */
+void zombify_proxy(struct proxy *p)
+{
+       struct listener *l;
+       struct listener *first_to_listen = NULL;
+
+       list_for_each_entry(l, &p->conf.listeners, by_fe) {
+               enum li_state oldstate = l->state;
+
+               unbind_listener_no_close(l);
+               if (l->state >= LI_ASSIGNED) {
+                       delete_listener(l);
+                       listeners--;
+                       jobs--;
+               }
+               /*
+                * Pretend we're still up and running so that the fd
+                * will be sent if asked.
+                */
+               l->state = LI_ZOMBIE;
+               if (!first_to_listen && oldstate >= LI_LISTEN)
+                       first_to_listen = l;
+       }
+       /* Quick hack : at stop time, to know we have to close the sockets
+        * despite the proxy being marked as stopped, make the first listener
+        * of the listener list an active one, so that we don't have to
+        * parse the whole list to be sure.
+        */
+       if (first_to_listen && LIST_ELEM(p->conf.listeners.n,
+           struct listener *, by_fe) != first_to_listen) {
+               LIST_DEL(&l->by_fe);
+               LIST_ADD(&p->conf.listeners, &l->by_fe);
+       }
+
+       p->state = PR_STSTOPPED;
+}
 
 /*
  * This function completely stops a proxy and releases its listeners. It has