[MEDIUM] listeners: put listeners in queue upon resource shortage

When an accept() fails because of a connection limit or a memory shortage,
we now disable it and queue it so that it's dequeued only when a connection
is released. This has improved the behaviour of the process near the fd limit
as now a listener with a no connection (eg: stats) will not loop forever
trying to get its connection accepted.

The solution is still not 100% perfect, as we'd like to have this used when
proxy limits are reached (use a per-proxy list) and for safety, we'd need
to have dedicated tasks to periodically re-enable them (eg: to overcome
temporary system-wide resource limitations when no connection is released).

diff --git a/include/types/global.h b/include/types/global.h
index b6c60dd0ac2f90fd011f249b62711cc5e0b4363c..a7f22ac7c8ddf4288a7cacee87ada17f3bad11e3 100644 (file)
--- a/include/types/global.h
+++ b/include/types/global.h
@@ -123,6 +123,7 @@ extern const struct linger nolinger;
 extern int stopping;   /* non zero means stopping in progress */
 extern char hostname[MAX_HOSTNAME_LEN];
 extern char localpeer[MAX_HOSTNAME_LEN];
+extern struct list global_listener_queue; /* list of the temporarily limited listeners */
 
 #endif /* _TYPES_GLOBAL_H */
 

diff --git a/src/haproxy.c b/src/haproxy.c
index 8da7aa9458235bd321454d519edfb4901dd42180..c3841dd6fed1fca77d51c5a918726c98b0fd5e4b 100644 (file)
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -159,6 +159,8 @@ const struct linger nolinger = { .l_onoff = 1, .l_linger = 0 };
 char hostname[MAX_HOSTNAME_LEN];
 char localpeer[MAX_HOSTNAME_LEN];
 
+/* list of the temporarily limited listeners because of lack of resource */
+struct list global_listener_queue = LIST_HEAD_INIT(global_listener_queue);
 
 /*********************************************************************/
 /*  general purpose functions  ***************************************/

diff --git a/src/session.c b/src/session.c
index 6e718c00cb59a8e72264b1d1b946b1cb011f6cfb..750cf402312ab2f2c364991eb4feb37c6cf56867 100644 (file)
--- a/src/session.c
+++ b/src/session.c
@@ -2092,6 +2092,10 @@ struct task *process_session(struct task *t)
        if (s->listener->state == LI_FULL)
                resume_listener(s->listener);
 
+       /* Dequeues all of the listeners waiting for a resource */
+       if (!LIST_ISEMPTY(&global_listener_queue))
+               dequeue_all_listeners(&global_listener_queue);
+
        if (unlikely((global.mode & MODE_DEBUG) &&
                     (!(global.mode & MODE_QUIET) || (global.mode & MODE_VERBOSE)))) {
                int len;

diff --git a/src/stream_sock.c b/src/stream_sock.c
index 10e13cb00a286e0dc9159cdda5e1197a605d138e..fa03a625b1e2d7fa357671c0e455203db491d5e6 100644 (file)
--- a/src/stream_sock.c
+++ b/src/stream_sock.c
@@ -1192,6 +1192,7 @@ int stream_sock_accept(int fd)
        int max_accept = global.tune.maxaccept;
        int cfd;
        int ret;
+       int loops = 0;
 
        if (unlikely(l->nbconn >= l->maxconn)) {
                listener_full(l);
@@ -1208,6 +1209,7 @@ int stream_sock_accept(int fd)
                struct sockaddr_storage addr;
                socklen_t laddr = sizeof(addr);
 
+               loops++;
                cfd = accept(fd, (struct sockaddr *)&addr, &laddr);
                if (unlikely(cfd == -1)) {
                        switch (errno) {
@@ -1220,16 +1222,14 @@ int stream_sock_accept(int fd)
                                        send_log(p, LOG_EMERG,
                                                 "Proxy %s reached system FD limit at %d. Please check system tunables.\n",
                                                 p->id, maxfd);
-                               if (l->nbconn)
-                                       listener_full(l);
+                               limit_listener(l, &global_listener_queue);
                                return 0;
                        case EMFILE:
                                if (p)
                                        send_log(p, LOG_EMERG,
                                                 "Proxy %s reached process FD limit at %d. Please check 'ulimit-n' and restart.\n",
                                                 p->id, maxfd);
-                               if (l->nbconn)
-                                       listener_full(l);
+                               limit_listener(l, &global_listener_queue);
                                return 0;
                        case ENOBUFS:
                        case ENOMEM:
@@ -1237,8 +1237,7 @@ int stream_sock_accept(int fd)
                                        send_log(p, LOG_EMERG,
                                                 "Proxy %s reached system memory limit at %d sockets. Please check system tunables.\n",
                                                 p->id, maxfd);
-                               if (l->nbconn)
-                                       listener_full(l);
+                               limit_listener(l, &global_listener_queue);
                                return 0;
                        default:
                                return 0;
@@ -1250,6 +1249,7 @@ int stream_sock_accept(int fd)
                                 "Proxy %s reached the configured maximum connection limit. Please check the global 'maxconn' value.\n",
                                 p->id);
                        close(cfd);
+                       limit_listener(l, &global_listener_queue);
                        return 0;
                }
 
@@ -1276,10 +1276,7 @@ int stream_sock_accept(int fd)
                        if (ret == 0) /* successful termination */
                                continue;
 
-                       if (p) {
-                               disable_listener(l);
-                               p->state = PR_STIDLE;
-                       }
+                       limit_listener(l, &global_listener_queue);
                        return 0;
                }
 
@@ -1289,6 +1286,11 @@ int stream_sock_accept(int fd)
                }
 
        } /* end of while (p->feconn < p->maxconn) */
+
+       /* if we did not even enter the loop, we've reached resource limits */
+       if (!loops && max_accept)
+               limit_listener(l, &global_listener_queue);
+
        return 0;
 }