MINOR: server: move actconns to the per-thread structure

The actconns list creates massive contention on low server counts because
it's in fact a list of streams using a server, all threads compete on the
list's head and it's still possible to see some watchdog panics on 48
threads under extreme contention with 47 threads trying to add and one
thread trying to delete.

Moving this list per thread is trivial because it's only used by
srv_shutdown_streams(), which simply required to iterate over the list.

The field was renamed to "streams" as it's really a list of streams
rather than a list of connections.

diff --git a/include/haproxy/server-t.h b/include/haproxy/server-t.h
index 37a3bce32410ba3abe9a14752b250144c081293e..ad3a40f6b4fabbb00459528eea9a229e9b4a29ab 100644 (file)
--- a/include/haproxy/server-t.h
+++ b/include/haproxy/server-t.h
@@ -205,6 +205,7 @@ struct tree_occ {
 
 /* Each server will have one occurrence of this structure per thread */
 struct srv_per_thread {
+       struct mt_list streams;                 /* streams using this server (used by "shutdown server sessions") */
        struct eb_root idle_conns;              /* Shareable idle connections */
        struct eb_root safe_conns;              /* Safe idle connections */
        struct eb_root avail_conns;             /* Connections in use, but with still new streams available */
@@ -236,8 +237,7 @@ struct server {
        struct be_counters counters;            /* statistics counters */
 
        struct eb_root pendconns;               /* pending connections */
-       struct mt_list actconns;                /* active connections (used by "shutdown server sessions") */
-       struct srv_per_thread *per_thr;         /* array of per-thread stuff such as connections lists, may be null */
+       struct srv_per_thread *per_thr;         /* array of per-thread stuff such as connections lists */
        unsigned int pool_purge_delay;          /* Delay before starting to purge the idle conns pool */
        unsigned int low_idle_conns;            /* min idle connection count to start picking from other threads */
        unsigned int max_idle_conns;            /* Max number of connection allowed in the orphan connections list */

diff --git a/include/haproxy/stream.h b/include/haproxy/stream.h
index 948ca732cbf6a5fa9115c0ba100b70da8df63fe5..aa6486e25a6965245f98e7ba0c5aedb8e74bf4e9 100644 (file)
--- a/include/haproxy/stream.h
+++ b/include/haproxy/stream.h
@@ -292,7 +292,7 @@ static inline void stream_add_srv_conn(struct stream *sess, struct server *srv)
         * from a conflict with an adjacent MT_LIST_DEL, and using it improves
         * the performance by about 3% on 32-cores.
         */
-       MT_LIST_ADD(&srv->actconns, &sess->by_srv);
+       MT_LIST_ADD(&srv->per_thr[tid].streams, &sess->by_srv);
        HA_ATOMIC_STORE(&sess->srv_conn, srv);
 }
 

diff --git a/src/cfgparse.c b/src/cfgparse.c
index eafd8ca8e277437f0c06dcc0884b0d4b936b9995..85995a5d1995bae4a02dee664f50fd95168c73ee 100644 (file)
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -3259,6 +3259,7 @@ out_uri_auth_compat:
                        newsrv->per_thr[i].idle_conns = EB_ROOT;
                        newsrv->per_thr[i].safe_conns = EB_ROOT;
                        newsrv->per_thr[i].avail_conns = EB_ROOT;
+                       MT_LIST_INIT(&newsrv->per_thr[i].streams);
                }
 
                if (newsrv->max_idle_conns != 0) {

diff --git a/src/hlua.c b/src/hlua.c
index 5e5414c22ee2573ec3837222e507deb4ddafe0dc..12a487267b967e89f2771c2bf50f74b8aba4575f 100644 (file)
--- a/src/hlua.c
+++ b/src/hlua.c
@@ -9175,7 +9175,6 @@ void hlua_init(void) {
        socket_tcp.next = NULL;
        socket_tcp.proxy = &socket_proxy;
        socket_tcp.obj_type = OBJ_TYPE_SERVER;
-       MT_LIST_INIT(&socket_tcp.actconns);
        socket_tcp.pendconns = EB_ROOT;
        LIST_ADD(&servers_list, &socket_tcp.global_list);
        socket_tcp.next_state = SRV_ST_RUNNING; /* early server setup */
@@ -9221,7 +9220,6 @@ void hlua_init(void) {
        socket_ssl.next = NULL;
        socket_ssl.proxy = &socket_proxy;
        socket_ssl.obj_type = OBJ_TYPE_SERVER;
-       MT_LIST_INIT(&socket_ssl.actconns);
        socket_ssl.pendconns = EB_ROOT;
        LIST_ADD(&servers_list, &socket_ssl.global_list);
        socket_ssl.next_state = SRV_ST_RUNNING; /* early server setup */

diff --git a/src/server.c b/src/server.c
index 5844c634503a1091ea9ab563b7e803d3b15539fd..04d2347798f4990ffda279cdb2803ec4c902965b 100644 (file)
--- a/src/server.c
+++ b/src/server.c
@@ -891,10 +891,12 @@ void srv_shutdown_streams(struct server *srv, int why)
 {
        struct stream *stream;
        struct mt_list *elt1, elt2;
+       int thr;
 
-       mt_list_for_each_entry_safe(stream, &srv->actconns, by_srv, elt1, elt2)
-               if (stream->srv_conn == srv)
-                       stream_shutdown(stream, why);
+       for (thr = 0; thr < global.nbthread; thr++)
+               mt_list_for_each_entry_safe(stream, &srv->per_thr[thr].streams, by_srv, elt1, elt2)
+                       if (stream->srv_conn == srv)
+                               stream_shutdown(stream, why);
 }
 
 /* Shutdown all connections of all backup servers of a proxy. The caller must
@@ -1750,7 +1752,6 @@ struct server *new_server(struct proxy *proxy)
 
        srv->obj_type = OBJ_TYPE_SERVER;
        srv->proxy = proxy;
-       MT_LIST_INIT(&srv->actconns);
        srv->pendconns = EB_ROOT;
        LIST_ADDQ(&servers_list, &srv->global_list);