MINOR: server/idle: make the next_takeover index per-tgroup

In order to evenly pick idle connections from other threads, there is
a "next_takeover" index in the server, that is incremented each time
a connection is picked from another thread, and indicates which one to
start from next time.

With thread groups this doesn't work well because the index is the same
regardless of the group, and if a group has more threads than another,
there's even a risk to reintroduce an imbalance.

This patch introduces a new per-tgroup storage in servers which, for now,
only contains an instance of this next_takeover index. This way each
thread will now only manipulate the index specific to its own group, and
the takeover will become fair again. More entries may come soon.

diff --git a/include/haproxy/server-t.h b/include/haproxy/server-t.h
index ff5faae6aa771a740495167adcfb5b06c2fd1534..1ff7d18b1ad13373a968453803ff23d26cecd1f7 100644 (file)
--- a/include/haproxy/server-t.h
+++ b/include/haproxy/server-t.h
@@ -212,6 +212,11 @@ struct srv_per_thread {
        struct eb_root avail_conns;             /* Connections in use, but with still new streams available */
 };
 
+/* Each server will have one occurrence of this structure per thread group */
+struct srv_per_tgroup {
+       unsigned int next_takeover;             /* thread ID to try to steal connections from next time */
+};
+
 /* Configure the protocol selection for websocket */
 enum __attribute__((__packed__)) srv_ws_mode {
        SRV_WS_AUTO = 0,
@@ -239,6 +244,7 @@ struct server {
        const struct mux_proto_list *mux_proto;       /* the mux to use for all outgoing connections (specified by the "proto" keyword) */
        unsigned maxconn, minconn;              /* max # of active sessions (0 = unlimited), min# for dynamic limit. */
        struct srv_per_thread *per_thr;         /* array of per-thread stuff such as connections lists */
+       struct srv_per_tgroup *per_tgrp;        /* array of per-tgroup stuff such as idle conns */
        unsigned int *curr_idle_thr;            /* Current number of orphan idling connections per thread */
 
        unsigned int pool_purge_delay;          /* Delay before starting to purge the idle conns pool */
@@ -282,7 +288,6 @@ struct server {
        unsigned int curr_used_conns;           /* Current number of used connections */
        unsigned int max_used_conns;            /* Max number of used connections (the counter is reset at each connection purges */
        unsigned int est_need_conns;            /* Estimate on the number of needed connections (max of curr and previous max_used) */
-       unsigned int next_takeover;             /* thread ID to try to steal connections from next time */
 
        struct queue queue;                     /* pending connections */
 

diff --git a/src/backend.c b/src/backend.c
index 4b1ce5ce3dda5fdfe0fc7a7ad57f8175faa8cf24..bac8a08a5536f26919fd2c70dfc79f34a92c2a53 100644 (file)
--- a/src/backend.c
+++ b/src/backend.c
@@ -1201,7 +1201,7 @@ static struct connection *conn_backend_get(struct stream *s, struct server *srv,
        /* Lookup all other threads for an idle connection, starting from last
         * unvisited thread, but always staying in the same group.
         */
-       stop = srv->next_takeover;
+       stop = srv->per_tgrp[tgid - 1].next_takeover;
        if (stop >= tg->count)
                stop %= tg->count;
 
@@ -1246,7 +1246,7 @@ static struct connection *conn_backend_get(struct stream *s, struct server *srv,
                conn = NULL;
  done:
        if (conn) {
-               _HA_ATOMIC_STORE(&srv->next_takeover, (i + 1 == tg->base + tg->count) ? tg->base : i + 1);
+               _HA_ATOMIC_STORE(&srv->per_tgrp[tgid - 1].next_takeover, (i + 1 == tg->base + tg->count) ? tg->base : i + 1);
 
                srv_use_conn(srv, conn);
 

diff --git a/src/server.c b/src/server.c
index 699113e889c9dcdf24df6b19d05197796184cbee..51ca0cdf3b9dd9447f8486db1984e1dc8aef09e1 100644 (file)
--- a/src/server.c
+++ b/src/server.c
@@ -2381,6 +2381,7 @@ struct server *srv_drop(struct server *srv)
        free(srv->hostname_dn);
        free((char*)srv->conf.file);
        free(srv->per_thr);
+       free(srv->per_tgrp);
        free(srv->curr_idle_thr);
        free(srv->resolvers_id);
        free(srv->addr_node.key);
@@ -4650,7 +4651,8 @@ int srv_init_per_thr(struct server *srv)
        int i;
 
        srv->per_thr = calloc(global.nbthread, sizeof(*srv->per_thr));
-       if (!srv->per_thr)
+       srv->per_tgrp = calloc(global.nbtgroups, sizeof(*srv->per_tgrp));
+       if (!srv->per_thr || !srv->per_tgrp)
                return -1;
 
        for (i = 0; i < global.nbthread; i++) {