From c35362a94a05a1c8341e21611643d5cc82713f8e Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Fri, 25 Apr 2014 13:58:37 +0200 Subject: [PATCH] MINOR: http: implement the max-keep-alive-queue setting Finn Arne Gangstad suggested that we should have the ability to break keep-alive when the target server has reached its maxconn and that a number of connections are present in the queue. After some discussion around his proposed patch, the following solution was suggested : have a per-proxy setting to fix a limit to the number of queued connections on a server after which we break keep-alive. This ensures that even in high latency networks where keep-alive is beneficial, we try to find a different server. This patch is partially based on his original proposal and implements this configurable threshold. --- doc/configuration.txt | 29 +++++++++++++++++++++++++++++ include/proto/queue.h | 5 +++++ include/types/proxy.h | 1 + src/backend.c | 6 +++++- src/cfgparse.c | 1 + src/proxy.c | 40 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 81 insertions(+), 1 deletion(-) diff --git a/doc/configuration.txt b/doc/configuration.txt index a1f58fb05d..5a185c3a9c 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -1275,6 +1275,7 @@ http-send-name-header - - X X id - X X X ignore-persist - X X X log (*) X X X X +max-keep-alive-queue X - X X maxconn X X X - mode X X X X monitor fail - X X - @@ -3379,6 +3380,34 @@ log-format See also : Custom Log Format (8.2.4) +max-keep-alive-queue + Set the maximum server queue size for maintaining keep-alive connections + May be used in sections: defaults | frontend | listen | backend + yes | no | yes | yes + + HTTP keep-alive tries to reuse the same server connection whenever possible, + but sometimes it can be counter-productive, for example if a server has a lot + of connections while other ones are idle. This is especially true for static + servers. + + The purpose of this setting is to set a threshold on the number of queued + connections at which haproxy stops trying to reuse the same server and prefers + to find another one. The default value, -1, means there is no limit. A value + of zero means that keep-alive requests will never be queued. For very close + servers which can be reached with a low latency and which are not sensible to + breaking keep-alive, a low value is recommended (eg: local static server can + use a value of 10 or less). For remote servers suffering from a high latency, + higher values might be needed to cover for the latency and/or the cost of + picking a different server. + + Note that this has no impact on responses which are maintained to the same + server consecutively to a 401 response. They will still go to the same server + even if they have to be queued. + + See also : "option http-server-close", "option prefer-last-server", server + "maxconn" and cookie persistence. + + maxconn Fix the maximum number of concurrent connections on a frontend May be used in sections : defaults | frontend | listen | backend diff --git a/include/proto/queue.h b/include/proto/queue.h index 7bf8137077..fb73926162 100644 --- a/include/proto/queue.h +++ b/include/proto/queue.h @@ -65,6 +65,11 @@ static inline struct pendconn *pendconn_from_px(const struct proxy *px) { return LIST_ELEM(px->pendconns.n, struct pendconn *, list); } +/* Returns 0 if all slots are full on a server, or 1 if there are slots available. */ +static inline int server_has_room(const struct server *s) { + return !s->maxconn || s->cur_sess < srv_dynamic_maxconn(s); +} + /* returns 0 if nothing has to be done for server regarding queued connections, * and non-zero otherwise. If the server is down, we only check its own queue. Suited * for and if/else usage. diff --git a/include/types/proxy.h b/include/types/proxy.h index 28be98472a..4c0c660e95 100644 --- a/include/types/proxy.h +++ b/include/types/proxy.h @@ -289,6 +289,7 @@ struct proxy { struct list pendconns; /* pending connections with no server assigned yet */ int nbpend; /* number of pending connections with no server assigned yet */ int totpend; /* total number of pending connections on this instance (for stats) */ + int max_ka_queue; /* 1+maximum requests in queue accepted for reusing a K-A conn (0=none) */ unsigned int feconn, beconn; /* # of active frontend and backends sessions */ struct freq_ctr fe_req_per_sec; /* HTTP requests per second on the frontend */ struct freq_ctr fe_conn_per_sec; /* received connections per second on the frontend */ diff --git a/src/backend.c b/src/backend.c index 4bb2ea364d..bc63903fc6 100644 --- a/src/backend.c +++ b/src/backend.c @@ -541,8 +541,12 @@ int assign_server(struct session *s) if (conn && (conn->flags & CO_FL_CONNECTED) && - ((s->be->options & PR_O_PREF_LAST) || (s->txn.flags & TX_PREFER_LAST)) && objt_server(conn->target) && __objt_server(conn->target)->proxy == s->be && + ((s->txn.flags & TX_PREFER_LAST) || + ((s->be->options & PR_O_PREF_LAST) && + (!s->be->max_ka_queue || + server_has_room(__objt_server(conn->target)) || + (__objt_server(conn->target)->nbpend + 1) < s->be->max_ka_queue))) && srv_is_usable(__objt_server(conn->target)->state, __objt_server(conn->target)->eweight)) { /* This session was relying on a server in a previous request * and the proxy has "option prefer-current-server" set, so diff --git a/src/cfgparse.c b/src/cfgparse.c index ec8f3ae1b3..7a869e36c8 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -1930,6 +1930,7 @@ int cfg_parse_listen(const char *file, int linenum, char **args, int kwm) if (curproxy->cap & PR_CAP_BE) { curproxy->fullconn = defproxy.fullconn; curproxy->conn_retries = defproxy.conn_retries; + curproxy->max_ka_queue = defproxy.max_ka_queue; if (defproxy.check_req) { curproxy->check_req = calloc(1, defproxy.check_len); diff --git a/src/proxy.c b/src/proxy.c index 2f2eb494c5..fb1a3b4258 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -275,6 +275,45 @@ static int proxy_parse_rate_limit(char **args, int section, struct proxy *proxy, return retval; } +/* This function parses a "max-keep-alive-queue" statement in a proxy section. + * It returns -1 if there is any error, 1 for a warning, otherwise zero. If it + * does not return zero, it will write an error or warning message into a + * preallocated buffer returned at . The function must be called with + * pointing to the first command line word, with pointing to + * the proxy being parsed, and to the default proxy or NULL. + */ +static int proxy_parse_max_ka_queue(char **args, int section, struct proxy *proxy, + struct proxy *defpx, const char *file, int line, + char **err) +{ + int retval; + char *res; + unsigned int val; + + retval = 0; + + if (*args[1] == 0) { + memprintf(err, "'%s' expects expects an integer value (or -1 to disable)", args[0]); + return -1; + } + + val = strtol(args[1], &res, 0); + if (*res) { + memprintf(err, "'%s' : unexpected character '%c' in integer value '%s'", args[0], *res, args[1]); + return -1; + } + + if (!(proxy->cap & PR_CAP_BE)) { + memprintf(err, "%s will be ignored because %s '%s' has no backend capability", + args[0], proxy_type_str(proxy), proxy->id); + retval = 1; + } + + /* we store so that a user-facing value of -1 is stored as zero (default) */ + proxy->max_ka_queue = val + 1; + return retval; +} + /* This function inserts proxy into the tree of known proxies. The proxy's * name is used as the storing key so it must already have been initialized. */ @@ -926,6 +965,7 @@ static struct cfg_kw_list cfg_kws = {ILH, { { CFG_LISTEN, "contimeout", proxy_parse_timeout }, { CFG_LISTEN, "srvtimeout", proxy_parse_timeout }, { CFG_LISTEN, "rate-limit", proxy_parse_rate_limit }, + { CFG_LISTEN, "max-keep-alive-queue", proxy_parse_max_ka_queue }, { 0, NULL, NULL }, }}; -- 2.39.5