From 0c18a6fe34ee447f33b2bc757b119e8e7df4d931 Mon Sep 17 00:00:00 2001 From: Olivier Houchard Date: Sun, 2 Dec 2018 14:11:41 +0100 Subject: [PATCH] MEDIUM: servers: Add a way to keep idle connections alive. Add a new keyword for servers, "idle-timeout". If set, unused connections are kept alive until the timeout happens, and will be picked for reuse if no other connection is available. --- doc/configuration.txt | 6 +++ include/types/connection.h | 2 +- include/types/server.h | 3 ++ src/backend.c | 17 ++++++++ src/haproxy.c | 8 ++++ src/server.c | 81 ++++++++++++++++++++++++++++++++++++++ src/session.c | 20 +++++++++- 7 files changed, 135 insertions(+), 2 deletions(-) diff --git a/doc/configuration.txt b/doc/configuration.txt index 209798ebc7..487039f8b6 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -11676,6 +11676,12 @@ id the proxy. An unused ID will automatically be assigned if unset. The first assigned value will be 1. This ID is currently only returned in statistics. +idle-timeout + Set the time to keep a connection alive before destroying it. By default + connections are destroyed as soon as they are unused, if idle-timeout is + non-zero, then connection are kept alive for up to before being + destroyed, and can be reused if no other connection is available. + init-addr {last | libc | none | },[...]* Indicate in what order the server's address should be resolved upon startup if it uses an FQDN. Attempts are made to resolve the address by applying in diff --git a/include/types/connection.h b/include/types/connection.h index cd9f1e7547..745546cf94 100644 --- a/include/types/connection.h +++ b/include/types/connection.h @@ -444,7 +444,7 @@ struct connection { struct sockaddr_storage from; /* client address, or address to spoof when connecting to the server */ struct sockaddr_storage to; /* address reached by the client, or address to connect to */ } addr; /* addresses of the remote side, client for producer and server for consumer */ - struct timeval idle_tv; /* Time the connection was added to the idle list */ + unsigned int idle_time; /* Time the connection was added to the idle list */ }; /* PROTO token registration */ diff --git a/include/types/server.h b/include/types/server.h index 70e621a8de..bb53523cd8 100644 --- a/include/types/server.h +++ b/include/types/server.h @@ -221,6 +221,9 @@ struct server { struct list *priv_conns; /* private idle connections attached to stream interfaces */ struct list *idle_conns; /* sharable idle connections attached or not to a stream interface */ struct list *safe_conns; /* safe idle connections attached to stream interfaces, shared */ + struct list *idle_orphan_conns; /* Orphan connections idling */ + unsigned int idle_timeout; /* Time to keep an idling orphan connection alive */ + struct task **idle_task; /* task responsible for cleaning idle orphan connections */ struct task *warmup; /* the task dedicated to the warmup when slowstart is set */ struct conn_src conn_src; /* connection source settings */ diff --git a/src/backend.c b/src/backend.c index 404c5b38e1..e62a3b8347 100644 --- a/src/backend.c +++ b/src/backend.c @@ -1118,6 +1118,7 @@ int connect_server(struct stream *s) struct conn_stream *srv_cs = NULL; struct server *srv; int reuse = 0; + int reuse_orphan = 0; int err; int i; @@ -1189,6 +1190,13 @@ int connect_server(struct stream *s) else if (srv->idle_conns && !LIST_ISEMPTY(&srv->idle_conns[tid]) && (s->be->options & PR_O_REUSE_MASK) == PR_O_REUSE_ALWS) { srv_conn = LIST_ELEM(srv->idle_conns[tid].n, struct connection *, list); + } else if (srv->idle_orphan_conns && !LIST_ISEMPTY(&srv->idle_orphan_conns[tid]) && + (((s->be->options & PR_O_REUSE_MASK) == PR_O_REUSE_ALWS) || + (((s->be->options & PR_O_REUSE_MASK) != PR_O_REUSE_NEVR) && + s->txn && (s->txn->flags & TX_NOT_FIRST)))) { + srv_conn = LIST_ELEM(srv->idle_orphan_conns[tid].n, + struct connection *, list); + reuse_orphan = 1; } /* If we've picked a connection from the pool, we now have to @@ -1216,6 +1224,15 @@ int connect_server(struct stream *s) reuse = 0; } } + /* If we're really reusing the connection, remove it from the orphan + * list and add it back to the idle list. + */ + if (reuse && reuse_orphan) { + LIST_DEL(&srv_conn->list); + LIST_ADDQ(&srv->idle_conns[tid], &srv_conn->list); + if (LIST_ISEMPTY(&srv->idle_orphan_conns[tid])) + task_unlink_wq(srv->idle_task[tid]); + } /* We're about to use another connection, let the mux know we're * done with this one diff --git a/src/haproxy.c b/src/haproxy.c index 6567b43553..08ad7e5a2a 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -2418,6 +2418,14 @@ void deinit(void) free(s->idle_conns); free(s->priv_conns); free(s->safe_conns); + free(s->idle_orphan_conns); + if (s->idle_task) { + int i; + + for (i = 0; i < global.nbthread; i++) + task_free(s->idle_task[i]); + free(s->idle_task); + } if (s->use_ssl || s->check.use_ssl) { if (xprt_get(XPRT_SSL) && xprt_get(XPRT_SSL)->destroy_srv) diff --git a/src/server.c b/src/server.c index 53b9177be1..bae9a3b669 100644 --- a/src/server.c +++ b/src/server.c @@ -50,6 +50,7 @@ static void srv_update_status(struct server *s); static void srv_update_state(struct server *srv, int version, char **params); static int srv_apply_lastaddr(struct server *srv, int *err_code); static int srv_set_fqdn(struct server *srv, const char *fqdn, int dns_locked); +static struct task *cleanup_idle_connections(struct task *task, void *ctx, unsigned short state); /* List head of all known server keywords */ static struct srv_kw_list srv_keywords = { @@ -357,6 +358,28 @@ static int srv_parse_enabled(char **args, int *cur_arg, return 0; } +static int srv_parse_idle_timeout(char **args, int *cur_arg, struct proxy *curproxy, struct server *newsrv, char **err) +{ + const char *res; + char *arg; + unsigned int time; + + arg = args[*cur_arg + 1]; + if (!*arg) { + memprintf(err, "'%s' expects as argument.\n", args[*cur_arg]); + return ERR_ALERT | ERR_FATAL; + } + res = parse_time_err(arg, &time, TIME_UNIT_MS); + if (res) { + memprintf(err, "unexpected character '%c' in argument to <%s>.\n", + *res, args[*cur_arg]); + return ERR_ALERT | ERR_FATAL; + } + newsrv->idle_timeout = time; + + return 0; +} + /* parse the "id" server keyword */ static int srv_parse_id(char **args, int *cur_arg, struct proxy *curproxy, struct server *newsrv, char **err) { @@ -1197,6 +1220,7 @@ static struct srv_kw_list srv_kws = { "ALL", { }, { { "disabled", srv_parse_disabled, 0, 1 }, /* Start the server in 'disabled' state */ { "enabled", srv_parse_enabled, 0, 1 }, /* Start the server in 'enabled' state */ { "id", srv_parse_id, 1, 0 }, /* set id# of server */ + { "idle-timeout", srv_parse_idle_timeout, 1, 1 }, /* Set the time before we destroy orphan idle connections, defaults to 0 */ { "namespace", srv_parse_namespace, 1, 1 }, /* Namespace the server socket belongs to (if supported) */ { "no-agent-check", srv_parse_no_agent_check, 0, 1 }, /* Do not enable any auxiliary agent check */ { "no-backup", srv_parse_no_backup, 0, 1 }, /* Flag as non-backup server */ @@ -1640,6 +1664,7 @@ static void srv_settings_cpy(struct server *srv, struct server *src, int srv_tmp srv->tcp_ut = src->tcp_ut; #endif srv->mux_proto = src->mux_proto; + srv->idle_timeout = src->idle_timeout; if (srv_tmpl) srv->srvrq = src->srvrq; @@ -1889,7 +1914,28 @@ static int server_finalize_init(const char *file, int linenum, char **args, int px->srv_act++; srv_lb_commit_status(srv); + if (!srv->tmpl_info.prefix && srv->idle_timeout != 0) { + int i; + + srv->idle_orphan_conns = calloc(global.nbthread, sizeof(*srv->idle_orphan_conns)); + if (!srv->idle_orphan_conns) + goto err; + srv->idle_task = calloc(global.nbthread, sizeof(*srv->idle_task)); + if (!srv->idle_task) + goto err; + for (i = 0; i < global.nbthread; i++) { + LIST_INIT(&srv->idle_orphan_conns[i]); + srv->idle_task[i] = task_new(1 << i); + if (!srv->idle_task[i]) + goto err; + srv->idle_task[i]->process = cleanup_idle_connections; + srv->idle_task[i]->context = srv; + } + } + return 0; +err: + return ERR_ALERT | ERR_FATAL; } /* @@ -1973,6 +2019,24 @@ static int server_template_init(struct server *srv, struct proxy *px) /* Linked backwards first. This will be restablished after parsing. */ newsrv->next = px->srv; px->srv = newsrv; + if (newsrv->idle_timeout != 0) { + int i; + + newsrv->idle_orphan_conns = calloc(global.nbthread, sizeof(*newsrv->idle_orphan_conns)); + if (!newsrv->idle_orphan_conns) + goto err; + newsrv->idle_task = calloc(global.nbthread, sizeof(*newsrv->idle_task)); + if (!newsrv->idle_task) + goto err; + for (i = 0; i < global.nbthread; i++) { + LIST_INIT(&newsrv->idle_orphan_conns[i]); + newsrv->idle_task[i] = task_new(1 << i); + if (!newsrv->idle_task[i]) + goto err; + newsrv->idle_task[i]->process = cleanup_idle_connections; + newsrv->idle_task[i]->context = newsrv; + } + } } srv_set_id_from_prefix(srv, srv->tmpl_info.prefix, srv->tmpl_info.nb_low); @@ -5230,6 +5294,23 @@ static void srv_update_status(struct server *s) *s->adm_st_chg_cause = 0; } +static struct task *cleanup_idle_connections(struct task *task, void *context, unsigned short state) +{ + struct server *srv = context; + struct connection *conn, *conn_back; + unsigned int next_wakeup = 0; + + list_for_each_entry_safe(conn, conn_back, &srv->idle_orphan_conns[tid], list) { + if (conn->idle_time + srv->idle_timeout > now_ms) { + next_wakeup = conn->idle_time + srv->idle_timeout; + break; + } + conn->mux->destroy(conn); + } + if (next_wakeup > 0) + task_schedule(task, next_wakeup); + return task; +} /* * Local variables: * c-indent-level: 8 diff --git a/src/session.c b/src/session.c index 8080c94dee..9c52efc9d6 100644 --- a/src/session.c +++ b/src/session.c @@ -86,10 +86,28 @@ void session_free(struct session *sess) list_for_each_entry_safe(conn, conn_back, &sess->srv_list[i].list, session_list) { count++; if (conn->mux) { + struct server *srv; + LIST_DEL(&conn->session_list); LIST_INIT(&conn->session_list); + srv = objt_server(conn->target); conn->owner = NULL; - conn->mux->destroy(conn); + if (srv && srv->idle_timeout > 0 && + !(conn->flags & CO_FL_PRIVATE) && + conn->mux->avail_streams(conn) == + conn->mux->max_streams(conn)) { + LIST_DEL(&conn->list); + + LIST_ADDQ(&srv->idle_orphan_conns[tid], + &conn->list); + + conn->idle_time = now_ms; + if (!(task_in_wq(srv->idle_task[tid])) && + !(task_in_rq(srv->idle_task[tid]))) + task_schedule(srv->idle_task[tid], + tick_add(now_ms, srv->idle_timeout)); + } else + conn->mux->destroy(conn); } else { /* We have a connection, but not yet an associated mux. * So destroy it now. -- 2.39.5