From: Baptiste Assmann Date: Tue, 10 Dec 2013 23:52:19 +0000 (+0100) Subject: MEDIUM: tcp-check new feature: connect X-Git-Tag: v1.5-dev22~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=69e273f3fcfbfb9cc0fb5a09668faad66cfbd36b;p=thirdparty%2Fhaproxy.git MEDIUM: tcp-check new feature: connect A new tcp-check rule type: connect. It allows HAProxy to test applications which stand on multiple ports or multiple applications load-balanced through the same backend. --- diff --git a/doc/configuration.txt b/doc/configuration.txt index b8814277da..524b83df42 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -1220,6 +1220,7 @@ http-check expect - - X X http-check send-state X - X X http-request - X X X http-response - X X X +tcp-check connect - - X X tcp-check expect - - X X tcp-check send - - X X tcp-check send-binary - - X X @@ -3021,6 +3022,63 @@ http-response { allow | deny | add-header | set-nice | ACL usage. +tcp-check connect [params*] + Opens a new connection + May be used in sections: defaults | frontend | listen | backend + no | no | yes | yes + + When an application lies on more than a single TCP port or when HAProxy + load-balance many services in a single backend, it makes sense to probe all + the services individually before considering a server as operational. + + When there are no TCP port configured on the server line neither server port + directive, then the 'tcp-check connect port ' must be the first step + of the sequence. + + In a tcp-check ruleset a 'connect' is required, it is also mandatory to start + the ruleset with a 'connect' rule. Purpose is to ensure admin know what they + do. + + Parameters : + They are optional and can be used to describe how HAProxy should open and + use the TCP connection. + + port if not set, check port or server port is used. + It tells HAProxy where to open the connection to. + must be a valid TCP port source integer, from 1 to 65535. + + send-proxy send a PROXY protocol string + + ssl opens a ciphered connection + + Examples: + # check HTTP and HTTPs services on a server. + # first open port 80 thanks to server line port directive, then + # tcp-check opens port 443, ciphered and run a request on it: + option tcp-check + tcp-check connect + tcp-check send GET\ /\ HTTP/1.0\r\n + tcp-check send Host:\ haproxy.1wt.eu\r\n + tcp-check send \r\n + tcp-check expect rstring (2..|3..) + tcp-check connect port 443 ssl + tcp-check send GET\ /\ HTTP/1.0\r\n + tcp-check send Host:\ haproxy.1wt.eu\r\n + tcp-check send \r\n + tcp-check expect rstring (2..|3..) + server www 10.0.0.1 check port 80 + + # check both POP and IMAP from a single server: + option tcp-check + tcp-check connect port 110 + tcp-check expect string +OK\ POP3\ ready + tcp-check connect port 143 + tcp-check expect string *\ OK\ IMAP4\ ready + server mail 10.0.0.1 check + + See also : "option tcp-check", "tcp-check send", "tcp-check expect" + + tcp-check expect [!] Specify data to be collected and analysed during a generic health check May be used in sections: defaults | frontend | listen | backend @@ -3098,8 +3156,8 @@ tcp-check expect [!] tcp-check expect string +OK - See also : "option tcp-check", "tcp-check send", "http-check expect", - tune.chksize + See also : "option tcp-check", "tcp-check connect", "tcp-check send", + "tcp-check send-binary", "http-check expect", tune.chksize tcp-check send @@ -3116,8 +3174,8 @@ tcp-check send tcp-check send info\ replication\r\n tcp-check expect string role:master - See also : "option tcp-check", "tcp-check expect", "tcp-check send-binary", - tune.chksize + See also : "option tcp-check", "tcp-check connect", "tcp-check expect", + "tcp-check send-binary", tune.chksize tcp-check send-binary @@ -3141,9 +3199,8 @@ tcp-check send-binary tcp-check expect binary 2b504F4e47 # +PONG - See also : "option tcp-check", "tcp-check expect", "tcp-check send", - tune.chksize - + See also : "option tcp-check", "tcp-check connect", "tcp-check expect", + "tcp-check send", tune.chksize http-send-name-header [
] diff --git a/include/types/checks.h b/include/types/checks.h index 7db68b4b0f..83ea2b510f 100644 --- a/include/types/checks.h +++ b/include/types/checks.h @@ -135,6 +135,7 @@ struct check { int use_ssl; /* use SSL for health checks */ int send_proxy; /* send a PROXY protocol header with checks */ struct tcpcheck_rule *current_step; /* current step when using tcpcheck */ + struct tcpcheck_rule *last_started_step;/* pointer to latest tcpcheck rule started */ int inter, fastinter, downinter; /* checks: time in milliseconds */ enum chk_result result; /* health-check result : CHK_RES_* */ int state; /* state of the check : CHK_ST_* */ @@ -160,8 +161,14 @@ struct analyze_status { enum { TCPCHK_ACT_SEND = 0, /* send action, regular string format */ TCPCHK_ACT_EXPECT, /* expect action, either regular or binary string */ + TCPCHK_ACT_CONNECT, /* connect action, to probe a new port */ }; +/* flags used by tcpcheck_rule->conn_opts */ +#define TCPCHK_OPT_NONE 0x0000 /* no options specified, default */ +#define TCPCHK_OPT_SEND_PROXY 0x0001 /* send proxy-protocol string */ +#define TCPCHK_OPT_SSL 0x0002 /* SSL connection */ + struct tcpcheck_rule { struct list list; /* list linked to from the proxy */ int action; /* action: send or expect */ @@ -171,6 +178,8 @@ struct tcpcheck_rule { int string_len; /* string lenght */ regex_t *expect_regex; /* expected */ int inverse; /* 0 = regular match, 1 = inverse match */ + unsigned short port; /* port to connect to */ + unsigned short conn_opts; /* options when setting up a new connection */ }; #endif /* _TYPES_CHECKS_H */ diff --git a/include/types/proxy.h b/include/types/proxy.h index 3f2b814ab6..af2a3abd3d 100644 --- a/include/types/proxy.h +++ b/include/types/proxy.h @@ -104,8 +104,7 @@ enum pr_mode { #define PR_O_HTTP_TUN 0x04000000 /* HTTP tunnel mode : no analysis past first request/response */ /* unassigned values : 0x05000000, 0x06000000, 0x07000000 */ #define PR_O_HTTP_MODE 0x07000000 /* MASK to retrieve the HTTP mode */ - -/* unused: 0x08000000 */ +#define PR_O_TCPCHK_SSL 0x08000000 /* at least one TCPCHECK connect rule requires SSL */ #define PR_O_CONTSTATS 0x10000000 /* continous counters */ #define PR_O_HTTP_PROXY 0x20000000 /* Enable session to use HTTP proxy operations */ #define PR_O_DISABLE404 0x40000000 /* Disable a server on a 404 response to a health-check */ diff --git a/src/cfgparse.c b/src/cfgparse.c index 4c52eed3be..9993c61d16 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -4089,7 +4089,70 @@ stats_error_parsing: if (warnifnotcap(curproxy, PR_CAP_BE, file, linenum, args[0], NULL)) err_code |= ERR_WARN; - if (strcmp(args[1], "send") == 0) { + if (strcmp(args[1], "connect") == 0) { + const char *ptr_arg; + int cur_arg; + struct tcpcheck_rule *tcpcheck; + struct list *l; + + /* check if first rule is also a 'connect' action */ + l = (struct list *)&curproxy->tcpcheck_rules; + if (l->p != l->n) { + tcpcheck = (struct tcpcheck_rule *)l->n; + if (tcpcheck && tcpcheck->action != TCPCHK_ACT_CONNECT) { + Alert("parsing [%s:%d] : first step MUST also be a 'connect' when there is a 'connect' step in the tcp-check ruleset.\n", + file, linenum); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + } + + cur_arg = 2; + tcpcheck = (struct tcpcheck_rule *)calloc(1, sizeof(*tcpcheck)); + tcpcheck->action = TCPCHK_ACT_CONNECT; + + /* parsing each parameters to fill up the rule */ + while (*(ptr_arg = args[cur_arg])) { + /* tcp port */ + if (strcmp(args[cur_arg], "port") == 0) { + if ( (atol(args[cur_arg + 1]) > 65535) || + (atol(args[cur_arg + 1]) < 1) ){ + Alert("parsing [%s:%d] : '%s %s %s' expects a valid TCP port (from range 1 to 65535), got %s.\n", + file, linenum, args[0], args[1], "port", args[cur_arg + 1]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + tcpcheck->port = atol(args[cur_arg + 1]); + cur_arg += 2; + } + /* send proxy protocol */ + else if (strcmp(args[cur_arg], "send-proxy") == 0) { + tcpcheck->conn_opts |= TCPCHK_OPT_SEND_PROXY; + cur_arg++; + } +#ifdef USE_OPENSSL + else if (strcmp(args[cur_arg], "ssl") == 0) { + curproxy->options |= PR_O_TCPCHK_SSL; + tcpcheck->conn_opts |= TCPCHK_OPT_SSL; + cur_arg++; + } +#endif /* USE_OPENSSL */ + else { +#ifdef USE_OPENSSL + Alert("parsing [%s:%d] : '%s %s' expects 'port', 'send-proxy' or 'ssl' but got '%s' as argument.\n", +#else /* USE_OPENSSL */ + Alert("parsing [%s:%d] : '%s %s' expects 'port', 'send-proxy' or but got '%s' as argument.\n", +#endif /* USE_OPENSSL */ + file, linenum, args[0], args[1], args[cur_arg]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + + } + + LIST_ADDQ(&curproxy->tcpcheck_rules, &tcpcheck->list); + } + else if (strcmp(args[1], "send") == 0) { if (! *(args[2]) ) { /* SEND string expected */ Alert("parsing [%s:%d] : '%s %s %s' expects as argument.\n", @@ -4235,7 +4298,7 @@ stats_error_parsing: } } else { - Alert("parsing [%s:%d] : '%s' only supports 'send' or 'expect'.\n", file, linenum, args[0]); + Alert("parsing [%s:%d] : '%s' only supports 'connect', 'send' or 'expect'.\n", file, linenum, args[0]); err_code |= ERR_ALERT | ERR_FATAL; goto out; } @@ -5191,7 +5254,7 @@ stats_error_parsing: */ if (!newsrv->check.port && !is_addr(&newsrv->check_common.addr)) { #ifdef USE_OPENSSL - newsrv->check.use_ssl |= newsrv->use_ssl; + newsrv->check.use_ssl |= (newsrv->use_ssl || (newsrv->proxy->options & PR_O_TCPCHK_SSL)); #endif newsrv->check.send_proxy |= (newsrv->state & SRV_SEND_PROXY); } @@ -5215,11 +5278,40 @@ stats_error_parsing: break; } } + /* + * We need at least a service port, a check port or the first tcp-check rule must + * be a 'connect' one + */ if (!newsrv->check.port) { - Alert("parsing [%s:%d] : server %s has neither service port nor check port. Check has been disabled.\n", - file, linenum, newsrv->id); - err_code |= ERR_ALERT | ERR_FATAL; - goto out; + struct tcpcheck_rule *n = NULL, *r = NULL; + struct list *l; + + r = (struct tcpcheck_rule *)newsrv->proxy->tcpcheck_rules.n; + if (!r) { + Alert("parsing [%s:%d] : server %s has neither service port nor check port. Check has been disabled.\n", + file, linenum, newsrv->id); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + if ((r->action != TCPCHK_ACT_CONNECT) || !r->port) { + Alert("parsing [%s:%d] : server %s has neither service port nor check port nor tcp_check rule 'connect' with port information. Check has been disabled.\n", + file, linenum, newsrv->id); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + else { + /* scan the tcp-check ruleset to ensure a port has been configured */ + l = &newsrv->proxy->tcpcheck_rules; + list_for_each_entry(n, l, list) { + r = (struct tcpcheck_rule *)n->list.p; + if ((r->action == TCPCHK_ACT_CONNECT) && (!r->port)) { + Alert("parsing [%s:%d] : server %s has neither service port nor check port, and a tcp_check rule 'connect' with no port information. Check has been disabled.\n", + file, linenum, newsrv->id); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + } + } } /* note: check type will be set during the config review phase */ diff --git a/src/checks.c b/src/checks.c index 70b5a2ee54..c3051aa43b 100644 --- a/src/checks.c +++ b/src/checks.c @@ -35,6 +35,11 @@ #include +#ifdef USE_OPENSSL +#include +#include +#endif /* USE_OPENSSL */ + #include #include #include @@ -44,6 +49,7 @@ #include #include #include +#include #include #include #include @@ -862,7 +868,10 @@ static void chk_report_conn_err(struct connection *conn, int errno_bck, int expi if (check->type == PR_O2_TCPCHK_CHK) { chunk_printf(chk, " at step %d of tcp-check", tcpcheck_get_step_id(check->server)); /* we were looking for a string */ - if (check->current_step && check->current_step->action == TCPCHK_ACT_EXPECT) { + if (check->current_step && check->current_step->action == TCPCHK_ACT_CONNECT) { + chunk_appendf(chk, " (connect)"); + } + else if (check->current_step && check->current_step->action == TCPCHK_ACT_EXPECT) { if (check->current_step->string) chunk_appendf(chk, " (string '%s')", check->current_step->string); else if (check->current_step->expect_regex) @@ -1564,7 +1573,15 @@ static struct task *process_chk(struct task *t) /* we'll connect to the addr on the server */ conn->addr.to = s->addr; - set_host_port(&conn->addr.to, check->port); + if (check->port) { + set_host_port(&conn->addr.to, check->port); + } + + if (check->type == PR_O2_TCPCHK_CHK) { + tcpcheck_main(conn); + return t; + } + /* It can return one of : * - SN_ERR_NONE if everything's OK @@ -1920,7 +1937,7 @@ static int tcpcheck_get_step_id(struct server *s) struct tcpcheck_rule *cur = NULL, *next = NULL; int i = 0; - cur = s->check.current_step; + cur = s->check.last_started_step; /* no step => first step */ if (cur == NULL) @@ -1948,8 +1965,11 @@ static void tcpcheck_main(struct connection *conn) struct server *s = check->server; struct task *t = check->task; - /* don't do anything until the connection is established */ - if (!(conn->flags & CO_FL_CONNECTED)) { + /* + * don't do anything until the connection is established but if we're running + * first step which must be a connect + */ + if (check->current_step && (!(conn->flags & CO_FL_CONNECTED))) { /* update expire time, should be done by process_chk */ /* we allow up to min(inter, timeout.connect) for a connection * to establish but only when timeout.check is set @@ -2023,7 +2043,129 @@ static void tcpcheck_main(struct connection *conn) break; } - if (check->current_step->action == TCPCHK_ACT_SEND) { + if (check->current_step->action == TCPCHK_ACT_CONNECT) { + struct protocol *proto; + struct xprt_ops *xprt; + + /* mark the step as started */ + check->last_started_step = check->current_step; + /* first, shut existing connection */ + conn_force_close(conn); + + /* prepare new connection */ + /* initialization */ + conn_init(conn); + conn_attach(conn, check, &check_conn_cb); + conn->target = &s->obj_type; + + /* no client address */ + clear_addr(&conn->addr.from); + + if (is_addr(&s->check_common.addr)) + /* we'll connect to the check addr specified on the server */ + conn->addr.to = s->check_common.addr; + else + /* we'll connect to the addr on the server */ + conn->addr.to = s->addr; + + /* protocol */ + proto = protocol_by_family(conn->addr.to.ss_family); + + /* port */ + if (check->current_step->port) + set_host_port(&conn->addr.to, check->current_step->port); + else if (check->port) + set_host_port(&conn->addr.to, check->port); + +#ifdef USE_OPENSSL + if (check->current_step->conn_opts & TCPCHK_OPT_SSL) { + xprt = &ssl_sock; + ssl_sock_prepare_srv_ctx(s, s->proxy); + } + else { + xprt = &raw_sock; + } +#else /* USE_OPENSSL */ + xprt = &raw_sock; +#endif /* USE_OPENSSL */ + conn_prepare(conn, proto, xprt); + + ret = SN_ERR_INTERNAL; + if (proto->connect) + ret = proto->connect(conn, check->type, (check->type) ? 0 : 2); + conn->flags |= CO_FL_WAKE_DATA; + if (check->current_step->conn_opts & TCPCHK_OPT_SEND_PROXY) { + conn->send_proxy_ofs = 1; + conn->flags |= CO_FL_SEND_PROXY; + } + + /* It can return one of : + * - SN_ERR_NONE if everything's OK + * - SN_ERR_SRVTO if there are no more servers + * - SN_ERR_SRVCL if the connection was refused by the server + * - SN_ERR_PRXCOND if the connection has been limited by the proxy (maxconn) + * - SN_ERR_RESOURCE if a system resource is lacking (eg: fd limits, ports, ...) + * - SN_ERR_INTERNAL for any other purely internal errors + * Additionnally, in the case of SN_ERR_RESOURCE, an emergency log will be emitted. + * Note that we try to prevent the network stack from sending the ACK during the + * connect() when a pure TCP check is used (without PROXY protocol). + */ + switch (ret) { + case SN_ERR_NONE: + /* we allow up to min(inter, timeout.connect) for a connection + * to establish but only when timeout.check is set + * as it may be to short for a full check otherwise + */ + t->expire = tick_add(now_ms, MS_TO_TICKS(check->inter)); + + if (s->proxy->timeout.check && s->proxy->timeout.connect) { + int t_con = tick_add(now_ms, s->proxy->timeout.connect); + t->expire = tick_first(t->expire, t_con); + } + break; + case SN_ERR_SRVTO: /* ETIMEDOUT */ + case SN_ERR_SRVCL: /* ECONNREFUSED, ENETUNREACH, ... */ + chunk_printf(&trash, "TCPCHK error establishing connection at step %d: %s", + tcpcheck_get_step_id(s), strerror(errno)); + set_server_check_status(check, HCHK_STATUS_L4CON, trash.str); + goto out_end_tcpcheck; + case SN_ERR_PRXCOND: + case SN_ERR_RESOURCE: + case SN_ERR_INTERNAL: + chunk_printf(&trash, "TCPCHK error establishing connection at step %d", + tcpcheck_get_step_id(s)); + set_server_check_status(check, HCHK_STATUS_SOCKERR, trash.str); + goto out_end_tcpcheck; + } + + /* allow next rule */ + cur = (struct tcpcheck_rule *)cur->list.n; + check->current_step = cur; + + /* don't do anything until the connection is established */ + if (!(conn->flags & CO_FL_CONNECTED)) { + /* update expire time, should be done by process_chk */ + /* we allow up to min(inter, timeout.connect) for a connection + * to establish but only when timeout.check is set + * as it may be to short for a full check otherwise + */ + while (tick_is_expired(t->expire, now_ms)) { + int t_con; + + t_con = tick_add(t->expire, s->proxy->timeout.connect); + t->expire = tick_add(t->expire, MS_TO_TICKS(check->inter)); + + if (s->proxy->timeout.check) + t->expire = tick_first(t->expire, t_con); + } + return; + } + + } /* end 'connect' */ + else if (check->current_step->action == TCPCHK_ACT_SEND) { + /* mark the step as started */ + check->last_started_step = check->current_step; + /* reset the read buffer */ if (*check->bi->data != '\0') { *check->bi->data = '\0'; @@ -2076,6 +2218,10 @@ static void tcpcheck_main(struct connection *conn) goto out_need_io; } + /* mark the step as started */ + check->last_started_step = check->current_step; + + /* Intermediate or complete response received. * Terminate string in check->bi->data buffer. */ @@ -2186,22 +2332,13 @@ static void tcpcheck_main(struct connection *conn) if (conn->flags & CO_FL_ERROR) chk_report_conn_err(conn, 0, 0); - /* Close the connection... We absolutely want to perform a hard close - * and reset the connection if some data are pending, otherwise we end - * up with many TIME_WAITs and eat all the source port range quickly. - * To avoid sending RSTs all the time, we first try to drain pending - * data. - */ - if (conn->xprt && conn->xprt->shutw) - conn->xprt->shutw(conn, 0); - + /* cleanup before leaving */ check->current_step = NULL; if (check->result == CHK_RES_FAILED) conn->flags |= CO_FL_ERROR; __conn_data_stop_both(conn); - task_wakeup(t, TASK_WOKEN_IO); return; }