From fd48b28315a923a6941ced201a2b6c58f426c427 Mon Sep 17 00:00:00 2001 From: Nathan Wehrman Date: Tue, 13 Aug 2024 11:25:38 -0700 Subject: [PATCH] MINOR: Implements new log format of option tcplog clf Some systems require log formats in the CLF format and that meant that I could not send my logs for proxies in mode tcp to those servers. This implements a format that uses log variables that are compatble with TCP mode frontends and replaces traditional HTTP values in the CLF format to make them stand out. Instead of logging method and URI like this "GET /example HTTP/1.1" it will log "TCP " and for a response code I used "000" so it would be easy to separate from legitimate HTTP traffic. Now your log servers that require a CLF format can see the timings for TCP traffic as well as HTTP. --- doc/configuration.txt | 19 +++++++++++++++++-- include/haproxy/log.h | 1 + src/cfgparse-listen.c | 32 +++++++++++++++++++++++++++----- src/log.c | 2 ++ src/proxy.c | 10 ++++++++-- 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/doc/configuration.txt b/doc/configuration.txt index 05e8d22ca1..aece65c810 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -10821,7 +10821,7 @@ option tcpka See also : "option clitcpka", "option srvtcpka" -option tcplog +option tcplog [clf] Enable advanced logging of TCP connections with stream state and timers May be used in the following contexts: tcp, http @@ -10829,7 +10829,14 @@ option tcplog May be used in sections : defaults | frontend | listen | backend yes | yes | yes | no - Arguments : none + Arguments : + clf if the "clf" argument is added, then the output format will be + the CLF format instead of HAProxy's default TCP format. You can + use this when you need to feed HAProxy's logs through a specific + log analyzer which only support the CLF format and which is not + extensible. Since this expects an HTTP format some of the + values have been pre set. The http request will show as TCP and + the response code will show as 000. By default, the log output format is very poor, as it only contains the source and destination addresses, and the instance name. By specifying @@ -25553,6 +25560,14 @@ Refer to section 8.2.6 "Custom log format" to see how to use this: # or using the HAPROXY_TCP_LOG_FMT variable log-format "${HAPROXY_TCP_LOG_FMT}" +And the CLF log format is internally declared as a custom log format based on +this exact string: + + # strict equivalent of "option tcplog clf" + log-format "%{Q}o %{-Q}ci - - [%T] \"TCP \" 000 %B \"\" \"\" %cp \ + %ms %ft %b %s %Th %Tw %Tc %Tt %U %ts-- %ac %fc %bc \ + %sc %rc %sq %bq \"\" \"\" " + A few fields may slightly vary depending on some configuration options, those are marked with a star ('*') after the field name below. diff --git a/include/haproxy/log.h b/include/haproxy/log.h index 10f50a523b..5a23ec2452 100644 --- a/include/haproxy/log.h +++ b/include/haproxy/log.h @@ -37,6 +37,7 @@ extern const char *log_levels[]; extern char *log_format; extern char httpclient_log_format[]; extern char default_tcp_log_format[]; +extern char clf_tcp_log_format[]; extern char default_http_log_format[]; extern char clf_http_log_format[]; extern char default_https_log_format[]; diff --git a/src/cfgparse-listen.c b/src/cfgparse-listen.c index f710b02799..2cab441095 100644 --- a/src/cfgparse-listen.c +++ b/src/cfgparse-listen.c @@ -2103,6 +2103,7 @@ stats_error_parsing: if (strcmp(args[1], "httplog") == 0) { char *logformat; + /* generate a complete HTTP log */ logformat = default_http_log_format; if (*(args[2]) != '\0') { @@ -2125,6 +2126,8 @@ stats_error_parsing: oldlogformat = "option httplog"; else if (curproxy->logformat.str == default_tcp_log_format) oldlogformat = "option tcplog"; + else if (curproxy->logformat.str == clf_tcp_log_format) + oldlogformat = "option tcplog clf"; else if (curproxy->logformat.str == clf_http_log_format) oldlogformat = "option httplog clf"; else if (curproxy->logformat.str == default_https_log_format) @@ -2146,28 +2149,46 @@ stats_error_parsing: } } else if (strcmp(args[1], "tcplog") == 0) { + char *logformat; + + /* generate a detailed TCP log */ + logformat = default_tcp_log_format; + if (*(args[2]) != '\0') { + if (strcmp(args[2], "clf") == 0) { + logformat = clf_tcp_log_format; + } else { + ha_alert("parsing [%s:%d] : keyword '%s' only supports option 'clf'.\n", file, linenum, args[1]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + if (alertif_too_many_args_idx(1, 1, file, linenum, args, &err_code)) + goto out; + } if (curproxy->logformat.str && curproxy->cap & PR_CAP_DEF) { char *oldlogformat = "log-format"; + char *clflogformat = ""; if (curproxy->logformat.str == default_http_log_format) oldlogformat = "option httplog"; else if (curproxy->logformat.str == default_tcp_log_format) oldlogformat = "option tcplog"; + else if (curproxy->logformat.str == clf_tcp_log_format) + oldlogformat = "option tcplog clf"; else if (curproxy->logformat.str == clf_http_log_format) oldlogformat = "option httplog clf"; else if (curproxy->logformat.str == default_https_log_format) oldlogformat = "option httpslog"; - ha_warning("parsing [%s:%d]: 'option tcplog' overrides previous '%s' in 'defaults' section.\n", - file, linenum, oldlogformat); + if (logformat == clf_tcp_log_format) + clflogformat = " clf"; + ha_warning("parsing [%s:%d]: 'option tcplog%s' overrides previous '%s' in 'defaults' section.\n", + file, linenum, clflogformat, oldlogformat); } /* generate a detailed TCP log */ lf_expr_deinit(&curproxy->logformat); - curproxy->logformat.str = default_tcp_log_format; + curproxy->logformat.str = logformat; curproxy->logformat.conf.file = strdup(curproxy->conf.args.file); curproxy->logformat.conf.line = curproxy->conf.args.line; - if (alertif_too_many_args_idx(0, 1, file, linenum, args, &err_code)) - goto out; if (!(curproxy->cap & PR_CAP_DEF) && !(curproxy->cap & PR_CAP_FE)) { ha_warning("parsing [%s:%d] : backend '%s' : 'option tcplog' directive is ignored in backends.\n", @@ -2177,6 +2198,7 @@ stats_error_parsing: } else if (strcmp(args[1], "httpslog") == 0) { char *logformat; + /* generate a complete HTTP log */ logformat = default_https_log_format; if (curproxy->logformat.str && curproxy->cap & PR_CAP_DEF) { diff --git a/src/log.c b/src/log.c index d338e1ec16..4b9573cdb7 100644 --- a/src/log.c +++ b/src/log.c @@ -294,6 +294,7 @@ char default_http_log_format[] = "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %S char default_https_log_format[] = "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r %[fc_err]/%[ssl_fc_err,hex]/%[ssl_c_err]/%[ssl_c_ca_err]/%[ssl_fc_is_resumed] %[ssl_fc_sni]/%sslv/%sslc"; char clf_http_log_format[] = "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B \"\" \"\" %cp %ms %ft %b %s %TR %Tw %Tc %Tr %Ta %tsc %ac %fc %bc %sc %rc %sq %bq %CC %CS %hrl %hsl"; char default_tcp_log_format[] = "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"; +char clf_tcp_log_format[] = "%{+Q}o %{-Q}ci - - [%T] \"TCP \" 000 %B \"\" \"\" %cp %ms %ft %b %s %Th %Tw %Tc %Tt %U %ts-- %ac %fc %bc %sc %rc %sq %bq \"\" \"\" "; char *log_format = NULL; /* Default string used for structured-data part in RFC5424 formatted @@ -311,6 +312,7 @@ static inline int logformat_str_isdefault(const char *str) str == default_https_log_format || str == clf_http_log_format || str == default_tcp_log_format || + str == clf_tcp_log_format || str == default_rfc5424_sd_log_format; } diff --git a/src/proxy.c b/src/proxy.c index b86e4f5b4f..25dd136dad 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -1328,14 +1328,20 @@ int proxy_cfg_ensure_no_http(struct proxy *curproxy) ha_warning("Layer 7 hash not possible for %s '%s' (needs 'mode http'). Falling back to round robin.\n", proxy_type_str(curproxy), curproxy->id); } - if (curproxy->logformat.str == default_http_log_format || - curproxy->logformat.str == clf_http_log_format) { + if (curproxy->logformat.str == default_http_log_format) { /* Note: we don't change the directive's file:line number */ curproxy->logformat.str = default_tcp_log_format; ha_warning("parsing [%s:%d] : 'option httplog' not usable with %s '%s' (needs 'mode http'). Falling back to 'option tcplog'.\n", curproxy->logformat.conf.file, curproxy->logformat.conf.line, proxy_type_str(curproxy), curproxy->id); } + else if (curproxy->logformat.str == clf_http_log_format) { + /* Note: we don't change the directive's file:line number */ + curproxy->logformat.str = clf_tcp_log_format; + ha_warning("parsing [%s:%d] : 'option httplog clf' not usable with %s '%s' (needs 'mode http'). Falling back to 'option tcplog clf'.\n", + curproxy->logformat.conf.file, curproxy->logformat.conf.line, + proxy_type_str(curproxy), curproxy->id); + } else if (curproxy->logformat.str == default_https_log_format) { /* Note: we don't change the directive's file:line number */ curproxy->logformat.str = default_tcp_log_format; -- 2.39.5