From 723b73ad75b618467a3829e17d08290395c91f2f Mon Sep 17 00:00:00 2001 From: William Lallemand Date: Wed, 8 Feb 2012 16:37:49 +0100 Subject: [PATCH] MINOR: config: Parse the string of the log-format config keyword parse_logformat_string: parse the string, detect the type: text, separator or variable parse_logformat_var: dectect variable name parse_logformat_var_args: parse arguments and flags add_to_logformat_list: add to the logformat linked list --- include/proto/log.h | 27 ++++ include/types/log.h | 69 ++++++++++ include/types/proxy.h | 1 + src/cfgparse.c | 38 ++++-- src/haproxy.c | 6 + src/log.c | 294 ++++++++++++++++++++++++++++++++++++++++++ src/proxy.c | 1 + 7 files changed, 428 insertions(+), 8 deletions(-) diff --git a/include/proto/log.h b/include/proto/log.h index fabd40011c..2cf114b7db 100644 --- a/include/proto/log.h +++ b/include/proto/log.h @@ -34,6 +34,33 @@ extern struct pool_head *pool2_requri; +extern char *log_format; +extern char default_http_log_format[]; +extern char clf_http_log_format[]; + +/* + * Parse args in a logformat_var + */ +int parse_logformat_var_args(char *args, struct logformat_node *node); + +/* + * Parse a variable '%varname' or '%{args}varname' in logformat + * + */ +int parse_logformat_var(char *str, size_t len, struct proxy *curproxy); + +/* + * add to the logformat linked list + */ +void add_to_logformat_list(char *start, char *end, int type, struct proxy *curproxy); + +/* + * Parse the log_format string and fill a linked list. + * Variable name are preceded by % and composed by characters [a-zA-Z0-9]* : %varname + * You can set arguments using { } : %{many arguments}varname + */ +void parse_logformat_string(char *str, struct proxy *curproxy); + /* * Displays the message on stderr with the date and pid. Overrides the quiet * mode during startup. diff --git a/include/types/log.h b/include/types/log.h index 9b700b0e08..0e78024116 100644 --- a/include/types/log.h +++ b/include/types/log.h @@ -32,6 +32,75 @@ #define NB_LOG_LEVELS 8 #define SYSLOG_PORT 514 +/* lists of fields that can be logged */ +enum { + + LOG_TEXT = 0, /* raw text */ + + LOG_SEPARATOR, /* separator replaced by one space */ + LOG_VARIABLE, + + /* information fields */ + LOG_GLOBAL, + LOG_CLIENTIP, + LOG_CLIENTPORT, + LOG_DATE, + LOG_DATEGMT, + LOG_MS, + LOG_FRONTEND, + LOG_BACKEND, + LOG_SERVER, + LOG_BYTES, + LOG_T, + LOG_TQ, + LOG_TW, + LOG_TC, + LOG_TR, + LOG_TT, + LOG_STATUS, + LOG_CCLIENT, + LOG_CSERVER, + LOG_TERMSTATE, + LOG_CONN, + LOG_ACTCONN, + LOG_FECONN, + LOG_BECONN, + LOG_SRVCONN, + LOG_RETRIES, + LOG_QUEUES, + LOG_SRVQUEUE, + LOG_BCKQUEUE, + LOG_HDRREQUEST, + LOG_HDRRESPONS, + LOG_HDRREQUESTLIST, + LOG_HDRRESPONSLIST, + LOG_REQ, +}; + +/* enum for parse_logformat */ +enum { + LF_TEXT = 0, + LF_SEPARATOR, + LF_VAR, // after % + + LF_STARTVAR, // % + LF_STARG, // { and within { } + LF_EDARG, // end arg } +}; + + +struct logformat_node { + struct list list; + int type; + int options; + char *arg; +}; + +#define LOG_OPT_WRITTEN 0x00000001 +#define LOG_OPT_MANDATORY 0x00000002 +#define LOG_OPT_QUOTE 0x00000004 + + /* fields that need to be logged. They appear as flags in session->logs.logwait */ #define LW_DATE 1 /* date */ diff --git a/include/types/proxy.h b/include/types/proxy.h index d0bc51cafd..6693d8288d 100644 --- a/include/types/proxy.h +++ b/include/types/proxy.h @@ -286,6 +286,7 @@ struct proxy { int (*accept)(struct session *s); /* application layer's accept() */ struct proxy *next; struct list logsrvs; + struct list logformat; /* log_format linked list */ int to_log; /* things to be logged (LW_*) */ int stop_time; /* date to stop listening, when stopping != 0 (int ticks) */ struct hdr_exp *req_exp; /* regular expressions for request headers */ diff --git a/src/cfgparse.c b/src/cfgparse.c index 73b7f7135c..815ffff7fa 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -1323,7 +1323,8 @@ int cfg_parse_listen(const char *file, int linenum, char **args, int kwm) unsigned val; int err_code = 0; struct acl_cond *cond = NULL; - struct logsrv *tmp; + struct logsrv *tmplogsrv; + struct logformat_node *tmplf; if (!strcmp(args[0], "listen")) rc = PR_CAP_LISTEN; @@ -1533,13 +1534,21 @@ int cfg_parse_listen(const char *file, int linenum, char **args, int kwm) curproxy->mode = defproxy.mode; /* copy default logsrvs to curproxy */ - list_for_each_entry(tmp, &defproxy.logsrvs, list) { + list_for_each_entry(tmplogsrv, &defproxy.logsrvs, list) { struct logsrv *node = malloc(sizeof(struct logsrv)); - memcpy(node, tmp, sizeof(struct logsrv)); + memcpy(node, tmplogsrv, sizeof(struct logsrv)); LIST_INIT(&node->list); LIST_ADDQ(&curproxy->logsrvs, &node->list); } + /* copy default log_format to curproxy */ + list_for_each_entry(tmplf, &defproxy.logformat, list) { + struct logformat_node *node = malloc(sizeof(struct logformat_node)); + memcpy(node, tmplf, sizeof(struct logformat_node)); + LIST_INIT(&node->list); + LIST_ADDQ(&curproxy->logformat, &node->list); + } + curproxy->grace = defproxy.grace; curproxy->conf.used_listener_id = EB_ROOT; curproxy->conf.used_server_id = EB_ROOT; @@ -3286,18 +3295,22 @@ stats_error_parsing: } if (!strcmp(args[1], "httplog")) { + char *logformat; /* generate a complete HTTP log */ curproxy->options2 &= ~PR_O2_CLFLOG; curproxy->to_log |= LW_DATE | LW_CLIP | LW_SVID | LW_REQ | LW_PXID | LW_RESP | LW_BYTES; + logformat = default_http_log_format; if (*(args[2]) != '\0') { if (!strcmp(args[2], "clf")) { curproxy->options2 |= PR_O2_CLFLOG; + logformat = clf_http_log_format; } else { Alert("parsing [%s:%d] : keyword '%s' only supports option 'clf'.\n", file, linenum, args[2]); err_code |= ERR_ALERT | ERR_FATAL; goto out; } } + parse_logformat_string(logformat, curproxy); } else if (!strcmp(args[1], "tcplog")) /* generate a detailed TCP log */ @@ -4533,6 +4546,15 @@ stats_error_parsing: newsrv->prev_state = newsrv->state; } } + else if (strcmp(args[0], "log-format") == 0) { + if (!*(args[1])) { + Alert("parsing [%s:%d] : %s expects an argument.\n", file, linenum, args[0]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + parse_logformat_string(args[1], curproxy); + } + else if (!strcmp(args[0], "log") && kwm == KWM_NO) { /* delete previous herited or defined syslog servers */ struct logsrv *back; @@ -4543,9 +4565,9 @@ stats_error_parsing: goto out; } - list_for_each_entry_safe(tmp, back, &curproxy->logsrvs, list) { - LIST_DEL(&tmp->list); - free(tmp); + list_for_each_entry_safe(tmplogsrv, back, &curproxy->logsrvs, list) { + LIST_DEL(&tmplogsrv->list); + free(tmplogsrv); } } else if (!strcmp(args[0], "log")) { /* syslog server address */ @@ -4553,9 +4575,9 @@ stats_error_parsing: if (*(args[1]) && *(args[2]) == 0 && !strcmp(args[1], "global")) { /* copy global.logrsvs linked list to the end of curproxy->logsrvs */ - list_for_each_entry(tmp, &global.logsrvs, list) { + list_for_each_entry(tmplogsrv, &global.logsrvs, list) { struct logsrv *node = malloc(sizeof(struct logsrv)); - memcpy(node, tmp, sizeof(struct logsrv)); + memcpy(node, tmplogsrv, sizeof(struct logsrv)); LIST_INIT(&node->list); LIST_ADDQ(&curproxy->logsrvs, &node->list); } diff --git a/src/haproxy.c b/src/haproxy.c index 66cf18f594..f14081a482 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -802,6 +802,7 @@ void deinit(void) struct cond_wordlist *cwl, *cwlb; struct uri_auth *uap, *ua = NULL; struct logsrv *log, *logb; + struct logformat_node *lf, *lfb; int i; deinit_signals(); @@ -912,6 +913,11 @@ void deinit(void) free(log); } + list_for_each_entry_safe(lf, lfb, &p->logformat, list) { + LIST_DEL(&lf->list); + free(lf); + } + deinit_tcp_rules(&p->tcp_req.inspect_rules); deinit_tcp_rules(&p->tcp_req.l4_rules); diff --git a/src/log.c b/src/log.c index 7c0e14873e..b80969ff1d 100644 --- a/src/log.c +++ b/src/log.c @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -55,6 +56,299 @@ const char *monthname[12] = { const char sess_term_cond[10] = "-cCsSPRIDK"; /* normal, CliTo, CliErr, SrvTo, SrvErr, PxErr, Resource, Internal, Down, Killed */ const char sess_fin_state[8] = "-RCHDLQT"; /* cliRequest, srvConnect, srvHeader, Data, Last, Queue, Tarpit */ + +/* log_format */ +struct logformat_type { + char *name; + int type; +}; + +/* log_format variable names */ +static const struct logformat_type logformat_keywords[] = { + { "o", LOG_GLOBAL }, /* global option */ + { "Ci", LOG_CLIENTIP }, /* client ip */ + { "Cp", LOG_CLIENTPORT }, /* client port */ + { "t", LOG_DATE }, /* date */ + { "T", LOG_DATEGMT }, /* date GMT */ + { "ms", LOG_MS }, /* accept date millisecond */ + { "f", LOG_FRONTEND }, /* frontend */ + { "b", LOG_BACKEND }, /* backend */ + { "s", LOG_SERVER }, /* server */ + { "B", LOG_BYTES }, /* bytes read */ + { "Tq", LOG_TQ }, /* Tq */ + { "Tw", LOG_TW }, /* Tw */ + { "Tc", LOG_TC }, /* Tc */ + { "Tr", LOG_TR }, /* Tr */ + { "Tt", LOG_TT }, /* Tt */ + { "st", LOG_STATUS }, /* status code */ + { "cc", LOG_CCLIENT }, /* client cookie */ + { "cs", LOG_CSERVER }, /* server cookie */ + { "ts", LOG_TERMSTATE },/* terminaison state */ + { "ac", LOG_ACTCONN }, /* actconn */ + { "fc", LOG_FECONN }, /* feconn */ + { "bc", LOG_BECONN }, /* beconn */ + { "sc", LOG_SRVCONN }, /* srv_conn */ + { "rc", LOG_RETRIES }, /* retries */ + { "sq", LOG_SRVQUEUE }, /* srv_queue */ + { "bq", LOG_BCKQUEUE }, /* backend_queue */ + { "hr", LOG_HDRREQUEST }, /* header request */ + { "hs", LOG_HDRRESPONS }, /* header response */ + { "hrl", LOG_HDRREQUESTLIST }, /* header request list */ + { "hsl", LOG_HDRRESPONSLIST }, /* header response list */ + { "r", LOG_REQ }, /* request */ + { 0, 0 } +}; + +char default_http_log_format[] = "%Ci:%Cp [%t] %f %b/%s %Tq/%Tw/%Tc/%Tr/%Tt %st %B %cc %cs %ts %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r"; // default format +char clf_http_log_format[] = "%{+Q}o %{-Q}Ci - - [%T] %r %st %B \"\" \"\" %Cp %ms %f %b %s %Tq %Tw %Tc %Tr %Tt %ts %ac %fc %bc %sc %rc %sq %bq %cc %cs %hrl %hsl"; +char *log_format = NULL; + +struct logformat_var_args { + char *name; + int mask; +}; + +struct logformat_var_args var_args_list[] = { +// global + { "M", LOG_OPT_MANDATORY }, + { "Q", LOG_OPT_QUOTE }, + { 0, 0 } +}; + +/* + * Parse args in a logformat_var + */ +int parse_logformat_var_args(char *args, struct logformat_node *node) +{ + int i = 0; + int end = 0; + int flags = 0; // 1 = + 2 = - + char *sp = NULL; // start pointer + + if (args == NULL) + return 1; + + while (1) { + if (*args == '\0') + end = 1; + + if (*args == '+') { + // add flag + sp = args + 1; + flags = 1; + } + if (*args == '-') { + // delete flag + sp = args + 1; + flags = 2; + } + + if (*args == '\0' || *args == ',') { + *args = '\0'; + for (i = 0; var_args_list[i].name; i++) { + if (strcmp(sp, var_args_list[i].name) == 0) { + if (flags == 1) { + node->options |= var_args_list[i].mask; + break; + } else if (flags == 2) { + node->options &= ~var_args_list[i].mask; + break; + } + } + } + sp = NULL; + if (end) + break; + } + args++; + } + return 0; +} + +/* + * Parse a variable '%varname' or '%{args}varname' in logformat + * + */ +int parse_logformat_var(char *str, size_t len, struct proxy *curproxy) +{ + int i, j; + char *arg = NULL; // arguments + int fparam = 0; + char *name = NULL; + struct logformat_node *node = NULL; + char varname[255] = { 0 }; // variable name + int logformat_options = 0x00000000; + + + for (i = 1; i < len; i++) { // escape first char % + if (!arg && str[i] == '{') { + arg = str + i; + fparam = 1; + } else if (arg && str[i] == '}') { + char *tmp = arg; + arg = calloc(str + i - tmp, 1); // without {} + strncpy(arg, tmp + 1, str + i - tmp - 1); // copy without { and } + arg[str + i - tmp - 1] = '\0'; + fparam = 0; + } else if (!name && !fparam) { + strncpy(varname, str + i, len - i + 1); + varname[len - i] = '\0'; + for (j = 0; logformat_keywords[j].name; j++) { // search a log type + if (strcmp(varname, logformat_keywords[j].name) == 0) { + node = calloc(1, sizeof(struct logformat_node)); + node->type = logformat_keywords[j].type; + node->options = logformat_options; + node->arg = arg; + parse_logformat_var_args(node->arg, node); + if (node->type == LOG_GLOBAL) { + logformat_options = node->options; + free(node); + } else { + LIST_ADDQ(&curproxy->logformat, &node->list); + } + return 0; + } + } + Warning("Warning: No such variable name '%s' in logformat\n", varname); + if (arg) + free(arg); + return -1; + } + } + return -1; +} + +/* + * push to the logformat linked list + * + * start: start pointer + * end: end text pointer + * type: string type + * + * LOG_TEXT: copy chars from start to end excluding end. + * +*/ +void add_to_logformat_list(char *start, char *end, int type, struct proxy *curproxy) +{ + char *str; + + if (type == LOG_TEXT) { /* type text */ + struct logformat_node *node = calloc(1, sizeof(struct logformat_node)); + + str = calloc(end - start + 1, 1); + strncpy(str, start, end - start); + + str[end - start] = '\0'; + node->arg = str; + node->type = LOG_TEXT; // type string + LIST_ADDQ(&curproxy->logformat, &node->list); + } else if (type == LOG_VARIABLE) { /* type variable */ + parse_logformat_var(start, end - start, curproxy); + } else if (type == LOG_SEPARATOR) { + struct logformat_node *node = calloc(1, sizeof(struct logformat_node)); + node->type = LOG_SEPARATOR; + LIST_ADDQ(&curproxy->logformat, &node->list); + } +} + +/* + * Parse the log_format string and fill a linked list. + * Variable name are preceded by % and composed by characters [a-zA-Z0-9]* : %varname + * You can set arguments using { } : %{many arguments}varname + */ +void parse_logformat_string(char *str, struct proxy *curproxy) +{ + char *sp = str; /* start pointer */ + int cformat = -1; /* current token format : LOG_TEXT, LOG_SEPARATOR, LOG_VARIABLE */ + int pformat = -1; /* previous token format */ + struct logformat_node *tmplf, *back; + + /* flush the list first. */ + list_for_each_entry_safe(tmplf, back, &curproxy->logformat, list) { + LIST_DEL(&tmplf->list); + free(tmplf); + } + + while (1) { + + // push the variable only if formats are different, not + // within a variable, and not the first iteration + if ((cformat != pformat && cformat != -1 && pformat != -1) || *str == '\0') { + if (((pformat != LF_STARTVAR && cformat != LF_VAR) && + (pformat != LF_STARTVAR && cformat != LF_STARG) && + (pformat != LF_STARG && cformat != LF_VAR)) || *str == '\0') { + if (pformat > LF_VAR) // unfinished string + pformat = LF_TEXT; + add_to_logformat_list(sp, str, pformat, curproxy); + sp = str; + if (*str == '\0') + break; + } + } + + if (cformat != -1) + str++; // consume the string, except on the first tour + + pformat = cformat; + + if (*str == '\0') { + cformat = LF_STARTVAR; // for breaking in all cases + continue; + } + + if (pformat == LF_STARTVAR) { // after a % + if ( (*str >= 'a' && *str <= 'z') || // parse varname + (*str >= 'A' && *str <= 'Z') || + (*str >= '0' && *str <= '9')) { + cformat = LF_VAR; // varname + continue; + } else if (*str == '{') { + cformat = LF_STARG; // variable arguments + continue; + } else { // another unexpected token + pformat = LF_TEXT; // redefine the format of the previous token to TEXT + cformat = LF_TEXT; + continue; + } + + } else if (pformat == LF_VAR) { // after a varname + if ( (*str >= 'a' && *str <= 'z') || // parse varname + (*str >= 'A' && *str <= 'Z') || + (*str >= '0' && *str <= '9')) { + cformat = LF_VAR; + continue; + } + } else if (pformat == LF_STARG) { // inside variable arguments + if (*str == '}') { // end of varname + cformat = LF_EDARG; + continue; + } else { // all tokens are acceptable within { } + cformat = LF_STARG; + continue; + } + } else if (pformat == LF_EDARG) { // after arguments + if ( (*str >= 'a' && *str <= 'z') || // parse a varname + (*str >= 'A' && *str <= 'Z') || + (*str >= '0' && *str <= '9')) { + cformat = LF_VAR; + continue; + } else { // if no varname after arguments, transform in TEXT + pformat = LF_TEXT; + cformat = LF_TEXT; + } + } + + // others tokens that don't match previous conditions + if (*str == '%') { + cformat = LF_STARTVAR; + } else if (*str == ' ') { + cformat = LF_SEPARATOR; + } else { + cformat = LF_TEXT; + } + } +} + /* * Displays the message on stderr with the date and pid. Overrides the quiet * mode during startup. diff --git a/src/proxy.c b/src/proxy.c index 28094a83d4..123fd61768 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -437,6 +437,7 @@ void init_new_proxy(struct proxy *p) LIST_INIT(&p->rsp_add); LIST_INIT(&p->listener_queue); LIST_INIT(&p->logsrvs); + LIST_INIT(&p->logformat); /* Timeouts are defined as -1 */ proxy_reset_timeouts(p); -- 2.39.5