MEDIUM: log: add the ability to include samples in logs

Using %[expression] it becomes possible to make the log engine fetch
some samples from the request or the response and provide them in the
logs. Note that this feature is still limited, it does not yet allow
to apply converters, to limit the output length, nor to specify the
direction which should be fetched when a fetch function works in both
directions.

However it's quite convenient to log SSL information or to include some
information that are used in stick tables.

It is worth noting that this has been done in the generic log format
handler, which means that the same information may be used to build the
unique-id header and to pass the information to a backend server.

diff --git a/doc/configuration.txt b/doc/configuration.txt
index b1c20e3cbcf88428782db03e1acd20876d02de20..f1a388e5f3c661648c1cce1bae90ecc386d5aab7 100644 (file)
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -10235,6 +10235,11 @@ Special variable "%o" may be used to propagate its flags to all other
 variables on the same format string. This is particularly handy with quoted
 string formats ("Q").
 
+If a variable is named between square brackets ('[' .. ']') then it is used
+as a pattern extraction rule (see section 7.8). This it useful to add some
+less common information such as the client's SSL certificate's DN, or to log
+the key that would be used to store an entry into a stick table.
+
 Note: spaces must be escaped. A space character is considered as a separator.
 HAproxy will automatically merge consecutive separators.
 

diff --git a/include/types/log.h b/include/types/log.h
index 0ed79d41421739a779aeadf12880adfcf4dfb820..2a3a14791e517bcff4788eb872043a9f1fceccc2 100644 (file)
--- a/include/types/log.h
+++ b/include/types/log.h
@@ -39,7 +39,7 @@
 enum {
 
        LOG_FMT_TEXT = 0, /* raw text */
-
+       LOG_FMT_EXPR,     /* sample expression */
        LOG_FMT_SEPARATOR, /* separator replaced by one space */
        LOG_FMT_VARIABLE,
 
@@ -106,21 +106,25 @@ enum {
        LF_STARTVAR,   // % in text
        LF_STARG,      // after '%{' and berore '}'
        LF_EDARG,      // '}' after '%{'
+       LF_STEXPR,     // after '%[' or '%{..}[' and berore ']'
+       LF_EDEXPR,     // ']' after '%['
        LF_END,        // \0 found
 };
 
 
 struct logformat_node {
        struct list list;
-       int type;
-       int options;
-       char *arg;
+       int type;      // LOG_FMT_*
+       int options;   // LOG_OPT_*
+       char *arg;     // text for LOG_FMT_TEXT, arg for others
+       void *expr;    // for use with LOG_FMT_EXPR
 };
 
 #define LOG_OPT_HEXA           0x00000001
 #define LOG_OPT_MANDATORY      0x00000002
 #define LOG_OPT_QUOTE          0x00000004
-
+#define LOG_OPT_REQ_CAP         0x00000008
+#define LOG_OPT_RES_CAP         0x00000010
 
 
 /* fields that need to be logged. They appear as flags in session->logs.logwait */

diff --git a/src/log.c b/src/log.c
index a1bc78fdc0a53cf59f2efabd0c04450dcadaa971..ae7d12217a001d261b9f8e32acc2601ff47053ba 100644 (file)
--- a/src/log.c
+++ b/src/log.c
@@ -33,6 +33,7 @@
 
 #include <proto/frontend.h>
 #include <proto/log.h>
+#include <proto/sample.h>
 #include <proto/stream_interface.h>
 #ifdef USE_OPENSSL
 #include <proto/ssl_sock.h>
@@ -303,6 +304,51 @@ void add_to_logformat_list(char *start, char *end, int type, struct list *list_f
        }
 }
 
+/*
+ * Parse the sample fetch expression <text> and add a node to <list_format> upon
+ * success. At the moment, sample converters are not yet supported but fetch arguments
+ * should work.
+ */
+void add_sample_to_logformat_list(char *text, char *arg, int arg_len, struct proxy *curpx, struct list *list_format, int options)
+{
+       char *cmd[2];
+       struct sample_expr *expr;
+       struct logformat_node *node;
+       int cmd_arg;
+
+       cmd[0] = text;
+       cmd[1] = "";
+       cmd_arg = 0;
+
+       expr = sample_parse_expr(cmd, &cmd_arg, trash.str, trash.size);
+       if (!expr) {
+               Warning("log-format: sample fetch <%s> failed with : %s\n", text, trash.str);
+               return;
+       }
+
+       node = calloc(1, sizeof(struct logformat_node));
+       node->type = LOG_FMT_EXPR;
+       node->expr = expr;
+       node->options = options;
+
+       if (arg_len) {
+               node->arg = my_strndup(arg, arg_len);
+               parse_logformat_var_args(node->arg, node);
+       }
+       if (expr->fetch->cap & SMP_CAP_REQ)
+               node->options |= LOG_OPT_REQ_CAP; /* fetch method is request-compatible */
+
+       if (expr->fetch->cap & SMP_CAP_RES)
+               node->options |= LOG_OPT_RES_CAP; /* fetch method is response-compatible */
+
+       /* check if we need to allocate an hdr_idx struct for HTTP parsing */
+       /* Note, we may also need to set curpx->to_log with certain fetches */
+       if (expr->fetch->cap & SMP_CAP_L7)
+               curpx->acl_requires |= ACL_USE_L7_ANY;
+
+       LIST_ADDQ(list_format, &node->list);
+}
+
 /*
  * Parse the log_format string and fill a linked list.
  * Variable name are preceded by % and composed by characters [a-zA-Z0-9]* : %varname
@@ -346,12 +392,16 @@ void parse_logformat_string(char *str, struct proxy *curproxy, struct list *list
                 */
                switch (pformat) {
                case LF_STARTVAR:                      // text immediately following a '%'
-                       arg = NULL;
+                       arg = NULL; var = NULL;
                        arg_len = var_len = 0;
                        if (*str == '{') {             // optional argument
                                cformat = LF_STARG;
                                arg = str + 1;
                        }
+                       else if (*str == '[') {
+                               cformat = LF_STEXPR;
+                               var = str + 1;         // store expr in variable name
+                       }
                        else if (isalnum((int)*str)) { // variable name
                                cformat = LF_VAR;
                                var = str;
@@ -371,23 +421,35 @@ void parse_logformat_string(char *str, struct proxy *curproxy, struct list *list
                        break;
 
                case LF_EDARG:                         // text immediately following '%{arg}'
-                       if (isalnum((int)*str)) {      // variable name
+                       if (*str == '[') {
+                               cformat = LF_STEXPR;
+                               var = str + 1;         // store expr in variable name
+                               break;
+                       }
+                       else if (isalnum((int)*str)) { // variable name
                                cformat = LF_VAR;
                                var = str;
                                break;
                        }
-
                        Warning("Skipping isolated argument in log-format line : '%%{%s}'\n", arg);
                        cformat = LF_INIT;
                        break;
 
+               case LF_STEXPR:                        // text immediately following '%['
+                       if (*str == ']') {             // end of arg
+                               cformat = LF_EDEXPR;
+                               var_len = str - var;
+                               *str = 0;              // needed for parsing the expression
+                       }
+                       break;
+
                case LF_VAR:                           // text part of a variable name
                        var_len = str - var;
                        if (!isalnum((int)*str))
                                cformat = LF_INIT;     // not variable name anymore
                        break;
 
-               default:                               // LF_INIT, LF_TEXT, LF_SEPARATOR, LF_END
+               default:                               // LF_INIT, LF_TEXT, LF_SEPARATOR, LF_END, LF_EDEXPR
                        cformat = LF_INIT;
                }
 
@@ -405,6 +467,9 @@ void parse_logformat_string(char *str, struct proxy *curproxy, struct list *list
                        case LF_VAR:
                                parse_logformat_var(arg, arg_len, var, var_len, curproxy, list_format, &options);
                                break;
+                       case LF_STEXPR:
+                               add_sample_to_logformat_list(var, arg, arg_len, curproxy, list_format, options);
+                               break;
                        case LF_TEXT:
                        case LF_SEPARATOR:
                                add_to_logformat_list(sp, str, pformat, list_format);
@@ -414,8 +479,8 @@ void parse_logformat_string(char *str, struct proxy *curproxy, struct list *list
                }
        }
 
-       if (pformat == LF_STARTVAR || pformat == LF_STARG)
-               Warning("Ignoring end of truncated log-format line after '%s'\n", arg ? arg : "%");
+       if (pformat == LF_STARTVAR || pformat == LF_STARG || pformat == LF_STEXPR)
+               Warning("Ignoring end of truncated log-format line after '%s'\n", var ? var : arg ? arg : "%");
 }
 
 /*
@@ -823,8 +888,9 @@ int build_logline(struct session *s, char *dst, size_t maxsize, struct list *lis
 
        list_for_each_entry(tmp, list_format, list) {
                const char *src = NULL;
-               switch (tmp->type) {
+               struct sample *key;
 
+               switch (tmp->type) {
                        case LOG_FMT_SEPARATOR:
                                if (!last_isspace) {
                                        LOGCHAR(' ');
@@ -841,6 +907,21 @@ int build_logline(struct session *s, char *dst, size_t maxsize, struct list *lis
                                last_isspace = 0;
                                break;
 
+                       case LOG_FMT_EXPR: // sample expression, may be request or response
+                               key = NULL;
+                               if (tmp->options & LOG_OPT_REQ_CAP)
+                                       key = sample_fetch_string(be, s, txn, SMP_OPT_DIR_REQ|SMP_OPT_FINAL, tmp->expr);
+                               if (!key && (tmp->options & LOG_OPT_RES_CAP))
+                                       key = sample_fetch_string(be, s, txn, SMP_OPT_DIR_RES|SMP_OPT_FINAL, tmp->expr);
+                               if (!key)
+                                       break;
+                               ret = lf_text_len(tmplog, key->data.str.str, key->data.str.len, dst + maxsize - tmplog, tmp);
+                               if (ret == 0)
+                                       goto out;
+                               tmplog = ret;
+                               last_isspace = 0;
+                               break;
+
                        case LOG_FMT_CLIENTIP:  // %ci
                                ret = lf_ip(tmplog, (struct sockaddr *)&s->req->prod->conn->addr.from,
                                            dst + maxsize - tmplog, tmp);