]> git.ipfire.org Git - thirdparty/haproxy.git/commitdiff
MEDIUM: proto_tcp: add support for tracking L7 information
authorWilly Tarreau <w@1wt.eu>
Sun, 9 Dec 2012 11:00:04 +0000 (12:00 +0100)
committerWilly Tarreau <w@1wt.eu>
Sun, 9 Dec 2012 13:08:47 +0000 (14:08 +0100)
Until now it was only possible to use track-sc1/sc2 with "src" which
is the IPv4 source address. Now we can use track-sc1/sc2 with any fetch
as well as any transformation type. It works just like the "stick"
directive.

Samples are automatically converted to the correct types for the table.

Only "tcp-request content" rules may use L7 information, and such information
must already be present when the tracking is set up. For example it becomes
possible to track the IP address passed in the X-Forwarded-For header.

HTTP request processing now also considers tracking from backend rules
because we want to be able to update the counters even when the request
was already parsed and tracked.

Some more controls need to be performed (eg: samples do not distinguish
between L4 and L6).

doc/configuration.txt
include/proto/session.h
include/types/session.h
src/cfgparse.c
src/proto_http.c
src/proto_tcp.c
src/session.c

index 7e9f069e890cd293b9320270a8673670bf91d114..bfd97299512e20d9203f4e99d9070809a659bcce 100644 (file)
@@ -6149,9 +6149,12 @@ tcp-request connection <action> [{if | unless} <condition>]
         the per-frontend counters and the second set for the per-backend ones.
 
         These actions take one or two arguments :
-          <key>   is mandatory, and defines the criterion the tracking key will
-                  be derived from. At the moment, only "src" is supported. With
-                  it, the key will be the connection's source IPv4 address.
+          <key>   is mandatory, and is a pattern extraction rule as described
+                  in section 7.8. It describes what elements of the incoming
+                  request or connection will be analysed, extracted, combined,
+                  and used to select which table entry to update the counters.
+                  Note that "tcp-request connection" cannot use content-based
+                  fetches.
 
          <table>  is an optional table to be used instead of the default one,
                   which is the stick-table declared in the current proxy. All
@@ -6163,6 +6166,11 @@ tcp-request connection <action> [{if | unless} <condition>]
         that entry is kept during all the session's life, and this entry's
         counters are updated as often as possible, every time the session's
         counters are updated, and also systematically when the session ends.
+        Counters are only updated for events that happen after the tracking has
+        been started. For example, connection counters will not be updated when
+        tracking layer 7 information, since the connection event happens before
+        layer7 information is extracted.
+
         If the entry tracks concurrent connection counters, one connection is
         counted for as long as the entry is tracked, and the entry will not
         expire during that time. Tracking counters also provides a performance
@@ -6237,9 +6245,12 @@ tcp-request content <action> [{if | unless} <condition>]
   Also, it is worth noting that if sticky counters are tracked from a rule
   defined in a backend, this tracking will automatically end when the session
   releases the backend. That allows per-backend counter tracking even in case
-  of HTTP keep-alive requests when the backend changes. While there is nothing
-  mandatory about it, it is recommended to use the track-sc1 pointer to track
-  per-frontend counters and track-sc2 to track per-backend counters.
+  of HTTP keep-alive requests when the backend changes. This makes a subtle
+  difference because tracking rules in "frontend" and "listen" section last for
+  all the session, as opposed to the backend rules. The difference appears when
+  some layer 7 information is tracked. While there is nothing mandatory about
+  it, it is recommended to use the track-sc1 pointer to track per-frontend
+  counters and track-sc2 to track per-backend counters.
 
   Note that the "if/unless" condition is optional. If no condition is set on
   the action, it is simply performed unconditionally. That can be useful for
@@ -6252,6 +6263,11 @@ tcp-request content <action> [{if | unless} <condition>]
   The parser which is involved there is exactly the same as for all other HTTP
   processing, so there is no risk of parsing something differently.
 
+  Tracking layer7 information is also possible provided that the information
+  are present when the rule is processed. The current solution for making the
+  rule engine wait for such information is to set an inspect delay and to
+  condition its execution with an ACL relying on such information.
+
   Example:
         # Accept HTTP requests containing a Host header saying "example.com"
         # and reject everything else.
@@ -6272,6 +6288,16 @@ tcp-request content <action> [{if | unless} <condition>]
         tcp-request content accept if content_present
         tcp-request content reject
 
+  Example:
+        # Track the last IP from X-Forwarded-For
+        tcp-request inspect-delay 10s
+        tcp-request content track-sc1 hdr(x-forwarded-for,-1) if HTTP
+
+  Example:
+        # track request counts per "base" (concatenation of Host+URL)
+        tcp-request inspect-delay 10s
+        tcp-request content track-sc1 base table req-rate if HTTP
+
   Example: track per-frontend and per-backend counters, block abusers at the
            frontend when the backend detects abuse.
 
index 463b2e4b02fdbba093fec940ff0229789a38cf92..3a1c985a5f4a4f946b229d9dd2c15f4dfa958d62 100644 (file)
@@ -190,6 +190,37 @@ static void inline session_inc_http_req_ctr(struct session *s)
        }
 }
 
+/* Increase the number of cumulated HTTP requests in the backend's tracked counters */
+static void inline session_inc_be_http_req_ctr(struct session *s)
+{
+       void *ptr;
+
+       if (likely(!(s->flags & (SN_BE_TRACK_SC1|SN_BE_TRACK_SC2))))
+               return;
+
+       if ((s->flags & SN_BE_TRACK_SC2) && s->stkctr2_entry) {
+               ptr = stktable_data_ptr(s->stkctr2_table, s->stkctr2_entry, STKTABLE_DT_HTTP_REQ_CNT);
+               if (ptr)
+                       stktable_data_cast(ptr, http_req_cnt)++;
+
+               ptr = stktable_data_ptr(s->stkctr2_table, s->stkctr2_entry, STKTABLE_DT_HTTP_REQ_RATE);
+               if (ptr)
+                       update_freq_ctr_period(&stktable_data_cast(ptr, http_req_rate),
+                                              s->stkctr2_table->data_arg[STKTABLE_DT_HTTP_REQ_RATE].u, 1);
+       }
+
+       if ((s->flags & SN_BE_TRACK_SC1) && s->stkctr1_entry) {
+               ptr = stktable_data_ptr(s->stkctr1_table, s->stkctr1_entry, STKTABLE_DT_HTTP_REQ_CNT);
+               if (ptr)
+                       stktable_data_cast(ptr, http_req_cnt)++;
+
+               ptr = stktable_data_ptr(s->stkctr1_table, s->stkctr1_entry, STKTABLE_DT_HTTP_REQ_RATE);
+               if (ptr)
+                       update_freq_ctr_period(&stktable_data_cast(ptr, http_req_rate),
+                                              s->stkctr1_table->data_arg[STKTABLE_DT_HTTP_REQ_RATE].u, 1);
+       }
+}
+
 /* Increase the number of cumulated failed HTTP requests in the tracked
  * counters. Only 4xx requests should be counted here so that we can
  * distinguish between errors caused by client behaviour and other ones.
index e0d3c2274f606f1aeea174507dd8e0c0c18f5b8a..4688dbf074c21d39a4d74851a3ebf152d89ffec0 100644 (file)
@@ -166,7 +166,7 @@ struct session {
 
 /* parameters to configure tracked counters */
 struct track_ctr_prm {
-       int type;                               /* type of the key */
+       struct sample_expr *expr;               /* expression used as the key */
        union {
                struct stktable *t;             /* a pointer to the table */
                char *n;                        /* or its name during parsing. */
index 4e5963e42fe522fc84d78492b7ec4ad4ba859ba9..f1c45a00531a578e3e6810f770e2388c0070d483 100644 (file)
@@ -6263,11 +6263,6 @@ int check_config_validity()
                                      curproxy->id, trule->act_prm.trk_ctr.table.n ? trule->act_prm.trk_ctr.table.n : curproxy->id);
                                cfgerr++;
                        }
-                       else if (trule->act_prm.trk_ctr.type != target->table.type) {
-                               Alert("Proxy '%s': type of tracking key for sticky counter not usable with type of stick-table '%s'.\n",
-                                     curproxy->id, trule->act_prm.trk_ctr.table.n ? trule->act_prm.trk_ctr.table.n : curproxy->id);
-                               cfgerr++;
-                       }
                        else {
                                free(trule->act_prm.trk_ctr.table.n);
                                trule->act_prm.trk_ctr.table.t = &target->table;
@@ -6301,11 +6296,6 @@ int check_config_validity()
                                      curproxy->id, trule->act_prm.trk_ctr.table.n ? trule->act_prm.trk_ctr.table.n : curproxy->id);
                                cfgerr++;
                        }
-                       else if (trule->act_prm.trk_ctr.type != target->table.type) {
-                               Alert("Proxy '%s': type of tracking key for sticky counter not usable with type of stick-table '%s'.\n",
-                                     curproxy->id, trule->act_prm.trk_ctr.table.n ? trule->act_prm.trk_ctr.table.n : curproxy->id);
-                               cfgerr++;
-                       }
                        else {
                                free(trule->act_prm.trk_ctr.table.n);
                                trule->act_prm.trk_ctr.table.t = &target->table;
index 54850d7829043a854336a776bfc8783401ac2c76..ab8061a8641e0e0e5ede091f84b1a74faabaf737 100644 (file)
@@ -3013,6 +3013,9 @@ int http_process_req_common(struct session *s, struct channel *req, int an_bit,
                }
        }
 
+       /* just in case we have some per-backend tracking */
+       session_inc_be_http_req_ctr(s);
+
        /* evaluate http-request rules */
        http_req_last_rule = http_check_access_rule(px, &px->http_req_rules, s, txn);
 
index eb4444235c9311c6850ecb1da12419556c6d6bd4..de95d7f6ead012e42117a8d395bd8df2335efe37 100644 (file)
@@ -836,32 +836,22 @@ int tcp_inspect_request(struct session *s, struct channel *req, int an_bit)
                                        s->flags |= SN_FINST_R;
                                return 0;
                        }
-                       else if (rule->action == TCP_ACT_TRK_SC1) {
-                               if (!s->stkctr1_entry) {
-                                       /* only the first valid track-sc1 directive applies.
-                                        * Also, note that right now we can only track SRC so we
-                                        * don't check how to get the key, but later we may need
-                                        * to consider rule->act_prm->trk_ctr.type.
-                                        */
-                                       t = rule->act_prm.trk_ctr.table.t;
-                                       ts = stktable_get_entry(t, addr_to_stktable_key(&s->si[0].conn->addr.from));
-                                       if (ts) {
+                       else if ((rule->action == TCP_ACT_TRK_SC1 && !s->stkctr1_entry) ||
+                                (rule->action == TCP_ACT_TRK_SC2 && !s->stkctr2_entry)) {
+                               /* Note: only the first valid tracking parameter of each
+                                * applies.
+                                */
+                               struct stktable_key *key;
+
+                               t = rule->act_prm.trk_ctr.table.t;
+                               key = stktable_fetch_key(t, s->be, s, &s->txn, SMP_OPT_DIR_REQ|SMP_OPT_FINAL, rule->act_prm.trk_ctr.expr);
+
+                               if (key && (ts = stktable_get_entry(t, key))) {
+                                       if (rule->action == TCP_ACT_TRK_SC1) {
                                                session_track_stkctr1(s, t, ts);
                                                if (s->fe != s->be)
                                                        s->flags |= SN_BE_TRACK_SC1;
-                                       }
-                               }
-                       }
-                       else if (rule->action == TCP_ACT_TRK_SC2) {
-                               if (!s->stkctr2_entry) {
-                                       /* only the first valid track-sc2 directive applies.
-                                        * Also, note that right now we can only track SRC so we
-                                        * don't check how to get the key, but later we may need
-                                        * to consider rule->act_prm->trk_ctr.type.
-                                        */
-                                       t = rule->act_prm.trk_ctr.table.t;
-                                       ts = stktable_get_entry(t, addr_to_stktable_key(&s->si[0].conn->addr.from));
-                                       if (ts) {
+                                       } else {
                                                session_track_stkctr2(s, t, ts);
                                                if (s->fe != s->be)
                                                        s->flags |= SN_BE_TRACK_SC2;
@@ -1006,29 +996,20 @@ int tcp_exec_req_rules(struct session *s)
                                result = 0;
                                break;
                        }
-                       else if (rule->action == TCP_ACT_TRK_SC1) {
-                               if (!s->stkctr1_entry) {
-                                       /* only the first valid track-sc1 directive applies.
-                                        * Also, note that right now we can only track SRC so we
-                                        * don't check how to get the key, but later we may need
-                                        * to consider rule->act_prm->trk_ctr.type.
-                                        */
-                                       t = rule->act_prm.trk_ctr.table.t;
-                                       ts = stktable_get_entry(t, addr_to_stktable_key(&s->si[0].conn->addr.from));
-                                       if (ts)
+                       else if ((rule->action == TCP_ACT_TRK_SC1 && !s->stkctr1_entry) ||
+                                (rule->action == TCP_ACT_TRK_SC2 && !s->stkctr2_entry)) {
+                               /* Note: only the first valid tracking parameter of each
+                                * applies.
+                                */
+                               struct stktable_key *key;
+
+                               t = rule->act_prm.trk_ctr.table.t;
+                               key = stktable_fetch_key(t, s->be, s, &s->txn, SMP_OPT_DIR_REQ|SMP_OPT_FINAL, rule->act_prm.trk_ctr.expr);
+
+                               if (key && (ts = stktable_get_entry(t, key))) {
+                                       if (rule->action == TCP_ACT_TRK_SC1)
                                                session_track_stkctr1(s, t, ts);
-                               }
-                       }
-                       else if (rule->action == TCP_ACT_TRK_SC2) {
-                               if (!s->stkctr2_entry) {
-                                       /* only the first valid track-sc2 directive applies.
-                                        * Also, note that right now we can only track SRC so we
-                                        * don't check how to get the key, but later we may need
-                                        * to consider rule->act_prm->trk_ctr.type.
-                                        */
-                                       t = rule->act_prm.trk_ctr.table.t;
-                                       ts = stktable_get_entry(t, addr_to_stktable_key(&s->si[0].conn->addr.from));
-                                       if (ts)
+                                       else
                                                session_track_stkctr2(s, t, ts);
                                }
                        }
@@ -1105,37 +1086,50 @@ static int tcp_parse_request_rule(char **args, int arg, int section_type,
                arg++;
                rule->action = TCP_ACT_REJECT;
        }
-       else if (strcmp(args[arg], "track-sc1") == 0) {
-               int ret;
+       else if (strcmp(args[arg], "track-sc1") == 0 || strcmp(args[arg], "track-sc2") == 0) {
+               struct sample_expr *expr;
                int kw = arg;
 
                arg++;
-               ret = parse_track_counters(args, &arg, section_type, curpx,
-                                          &rule->act_prm.trk_ctr, defpx, err);
 
-               if (ret < 0) { /* nb: warnings are not handled yet */
+               expr = sample_parse_expr(args, &arg, trash.str, trash.size);
+               if (!expr) {
                        memprintf(err,
-                                 "'%s %s %s' : %s in %s '%s'",
-                                 args[0], args[1], args[kw], *err, proxy_type_str(curpx), curpx->id);
-                       return ret;
+                                 "'%s %s %s' : %s",
+                                 args[0], args[1], args[kw], trash.str);
+                       return -1;
                }
-               rule->action = TCP_ACT_TRK_SC1;
-       }
-       else if (strcmp(args[arg], "track-sc2") == 0) {
-               int ret;
-               int kw = arg;
 
-               arg++;
-               ret = parse_track_counters(args, &arg, section_type, curpx,
-                                          &rule->act_prm.trk_ctr, defpx, err);
-
-               if (ret < 0) { /* nb: warnings are not handled yet */
+               if (!(expr->fetch->cap & SMP_CAP_REQ)) {
                        memprintf(err,
-                                 "'%s %s %s' : %s in %s '%s'",
-                                 args[0], args[1], args[kw], *err, proxy_type_str(curpx), curpx->id);
-                       return ret;
+                                 "'%s %s %s' : fetch method '%s' cannot be used on request",
+                                 args[0], args[1], args[kw], trash.str);
+                       free(expr);
+                       return -1;
+               }
+
+               /* check if we need to allocate an hdr_idx struct for HTTP parsing */
+               if (expr->fetch->cap & SMP_CAP_L7)
+                       curpx->acl_requires |= ACL_USE_L7_ANY;
+
+               if (strcmp(args[arg], "table") == 0) {
+                       if (!args[arg + 1]) {
+                               memprintf(err,
+                                         "'%s %s %s' : missing table name",
+                                         args[0], args[1], args[kw]);
+                               free(expr);
+                               return -1;
+                       }
+                       /* we copy the table name for now, it will be resolved later */
+                       rule->act_prm.trk_ctr.table.n = strdup(args[arg + 1]);
+                       arg++;
                }
-               rule->action = TCP_ACT_TRK_SC2;
+               rule->act_prm.trk_ctr.expr = expr;
+
+               if (args[kw][8] == '1')
+                       rule->action = TCP_ACT_TRK_SC1;
+               else
+                       rule->action = TCP_ACT_TRK_SC2;
        }
        else {
                memprintf(err,
@@ -1312,6 +1306,15 @@ static int tcp_parse_tcp_req(char **args, int section_type, struct proxy *curpx,
                                  name, args[0], args[1]);
                        warn++;
                }
+
+               if ((rule->action == TCP_ACT_TRK_SC1 || rule->action == TCP_ACT_TRK_SC2) &&
+                   (rule->act_prm.trk_ctr.expr->fetch->cap & SMP_CAP_RES)) {
+                       memprintf(err,
+                                 "fetch '%s' involves some response-only criteria which will be ignored in '%s %s'",
+                                 rule->act_prm.trk_ctr.expr->fetch->kw, args[0], args[1]);
+                       warn++;
+               }
+
                LIST_ADDQ(&curpx->tcp_req.inspect_rules, &rule->list);
        }
        else if (strcmp(args[1], "connection") == 0) {
@@ -1346,6 +1349,23 @@ static int tcp_parse_tcp_req(char **args, int section_type, struct proxy *curpx,
                                          name, args[0], args[1]);
                        warn++;
                }
+
+               if ((rule->action == TCP_ACT_TRK_SC1 || rule->action == TCP_ACT_TRK_SC2) &&
+                   (rule->act_prm.trk_ctr.expr->fetch->cap & SMP_CAP_RES)) {
+                       memprintf(err,
+                                 "fetch '%s' involves some response-only criteria which will be ignored in '%s %s'",
+                                 rule->act_prm.trk_ctr.expr->fetch->kw, args[0], args[1]);
+                       warn++;
+               }
+
+               if ((rule->action == TCP_ACT_TRK_SC1 || rule->action == TCP_ACT_TRK_SC2) &&
+                   (rule->act_prm.trk_ctr.expr->fetch->cap & SMP_CAP_L7)) {
+                       memprintf(err,
+                                 "fetch '%s' involves some layer7-only criteria which will be ignored in '%s %s'",
+                                 rule->act_prm.trk_ctr.expr->fetch->kw, args[0], args[1]);
+                       warn++;
+               }
+
                LIST_ADDQ(&curpx->tcp_req.l4_rules, &rule->list);
        }
        else {
index 6e098e2603e90fdb8fea1d09246713ff71d7813a..0bee9576ee9a32f3b2900a07d75ccb945696c42c 100644 (file)
@@ -3682,54 +3682,6 @@ static struct acl_kw_list acl_kws = {{ },{
        { NULL, NULL, NULL, NULL },
 }};
 
-
-/* Parse a "track-sc[12]" line starting with "track-sc[12]" in args[arg-1].
- * Returns the number of warnings emitted, or -1 in case of fatal errors. The
- * <prm> struct is fed with the table name if any. If unspecified, the caller
- * will assume that the current proxy's table is used.
- */
-int parse_track_counters(char **args, int *arg,
-                        int section_type, struct proxy *curpx,
-                        struct track_ctr_prm *prm,
-                        struct proxy *defpx, char **err)
-{
-       int sample_type = 0;
-
-       /* parse the arguments of "track-sc[12]" before the condition in the
-        * following form :
-        *      track-sc[12] src [ table xxx ] [ if|unless ... ]
-        */
-       while (args[*arg]) {
-               if (strcmp(args[*arg], "src") == 0) {
-                       prm->type = STKTABLE_TYPE_IP;
-                       sample_type = 1;
-               }
-               else if (strcmp(args[*arg], "table") == 0) {
-                       if (!args[*arg + 1]) {
-                               memprintf(err, "missing table name");
-                               return -1;
-                       }
-                       /* we copy the table name for now, it will be resolved later */
-                       prm->table.n = strdup(args[*arg + 1]);
-                       (*arg)++;
-               }
-               else {
-                       /* unhandled keywords are handled by the caller */
-                       break;
-               }
-               (*arg)++;
-       }
-
-       if (!sample_type) {
-               memprintf(err,
-                         "tracking key not specified (found %s, only 'src' is supported)",
-                         quote_arg(args[*arg]));
-               return -1;
-       }
-
-       return 0;
-}
-
 __attribute__((constructor))
 static void __session_init(void)
 {