[MEDIUM] acl: add tree-based lookups of exact strings

Now if some ACL patterns are loaded from a file and the operation is
an exact string match, the data will be arranged in a tree, yielding
a significant performance boost on large data sets. Note that this
only works when case is sensitive.

A new dedicated function, acl_lookup_str(), has been created for this
matching. It is called for every possible input data to test and it
looks the tree up for the data. Since the keywords are loosely typed,
we would have had to add a new columns to all keywords to adjust the
function depending on the type. Instead, we just compare on the match
function. We call acl_lookup_str() when we could use acl_match_str().
The tree lookup is performed first, then the remaining patterns are
attempted if the tree returned nothing.

A quick test shows that when matching a header against a list of 52000
network names, haproxy uses 68% of one core on a core2-duo 3.2 GHz at
42000 requests per second, versus 66% without any rule, which means
only a 2% CPU increase for 52000 rules. Doing the same test without
the tree leads to 100% CPU at 6900 requests/s. Also it was possible
to run the same test at full speed with about 50 sets of 52000 rules
without any measurable performance drop.

diff --git a/src/acl.c b/src/acl.c
index ad3feaf910c7bdfaa18c8ef06d78c33c0470507c..b474c82c375ea4cfcf4a8ac2abdfd9f8afbf2614 100644 (file)
--- a/src/acl.c
+++ b/src/acl.c
@@ -25,6 +25,8 @@
 #include <proto/auth.h>
 #include <proto/log.h>
 
+#include <ebsttree.h>
+
 /* The capabilities of filtering hooks describe the type of information
  * available to each of them.
  */
@@ -130,6 +132,25 @@ int acl_match_str(struct acl_test *test, struct acl_pattern *pattern)
        return ACL_PAT_FAIL;
 }
 
+/* Lookup a string in the expression's pattern tree. The node is returned if it
+ * exists, otherwise NULL.
+ */
+void *acl_lookup_str(struct acl_test *test, struct acl_expr *expr)
+{
+       /* data are stored in a tree */
+       struct ebmb_node *node;
+       char prev;
+
+       /* we may have to force a trailing zero on the test pattern */
+       prev = test->ptr[test->len];
+       if (prev)
+               test->ptr[test->len] = '\0';
+       node = ebst_lookup(&expr->pattern_tree, test->ptr);
+       if (prev)
+               test->ptr[test->len] = prev;
+       return node;
+}
+
 /* Executes a regex. It needs to change the data. If it is marked READ_ONLY
  * then it will be allocated and duplicated in place so that others may use
  * it later on. Note that this is embarrassing because we always try to avoid
@@ -329,6 +350,23 @@ int acl_parse_str(const char **text, struct acl_pattern *pattern, int *opaque)
        int len;
 
        len  = strlen(*text);
+
+       if (pattern->flags & ACL_PAT_F_TREE_OK) {
+               /* we're allowed to put the data in a tree whose root is pointed
+                * to by val.tree.
+                */
+               struct ebmb_node *node;
+
+               node = calloc(1, sizeof(*node) + len + 1);
+               if (!node)
+                       return 0;
+               memcpy(node->key, *text, len + 1);
+               if (ebst_insert(pattern->val.tree, node) != node)
+                       free(node); /* was a duplicate */
+               pattern->flags |= ACL_PAT_F_TREE; /* this pattern now contains a tree */
+               return 1;
+       }
+
        pattern->ptr.str = strdup(*text);
        if (!pattern->ptr.str)
                return 0;
@@ -1224,6 +1262,12 @@ int acl_exec_cond(struct acl_cond *cond, struct proxy *px, struct session *l4, v
                                                acl_res |= ACL_PAT_FAIL;
                                }
                                else {
+                                       if (expr->pattern_tree.b[EB_LEFT]) {
+                                               /* a tree is present, let's check what type it is */
+                                               if (expr->kw->match == acl_match_str)
+                                                       acl_res |= acl_lookup_str(&test, expr) ? ACL_PAT_PASS : ACL_PAT_FAIL;
+                                       }
+
                                        /* call the match() function for all tests on this value */
                                        list_for_each_entry(pattern, &expr->patterns, list) {
                                                if (acl_res == ACL_PAT_PASS)

diff --git a/src/proto_http.c b/src/proto_http.c
index 882f0dc4dca7a075ef264ea6c3e6c796c2998725..c8835a5b5004c879fa60caa96a5847595e9c4102 100644 (file)
--- a/src/proto_http.c
+++ b/src/proto_http.c
@@ -7360,11 +7360,11 @@ static struct acl_kw_list acl_kws = {{ },{
        { "req_proto_http", acl_parse_nothing, acl_fetch_proto_http, acl_match_nothing, ACL_USE_L7REQ_PERMANENT },
 
        { "method",     acl_parse_meth,  acl_fetch_meth,   acl_match_meth, ACL_USE_L7REQ_PERMANENT },
-       { "req_ver",    acl_parse_ver,   acl_fetch_rqver,  acl_match_str,  ACL_USE_L7REQ_VOLATILE  },
-       { "resp_ver",   acl_parse_ver,   acl_fetch_stver,  acl_match_str,  ACL_USE_L7RTR_VOLATILE  },
+       { "req_ver",    acl_parse_ver,   acl_fetch_rqver,  acl_match_str,  ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP },
+       { "resp_ver",   acl_parse_ver,   acl_fetch_stver,  acl_match_str,  ACL_USE_L7RTR_VOLATILE|ACL_MAY_LOOKUP },
        { "status",     acl_parse_int,   acl_fetch_stcode, acl_match_int,  ACL_USE_L7RTR_PERMANENT },
 
-       { "url",        acl_parse_str,   acl_fetch_url,      acl_match_str,  ACL_USE_L7REQ_VOLATILE },
+       { "url",        acl_parse_str,   acl_fetch_url,      acl_match_str,  ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP },
        { "url_beg",    acl_parse_str,   acl_fetch_url,      acl_match_beg,  ACL_USE_L7REQ_VOLATILE },
        { "url_end",    acl_parse_str,   acl_fetch_url,      acl_match_end,  ACL_USE_L7REQ_VOLATILE },
        { "url_sub",    acl_parse_str,   acl_fetch_url,      acl_match_sub,  ACL_USE_L7REQ_VOLATILE },
@@ -7375,7 +7375,7 @@ static struct acl_kw_list acl_kws = {{ },{
        { "url_port",   acl_parse_int,   acl_fetch_url_port, acl_match_int,  ACL_USE_L7REQ_VOLATILE },
 
        /* note: we should set hdr* to use ACL_USE_HDR_VOLATILE, and chdr* to use L7REQ_VOLATILE */
-       { "hdr",        acl_parse_str,   acl_fetch_chdr,    acl_match_str, ACL_USE_L7REQ_VOLATILE },
+       { "hdr",        acl_parse_str,   acl_fetch_chdr,    acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP },
        { "hdr_reg",    acl_parse_reg,   acl_fetch_chdr,    acl_match_reg, ACL_USE_L7REQ_VOLATILE },
        { "hdr_beg",    acl_parse_str,   acl_fetch_chdr,    acl_match_beg, ACL_USE_L7REQ_VOLATILE },
        { "hdr_end",    acl_parse_str,   acl_fetch_chdr,    acl_match_end, ACL_USE_L7REQ_VOLATILE },
@@ -7386,7 +7386,7 @@ static struct acl_kw_list acl_kws = {{ },{
        { "hdr_val",    acl_parse_int,   acl_fetch_chdr_val,acl_match_int, ACL_USE_L7REQ_VOLATILE },
        { "hdr_ip",     acl_parse_ip,    acl_fetch_chdr_ip, acl_match_ip,  ACL_USE_L7REQ_VOLATILE },
 
-       { "shdr",       acl_parse_str,   acl_fetch_shdr,    acl_match_str, ACL_USE_L7RTR_VOLATILE },
+       { "shdr",       acl_parse_str,   acl_fetch_shdr,    acl_match_str, ACL_USE_L7RTR_VOLATILE|ACL_MAY_LOOKUP },
        { "shdr_reg",   acl_parse_reg,   acl_fetch_shdr,    acl_match_reg, ACL_USE_L7RTR_VOLATILE },
        { "shdr_beg",   acl_parse_str,   acl_fetch_shdr,    acl_match_beg, ACL_USE_L7RTR_VOLATILE },
        { "shdr_end",   acl_parse_str,   acl_fetch_shdr,    acl_match_end, ACL_USE_L7RTR_VOLATILE },
@@ -7397,7 +7397,7 @@ static struct acl_kw_list acl_kws = {{ },{
        { "shdr_val",   acl_parse_int,   acl_fetch_shdr_val,acl_match_int, ACL_USE_L7RTR_VOLATILE },
        { "shdr_ip",    acl_parse_ip,    acl_fetch_shdr_ip, acl_match_ip,  ACL_USE_L7RTR_VOLATILE },
 
-       { "path",       acl_parse_str,   acl_fetch_path,   acl_match_str, ACL_USE_L7REQ_VOLATILE },
+       { "path",       acl_parse_str,   acl_fetch_path,   acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP },
        { "path_reg",   acl_parse_reg,   acl_fetch_path,   acl_match_reg, ACL_USE_L7REQ_VOLATILE },
        { "path_beg",   acl_parse_str,   acl_fetch_path,   acl_match_beg, ACL_USE_L7REQ_VOLATILE },
        { "path_end",   acl_parse_str,   acl_fetch_path,   acl_match_end, ACL_USE_L7REQ_VOLATILE },

diff --git a/src/proto_tcp.c b/src/proto_tcp.c
index 6494db7a55245ea42aa37d955492cbf60689308e..7c958fc4d8769f414c9b20220ec9518f52e96cf7 100644 (file)
--- a/src/proto_tcp.c
+++ b/src/proto_tcp.c
@@ -1138,7 +1138,7 @@ static struct cfg_kw_list cfg_kws = {{ },{
 static struct acl_kw_list acl_kws = {{ },{
        { "req_len",      acl_parse_int,        acl_fetch_req_len,     acl_match_int, ACL_USE_L4REQ_VOLATILE },
        { "req_ssl_ver",  acl_parse_dotted_ver, acl_fetch_req_ssl_ver, acl_match_int, ACL_USE_L4REQ_VOLATILE },
-       { "req_rdp_cookie",     acl_parse_str,  acl_fetch_rdp_cookie,     acl_match_str, ACL_USE_L4REQ_VOLATILE },
+       { "req_rdp_cookie",     acl_parse_str,  acl_fetch_rdp_cookie,     acl_match_str, ACL_USE_L4REQ_VOLATILE|ACL_MAY_LOOKUP },
        { "req_rdp_cookie_cnt", acl_parse_int,  acl_fetch_rdp_cookie_cnt, acl_match_int, ACL_USE_L4REQ_VOLATILE },
        { NULL, NULL, NULL, NULL },
 }};