From: Willy Tarreau Date: Mon, 10 May 2010 21:42:40 +0000 (+0200) Subject: [MEDIUM] acl: add tree-based lookups of exact strings X-Git-Tag: v1.4.5~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=c4262961f8c47b67b22643b9fa4c854246ba7f3e;p=thirdparty%2Fhaproxy.git [MEDIUM] acl: add tree-based lookups of exact strings Now if some ACL patterns are loaded from a file and the operation is an exact string match, the data will be arranged in a tree, yielding a significant performance boost on large data sets. Note that this only works when case is sensitive. A new dedicated function, acl_lookup_str(), has been created for this matching. It is called for every possible input data to test and it looks the tree up for the data. Since the keywords are loosely typed, we would have had to add a new columns to all keywords to adjust the function depending on the type. Instead, we just compare on the match function. We call acl_lookup_str() when we could use acl_match_str(). The tree lookup is performed first, then the remaining patterns are attempted if the tree returned nothing. A quick test shows that when matching a header against a list of 52000 network names, haproxy uses 68% of one core on a core2-duo 3.2 GHz at 42000 requests per second, versus 66% without any rule, which means only a 2% CPU increase for 52000 rules. Doing the same test without the tree leads to 100% CPU at 6900 requests/s. Also it was possible to run the same test at full speed with about 50 sets of 52000 rules without any measurable performance drop. --- diff --git a/src/acl.c b/src/acl.c index ad3feaf910..b474c82c37 100644 --- a/src/acl.c +++ b/src/acl.c @@ -25,6 +25,8 @@ #include #include +#include + /* The capabilities of filtering hooks describe the type of information * available to each of them. */ @@ -130,6 +132,25 @@ int acl_match_str(struct acl_test *test, struct acl_pattern *pattern) return ACL_PAT_FAIL; } +/* Lookup a string in the expression's pattern tree. The node is returned if it + * exists, otherwise NULL. + */ +void *acl_lookup_str(struct acl_test *test, struct acl_expr *expr) +{ + /* data are stored in a tree */ + struct ebmb_node *node; + char prev; + + /* we may have to force a trailing zero on the test pattern */ + prev = test->ptr[test->len]; + if (prev) + test->ptr[test->len] = '\0'; + node = ebst_lookup(&expr->pattern_tree, test->ptr); + if (prev) + test->ptr[test->len] = prev; + return node; +} + /* Executes a regex. It needs to change the data. If it is marked READ_ONLY * then it will be allocated and duplicated in place so that others may use * it later on. Note that this is embarrassing because we always try to avoid @@ -329,6 +350,23 @@ int acl_parse_str(const char **text, struct acl_pattern *pattern, int *opaque) int len; len = strlen(*text); + + if (pattern->flags & ACL_PAT_F_TREE_OK) { + /* we're allowed to put the data in a tree whose root is pointed + * to by val.tree. + */ + struct ebmb_node *node; + + node = calloc(1, sizeof(*node) + len + 1); + if (!node) + return 0; + memcpy(node->key, *text, len + 1); + if (ebst_insert(pattern->val.tree, node) != node) + free(node); /* was a duplicate */ + pattern->flags |= ACL_PAT_F_TREE; /* this pattern now contains a tree */ + return 1; + } + pattern->ptr.str = strdup(*text); if (!pattern->ptr.str) return 0; @@ -1224,6 +1262,12 @@ int acl_exec_cond(struct acl_cond *cond, struct proxy *px, struct session *l4, v acl_res |= ACL_PAT_FAIL; } else { + if (expr->pattern_tree.b[EB_LEFT]) { + /* a tree is present, let's check what type it is */ + if (expr->kw->match == acl_match_str) + acl_res |= acl_lookup_str(&test, expr) ? ACL_PAT_PASS : ACL_PAT_FAIL; + } + /* call the match() function for all tests on this value */ list_for_each_entry(pattern, &expr->patterns, list) { if (acl_res == ACL_PAT_PASS) diff --git a/src/proto_http.c b/src/proto_http.c index 882f0dc4dc..c8835a5b50 100644 --- a/src/proto_http.c +++ b/src/proto_http.c @@ -7360,11 +7360,11 @@ static struct acl_kw_list acl_kws = {{ },{ { "req_proto_http", acl_parse_nothing, acl_fetch_proto_http, acl_match_nothing, ACL_USE_L7REQ_PERMANENT }, { "method", acl_parse_meth, acl_fetch_meth, acl_match_meth, ACL_USE_L7REQ_PERMANENT }, - { "req_ver", acl_parse_ver, acl_fetch_rqver, acl_match_str, ACL_USE_L7REQ_VOLATILE }, - { "resp_ver", acl_parse_ver, acl_fetch_stver, acl_match_str, ACL_USE_L7RTR_VOLATILE }, + { "req_ver", acl_parse_ver, acl_fetch_rqver, acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP }, + { "resp_ver", acl_parse_ver, acl_fetch_stver, acl_match_str, ACL_USE_L7RTR_VOLATILE|ACL_MAY_LOOKUP }, { "status", acl_parse_int, acl_fetch_stcode, acl_match_int, ACL_USE_L7RTR_PERMANENT }, - { "url", acl_parse_str, acl_fetch_url, acl_match_str, ACL_USE_L7REQ_VOLATILE }, + { "url", acl_parse_str, acl_fetch_url, acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP }, { "url_beg", acl_parse_str, acl_fetch_url, acl_match_beg, ACL_USE_L7REQ_VOLATILE }, { "url_end", acl_parse_str, acl_fetch_url, acl_match_end, ACL_USE_L7REQ_VOLATILE }, { "url_sub", acl_parse_str, acl_fetch_url, acl_match_sub, ACL_USE_L7REQ_VOLATILE }, @@ -7375,7 +7375,7 @@ static struct acl_kw_list acl_kws = {{ },{ { "url_port", acl_parse_int, acl_fetch_url_port, acl_match_int, ACL_USE_L7REQ_VOLATILE }, /* note: we should set hdr* to use ACL_USE_HDR_VOLATILE, and chdr* to use L7REQ_VOLATILE */ - { "hdr", acl_parse_str, acl_fetch_chdr, acl_match_str, ACL_USE_L7REQ_VOLATILE }, + { "hdr", acl_parse_str, acl_fetch_chdr, acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP }, { "hdr_reg", acl_parse_reg, acl_fetch_chdr, acl_match_reg, ACL_USE_L7REQ_VOLATILE }, { "hdr_beg", acl_parse_str, acl_fetch_chdr, acl_match_beg, ACL_USE_L7REQ_VOLATILE }, { "hdr_end", acl_parse_str, acl_fetch_chdr, acl_match_end, ACL_USE_L7REQ_VOLATILE }, @@ -7386,7 +7386,7 @@ static struct acl_kw_list acl_kws = {{ },{ { "hdr_val", acl_parse_int, acl_fetch_chdr_val,acl_match_int, ACL_USE_L7REQ_VOLATILE }, { "hdr_ip", acl_parse_ip, acl_fetch_chdr_ip, acl_match_ip, ACL_USE_L7REQ_VOLATILE }, - { "shdr", acl_parse_str, acl_fetch_shdr, acl_match_str, ACL_USE_L7RTR_VOLATILE }, + { "shdr", acl_parse_str, acl_fetch_shdr, acl_match_str, ACL_USE_L7RTR_VOLATILE|ACL_MAY_LOOKUP }, { "shdr_reg", acl_parse_reg, acl_fetch_shdr, acl_match_reg, ACL_USE_L7RTR_VOLATILE }, { "shdr_beg", acl_parse_str, acl_fetch_shdr, acl_match_beg, ACL_USE_L7RTR_VOLATILE }, { "shdr_end", acl_parse_str, acl_fetch_shdr, acl_match_end, ACL_USE_L7RTR_VOLATILE }, @@ -7397,7 +7397,7 @@ static struct acl_kw_list acl_kws = {{ },{ { "shdr_val", acl_parse_int, acl_fetch_shdr_val,acl_match_int, ACL_USE_L7RTR_VOLATILE }, { "shdr_ip", acl_parse_ip, acl_fetch_shdr_ip, acl_match_ip, ACL_USE_L7RTR_VOLATILE }, - { "path", acl_parse_str, acl_fetch_path, acl_match_str, ACL_USE_L7REQ_VOLATILE }, + { "path", acl_parse_str, acl_fetch_path, acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP }, { "path_reg", acl_parse_reg, acl_fetch_path, acl_match_reg, ACL_USE_L7REQ_VOLATILE }, { "path_beg", acl_parse_str, acl_fetch_path, acl_match_beg, ACL_USE_L7REQ_VOLATILE }, { "path_end", acl_parse_str, acl_fetch_path, acl_match_end, ACL_USE_L7REQ_VOLATILE }, diff --git a/src/proto_tcp.c b/src/proto_tcp.c index 6494db7a55..7c958fc4d8 100644 --- a/src/proto_tcp.c +++ b/src/proto_tcp.c @@ -1138,7 +1138,7 @@ static struct cfg_kw_list cfg_kws = {{ },{ static struct acl_kw_list acl_kws = {{ },{ { "req_len", acl_parse_int, acl_fetch_req_len, acl_match_int, ACL_USE_L4REQ_VOLATILE }, { "req_ssl_ver", acl_parse_dotted_ver, acl_fetch_req_ssl_ver, acl_match_int, ACL_USE_L4REQ_VOLATILE }, - { "req_rdp_cookie", acl_parse_str, acl_fetch_rdp_cookie, acl_match_str, ACL_USE_L4REQ_VOLATILE }, + { "req_rdp_cookie", acl_parse_str, acl_fetch_rdp_cookie, acl_match_str, ACL_USE_L4REQ_VOLATILE|ACL_MAY_LOOKUP }, { "req_rdp_cookie_cnt", acl_parse_int, acl_fetch_rdp_cookie_cnt, acl_match_int, ACL_USE_L4REQ_VOLATILE }, { NULL, NULL, NULL, NULL }, }};