[MEDIUM] acl: add tree-based lookups of exact strings
Now if some ACL patterns are loaded from a file and the operation is
an exact string match, the data will be arranged in a tree, yielding
a significant performance boost on large data sets. Note that this
only works when case is sensitive.
A new dedicated function, acl_lookup_str(), has been created for this
matching. It is called for every possible input data to test and it
looks the tree up for the data. Since the keywords are loosely typed,
we would have had to add a new columns to all keywords to adjust the
function depending on the type. Instead, we just compare on the match
function. We call acl_lookup_str() when we could use acl_match_str().
The tree lookup is performed first, then the remaining patterns are
attempted if the tree returned nothing.
A quick test shows that when matching a header against a list of 52000
network names, haproxy uses 68% of one core on a core2-duo 3.2 GHz at
42000 requests per second, versus 66% without any rule, which means
only a 2% CPU increase for 52000 rules. Doing the same test without
the tree leads to 100% CPU at 6900 requests/s. Also it was possible
to run the same test at full speed with about 50 sets of 52000 rules
without any measurable performance drop.
diff --git a/src/acl.c b/src/acl.c
index ad3feaf..b474c82 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -25,6 +25,8 @@
#include <proto/auth.h>
#include <proto/log.h>
+#include <ebsttree.h>
+
/* The capabilities of filtering hooks describe the type of information
* available to each of them.
*/
@@ -130,6 +132,25 @@
return ACL_PAT_FAIL;
}
+/* Lookup a string in the expression's pattern tree. The node is returned if it
+ * exists, otherwise NULL.
+ */
+void *acl_lookup_str(struct acl_test *test, struct acl_expr *expr)
+{
+ /* data are stored in a tree */
+ struct ebmb_node *node;
+ char prev;
+
+ /* we may have to force a trailing zero on the test pattern */
+ prev = test->ptr[test->len];
+ if (prev)
+ test->ptr[test->len] = '\0';
+ node = ebst_lookup(&expr->pattern_tree, test->ptr);
+ if (prev)
+ test->ptr[test->len] = prev;
+ return node;
+}
+
/* Executes a regex. It needs to change the data. If it is marked READ_ONLY
* then it will be allocated and duplicated in place so that others may use
* it later on. Note that this is embarrassing because we always try to avoid
@@ -329,6 +350,23 @@
int len;
len = strlen(*text);
+
+ if (pattern->flags & ACL_PAT_F_TREE_OK) {
+ /* we're allowed to put the data in a tree whose root is pointed
+ * to by val.tree.
+ */
+ struct ebmb_node *node;
+
+ node = calloc(1, sizeof(*node) + len + 1);
+ if (!node)
+ return 0;
+ memcpy(node->key, *text, len + 1);
+ if (ebst_insert(pattern->val.tree, node) != node)
+ free(node); /* was a duplicate */
+ pattern->flags |= ACL_PAT_F_TREE; /* this pattern now contains a tree */
+ return 1;
+ }
+
pattern->ptr.str = strdup(*text);
if (!pattern->ptr.str)
return 0;
@@ -1224,6 +1262,12 @@
acl_res |= ACL_PAT_FAIL;
}
else {
+ if (expr->pattern_tree.b[EB_LEFT]) {
+ /* a tree is present, let's check what type it is */
+ if (expr->kw->match == acl_match_str)
+ acl_res |= acl_lookup_str(&test, expr) ? ACL_PAT_PASS : ACL_PAT_FAIL;
+ }
+
/* call the match() function for all tests on this value */
list_for_each_entry(pattern, &expr->patterns, list) {
if (acl_res == ACL_PAT_PASS)
diff --git a/src/proto_http.c b/src/proto_http.c
index 882f0dc..c8835a5 100644
--- a/src/proto_http.c
+++ b/src/proto_http.c
@@ -7360,11 +7360,11 @@
{ "req_proto_http", acl_parse_nothing, acl_fetch_proto_http, acl_match_nothing, ACL_USE_L7REQ_PERMANENT },
{ "method", acl_parse_meth, acl_fetch_meth, acl_match_meth, ACL_USE_L7REQ_PERMANENT },
- { "req_ver", acl_parse_ver, acl_fetch_rqver, acl_match_str, ACL_USE_L7REQ_VOLATILE },
- { "resp_ver", acl_parse_ver, acl_fetch_stver, acl_match_str, ACL_USE_L7RTR_VOLATILE },
+ { "req_ver", acl_parse_ver, acl_fetch_rqver, acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP },
+ { "resp_ver", acl_parse_ver, acl_fetch_stver, acl_match_str, ACL_USE_L7RTR_VOLATILE|ACL_MAY_LOOKUP },
{ "status", acl_parse_int, acl_fetch_stcode, acl_match_int, ACL_USE_L7RTR_PERMANENT },
- { "url", acl_parse_str, acl_fetch_url, acl_match_str, ACL_USE_L7REQ_VOLATILE },
+ { "url", acl_parse_str, acl_fetch_url, acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP },
{ "url_beg", acl_parse_str, acl_fetch_url, acl_match_beg, ACL_USE_L7REQ_VOLATILE },
{ "url_end", acl_parse_str, acl_fetch_url, acl_match_end, ACL_USE_L7REQ_VOLATILE },
{ "url_sub", acl_parse_str, acl_fetch_url, acl_match_sub, ACL_USE_L7REQ_VOLATILE },
@@ -7375,7 +7375,7 @@
{ "url_port", acl_parse_int, acl_fetch_url_port, acl_match_int, ACL_USE_L7REQ_VOLATILE },
/* note: we should set hdr* to use ACL_USE_HDR_VOLATILE, and chdr* to use L7REQ_VOLATILE */
- { "hdr", acl_parse_str, acl_fetch_chdr, acl_match_str, ACL_USE_L7REQ_VOLATILE },
+ { "hdr", acl_parse_str, acl_fetch_chdr, acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP },
{ "hdr_reg", acl_parse_reg, acl_fetch_chdr, acl_match_reg, ACL_USE_L7REQ_VOLATILE },
{ "hdr_beg", acl_parse_str, acl_fetch_chdr, acl_match_beg, ACL_USE_L7REQ_VOLATILE },
{ "hdr_end", acl_parse_str, acl_fetch_chdr, acl_match_end, ACL_USE_L7REQ_VOLATILE },
@@ -7386,7 +7386,7 @@
{ "hdr_val", acl_parse_int, acl_fetch_chdr_val,acl_match_int, ACL_USE_L7REQ_VOLATILE },
{ "hdr_ip", acl_parse_ip, acl_fetch_chdr_ip, acl_match_ip, ACL_USE_L7REQ_VOLATILE },
- { "shdr", acl_parse_str, acl_fetch_shdr, acl_match_str, ACL_USE_L7RTR_VOLATILE },
+ { "shdr", acl_parse_str, acl_fetch_shdr, acl_match_str, ACL_USE_L7RTR_VOLATILE|ACL_MAY_LOOKUP },
{ "shdr_reg", acl_parse_reg, acl_fetch_shdr, acl_match_reg, ACL_USE_L7RTR_VOLATILE },
{ "shdr_beg", acl_parse_str, acl_fetch_shdr, acl_match_beg, ACL_USE_L7RTR_VOLATILE },
{ "shdr_end", acl_parse_str, acl_fetch_shdr, acl_match_end, ACL_USE_L7RTR_VOLATILE },
@@ -7397,7 +7397,7 @@
{ "shdr_val", acl_parse_int, acl_fetch_shdr_val,acl_match_int, ACL_USE_L7RTR_VOLATILE },
{ "shdr_ip", acl_parse_ip, acl_fetch_shdr_ip, acl_match_ip, ACL_USE_L7RTR_VOLATILE },
- { "path", acl_parse_str, acl_fetch_path, acl_match_str, ACL_USE_L7REQ_VOLATILE },
+ { "path", acl_parse_str, acl_fetch_path, acl_match_str, ACL_USE_L7REQ_VOLATILE|ACL_MAY_LOOKUP },
{ "path_reg", acl_parse_reg, acl_fetch_path, acl_match_reg, ACL_USE_L7REQ_VOLATILE },
{ "path_beg", acl_parse_str, acl_fetch_path, acl_match_beg, ACL_USE_L7REQ_VOLATILE },
{ "path_end", acl_parse_str, acl_fetch_path, acl_match_end, ACL_USE_L7REQ_VOLATILE },
diff --git a/src/proto_tcp.c b/src/proto_tcp.c
index 6494db7..7c958fc 100644
--- a/src/proto_tcp.c
+++ b/src/proto_tcp.c
@@ -1138,7 +1138,7 @@
static struct acl_kw_list acl_kws = {{ },{
{ "req_len", acl_parse_int, acl_fetch_req_len, acl_match_int, ACL_USE_L4REQ_VOLATILE },
{ "req_ssl_ver", acl_parse_dotted_ver, acl_fetch_req_ssl_ver, acl_match_int, ACL_USE_L4REQ_VOLATILE },
- { "req_rdp_cookie", acl_parse_str, acl_fetch_rdp_cookie, acl_match_str, ACL_USE_L4REQ_VOLATILE },
+ { "req_rdp_cookie", acl_parse_str, acl_fetch_rdp_cookie, acl_match_str, ACL_USE_L4REQ_VOLATILE|ACL_MAY_LOOKUP },
{ "req_rdp_cookie_cnt", acl_parse_int, acl_fetch_rdp_cookie_cnt, acl_match_int, ACL_USE_L4REQ_VOLATILE },
{ NULL, NULL, NULL, NULL },
}};