detect: optimize rule address parsing

Many rules have the same address vars, so instead of parsing them
each time use a hash to store the string and the parsed result.

Rules now reference the stored result in the hash table.

diff --git a/src/detect-engine-address.c b/src/detect-engine-address.c
index 8c07d08fc8551b4cfe847cf5e0dd9fbba4e1ed42..60964cd9c837b5852e94e47de92b5f78ce056d2e 100644 (file)
--- a/src/detect-engine-address.c
+++ b/src/detect-engine-address.c
@@ -815,7 +815,7 @@ error:
  */
 static int DetectAddressParse2(const DetectEngineCtx *de_ctx,
         DetectAddressHead *gh, DetectAddressHead *ghn,
-        char *s, int negate, ResolvedVariablesList *var_list)
+        const char *s, int negate, ResolvedVariablesList *var_list)
 {
     size_t x = 0;
     size_t u = 0;
@@ -1023,6 +1023,8 @@ static int DetectAddressParse2(const DetectEngineCtx *de_ctx,
                 if (DetectAddressParse2(de_ctx, gh, ghn, temp_rule_var_address,
                                     (negate + n_set) % 2, var_list) < 0) {
                     SCLogDebug("DetectAddressParse2 hates us");
+                    if (temp_rule_var_address != rule_var_address)
+                        SCFree(temp_rule_var_address);
                     goto error;
                 }
                 d_set = 0;
@@ -1345,6 +1347,98 @@ int DetectAddressTestConfVars(void)
     return -1;
 }
 
+#include "util-hash-lookup3.h"
+
+typedef struct DetectAddressMap_ {
+    char *string;
+    DetectAddressHead *address;
+} DetectAddressMap;
+
+static uint32_t DetectAddressMapHashFunc(HashListTable *ht, void *data, uint16_t datalen)
+{
+    const DetectAddressMap *map = (DetectAddressMap *)data;
+    uint32_t hash = 0;
+
+    hash = hashlittle_safe(map->string, strlen(map->string), 0);
+    hash %= ht->array_size;
+
+    return hash;
+}
+
+static char DetectAddressMapCompareFunc(void *data1, uint16_t len1, void *data2,
+                                        uint16_t len2)
+{
+    DetectAddressMap *map1 = (DetectAddressMap *)data1;
+    DetectAddressMap *map2 = (DetectAddressMap *)data2;
+
+
+    int r = (strcmp(map1->string, map2->string) == 0);
+    return r;
+}
+
+static void DetectAddressMapFreeFunc(void *data)
+{
+    DetectAddressMap *map = (DetectAddressMap *)data;
+    if (map != NULL) {
+        DetectAddressHeadFree(map->address);
+        SCFree(map->string);
+    }
+    SCFree(map);
+}
+
+int DetectAddressMapInit(DetectEngineCtx *de_ctx)
+{
+    de_ctx->address_table = HashListTableInit(4096, DetectAddressMapHashFunc,
+                                                    DetectAddressMapCompareFunc,
+                                                    DetectAddressMapFreeFunc);
+    if (de_ctx->address_table == NULL)
+        return -1;
+
+    return 0;
+}
+
+void DetectAddressMapFree(DetectEngineCtx *de_ctx)
+{
+    if (de_ctx->address_table == NULL)
+        return;
+
+    HashListTableFree(de_ctx->address_table);
+    de_ctx->address_table = NULL;
+    return;
+}
+
+int DetectAddressMapAdd(DetectEngineCtx *de_ctx, const char *string,
+                        DetectAddressHead *address)
+{
+    DetectAddressMap *map = SCCalloc(1, sizeof(*map));
+    if (map == NULL)
+        return -1;
+
+    map->string = SCStrdup(string);
+    if (map->string == NULL) {
+        SCFree(map);
+        return -1;
+    }
+    map->address = address;
+
+    BUG_ON(HashListTableAdd(de_ctx->address_table, (void *)map, 0) != 0);
+    return 0;
+}
+
+const DetectAddressHead *DetectAddressMapLookup(DetectEngineCtx *de_ctx,
+                                                const char *string)
+{
+    DetectAddressMap map = { (char *)string, NULL };
+
+    const DetectAddressMap *res = HashListTableLookup(de_ctx->address_table,
+            &map, 0);
+    if (res == NULL)
+        return NULL;
+    else {
+        return (const DetectAddressHead *)res->address;
+    }
+}
+
 /**
  * \brief Parses an address group sent as a character string and updates the
  *        DetectAddressHead sent as the argument with the relevant address
@@ -1358,7 +1452,7 @@ int DetectAddressTestConfVars(void)
  * \retval -1 On failure.
  */
 int DetectAddressParse(const DetectEngineCtx *de_ctx,
-                       DetectAddressHead *gh, char *str)
+                       DetectAddressHead *gh, const char *str)
 {
     int r;
     DetectAddressHead *ghn = NULL;
@@ -1401,6 +1495,31 @@ error:
     return -1;
 }
 
+const DetectAddressHead *DetectParseAddress(DetectEngineCtx *de_ctx,
+        const char *string)
+{
+    const DetectAddressHead *h = DetectAddressMapLookup(de_ctx, string);
+    if (h != NULL) {
+        SCLogDebug("found: %s :: %p", string, h);
+        return h;
+    }
+
+    SCLogDebug("%s not found", string);
+
+    DetectAddressHead *head = DetectAddressHeadInit();
+    if (head == NULL)
+        return NULL;
+
+    if (DetectAddressParse(de_ctx, head, string) == -1)
+    {
+        DetectAddressHeadFree(head);
+        return NULL;
+    }
+
+    DetectAddressMapAdd((DetectEngineCtx *)de_ctx, string, head);
+    return head;
+}
+
 /**
  * \brief Returns a new instance of DetectAddressHead.
  *

diff --git a/src/detect-engine-address.h b/src/detect-engine-address.h
index 3b84f225770335a7bcf40d26f040296064f3cb3f..dcbde2e4277bbcb2a02d79b7b5cbfc28e7c8c543 100644 (file)
--- a/src/detect-engine-address.h
+++ b/src/detect-engine-address.h
@@ -32,7 +32,7 @@ void DetectAddressHeadFree(DetectAddressHead *);
 void DetectAddressHeadCleanup(DetectAddressHead *);
 
 int DetectAddressParseString(DetectAddress *, char *);
-int DetectAddressParse(const DetectEngineCtx *, DetectAddressHead *, char *);
+int DetectAddressParse(const DetectEngineCtx *, DetectAddressHead *, const char *);
 
 DetectAddress *DetectAddressInit(void);
 void DetectAddressFree(DetectAddress *);
@@ -59,4 +59,9 @@ int DetectAddressTestConfVars(void);
 
 void DetectAddressTests(void);
 
+int DetectAddressMapInit(DetectEngineCtx *de_ctx);
+void DetectAddressMapFree(DetectEngineCtx *de_ctx);
+const DetectAddressHead *DetectParseAddress(DetectEngineCtx *de_ctx,
+        const char *string);
+
 #endif /* __DETECT_ADDRESS_H__ */

diff --git a/src/detect-engine.c b/src/detect-engine.c
index e108258d9c9938654600e7036e64e3fa9808f63b..d7aee2242a04762265078820ce457124a92cf20e 100644 (file)
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -854,6 +854,7 @@ static DetectEngineCtx *DetectEngineCtxInitReal(int minimal, const char *prefix)
     ThresholdHashInit(de_ctx);
     VariableNameInitHash(de_ctx);
     DetectParseDupSigHashInit(de_ctx);
+    DetectAddressMapInit(de_ctx);
 
     /* init iprep... ignore errors for now */
     (void)SRepInit(de_ctx);
@@ -964,6 +965,8 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
     DetectEngineCtxFreeThreadKeywordData(de_ctx);
     SRepDestroy(de_ctx);
 
+    DetectAddressMapFree(de_ctx);
+
     /* if we have a config prefix, remove the config from the tree */
     if (strlen(de_ctx->config_prefix) > 0) {
         /* remove config */

diff --git a/src/detect-parse.c b/src/detect-parse.c
index f78b6033c78ba8b15c9a80f4b9fd9e6b7d9f47b7..9e970e38963cbae359f5540d7d56f8525082f336 100644 (file)
--- a/src/detect-parse.c
+++ b/src/detect-parse.c
@@ -624,7 +624,7 @@ error:
  *
  *  \retval 0 ok, -1 error
  */
-int SigParseAddress(const DetectEngineCtx *de_ctx,
+static int SigParseAddress(DetectEngineCtx *de_ctx,
         Signature *s, const char *addrstr, char flag)
 {
     SCLogDebug("Address Group \"%s\" to be parsed now", addrstr);
@@ -634,13 +634,15 @@ int SigParseAddress(const DetectEngineCtx *de_ctx,
         if (strcasecmp(addrstr, "any") == 0)
             s->flags |= SIG_FLAG_SRC_ANY;
 
-        if (DetectAddressParse(de_ctx, &s->src, (char *)addrstr) < 0)
+        s->src = DetectParseAddress(de_ctx, addrstr);
+        if (s->src == NULL)
             goto error;
     } else {
         if (strcasecmp(addrstr, "any") == 0)
             s->flags |= SIG_FLAG_DST_ANY;
 
-        if (DetectAddressParse(de_ctx, &s->dst, (char *)addrstr) < 0)
+        s->dst = DetectParseAddress(de_ctx, addrstr);
+        if (s->dst == NULL)
             goto error;
     }
 
@@ -808,7 +810,7 @@ int SigParseAction(Signature *s, const char *action)
  *  \internal
  *  \brief split a signature string into a few blocks for further parsing
  */
-static int SigParseBasics(const DetectEngineCtx *de_ctx,
+static int SigParseBasics(DetectEngineCtx *de_ctx,
         Signature *s, const char *sigstr, SignatureParser *parser, uint8_t addrs_direction)
 {
 #define MAX_SUBSTRINGS 30
@@ -1018,9 +1020,6 @@ void SigFree(Signature *s)
     }
     SigMatchFreeArrays(s);
 
-    DetectAddressHeadCleanup(&s->src);
-    DetectAddressHeadCleanup(&s->dst);
-
     if (s->sp != NULL) {
         DetectPortCleanupList(s->sp);
     }
@@ -1060,7 +1059,7 @@ static void SigBuildAddressMatchArray(Signature *s)
     /* source addresses */
     uint16_t cnt = 0;
     uint16_t idx = 0;
-    DetectAddress *da = s->src.ipv4_head;
+    DetectAddress *da = s->src->ipv4_head;
     for ( ; da != NULL; da = da->next) {
         cnt++;
     }
@@ -1070,7 +1069,7 @@ static void SigBuildAddressMatchArray(Signature *s)
             exit(EXIT_FAILURE);
         }
 
-        for (da = s->src.ipv4_head; da != NULL; da = da->next) {
+        for (da = s->src->ipv4_head; da != NULL; da = da->next) {
             s->addr_src_match4[idx].ip = ntohl(da->ip.addr_data32[0]);
             s->addr_src_match4[idx].ip2 = ntohl(da->ip2.addr_data32[0]);
             idx++;
@@ -1081,7 +1080,7 @@ static void SigBuildAddressMatchArray(Signature *s)
     /* destination addresses */
     cnt = 0;
     idx = 0;
-    da = s->dst.ipv4_head;
+    da = s->dst->ipv4_head;
     for ( ; da != NULL; da = da->next) {
         cnt++;
     }
@@ -1091,7 +1090,7 @@ static void SigBuildAddressMatchArray(Signature *s)
             exit(EXIT_FAILURE);
         }
 
-        for (da = s->dst.ipv4_head; da != NULL; da = da->next) {
+        for (da = s->dst->ipv4_head; da != NULL; da = da->next) {
             s->addr_dst_match4[idx].ip = ntohl(da->ip.addr_data32[0]);
             s->addr_dst_match4[idx].ip2 = ntohl(da->ip2.addr_data32[0]);
             idx++;
@@ -1102,7 +1101,7 @@ static void SigBuildAddressMatchArray(Signature *s)
     /* source addresses IPv6 */
     cnt = 0;
     idx = 0;
-    da = s->src.ipv6_head;
+    da = s->src->ipv6_head;
     for ( ; da != NULL; da = da->next) {
         cnt++;
     }
@@ -1112,7 +1111,7 @@ static void SigBuildAddressMatchArray(Signature *s)
             exit(EXIT_FAILURE);
         }
 
-        for (da = s->src.ipv6_head; da != NULL; da = da->next) {
+        for (da = s->src->ipv6_head; da != NULL; da = da->next) {
             s->addr_src_match6[idx].ip[0] = ntohl(da->ip.addr_data32[0]);
             s->addr_src_match6[idx].ip[1] = ntohl(da->ip.addr_data32[1]);
             s->addr_src_match6[idx].ip[2] = ntohl(da->ip.addr_data32[2]);
@@ -1129,7 +1128,7 @@ static void SigBuildAddressMatchArray(Signature *s)
     /* destination addresses IPv6 */
     cnt = 0;
     idx = 0;
-    da = s->dst.ipv6_head;
+    da = s->dst->ipv6_head;
     for ( ; da != NULL; da = da->next) {
         cnt++;
     }
@@ -1139,7 +1138,7 @@ static void SigBuildAddressMatchArray(Signature *s)
             exit(EXIT_FAILURE);
         }
 
-        for (da = s->dst.ipv6_head; da != NULL; da = da->next) {
+        for (da = s->dst->ipv6_head; da != NULL; da = da->next) {
             s->addr_dst_match6[idx].ip[0] = ntohl(da->ip.addr_data32[0]);
             s->addr_dst_match6[idx].ip[1] = ntohl(da->ip.addr_data32[1]);
             s->addr_dst_match6[idx].ip[2] = ntohl(da->ip.addr_data32[2]);

diff --git a/src/detect.h b/src/detect.h
index 807dcafc2cf3917a23b68d1552fa920d5655e5dc..e0f555674b68c964d29dc2a67ab3920559827b06 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -425,7 +425,7 @@ typedef struct Signature_ {
     DetectReference *references;
 
     /** address settings for this signature */
-    DetectAddressHead src, dst;
+    const DetectAddressHead *src, *dst;
 
     /* used at init to determine max dsize */
     SigMatch *dsize_sm;
@@ -677,6 +677,9 @@ typedef struct DetectEngineCtx_ {
     DetectPort *tcp_whitelist;
     DetectPort *udp_whitelist;
 
+    /** table for storing the string representation with the parsers result */
+    HashListTable *address_table;
+
 } DetectEngineCtx;
 
 /* Engine groups profiles (low, medium, high, custom) */

diff --git a/src/util-var.c b/src/util-var.c
index 40d4f7f31e830b00ac0185ea901610232298ebd1..dd7bbc132d919ec2a2d0f052edb3e82e74d57b1b 100644 (file)
--- a/src/util-var.c
+++ b/src/util-var.c
@@ -130,7 +130,7 @@ void GenericVarRemove(GenericVar **list, GenericVar *gv)
 }
 
 // Checks if a variable is already in a resolve list and if it's not, adds it.
-int AddVariableToResolveList(ResolvedVariablesList *list, char *var)
+int AddVariableToResolveList(ResolvedVariablesList *list, const char *var)
 {
     ResolvedVariable *p_item;
 

diff --git a/src/util-var.h b/src/util-var.h
index dc284725ffa250bd26484e36771f7967267df550..b95b59fd2b257ac54d0ff59e2a6a4bb75454f38d 100644 (file)
--- a/src/util-var.h
+++ b/src/util-var.h
@@ -70,7 +70,7 @@ void GenericVarFree(GenericVar *);
 void GenericVarAppend(GenericVar **, GenericVar *);
 void GenericVarRemove(GenericVar **, GenericVar *);
 
-int AddVariableToResolveList(ResolvedVariablesList *list, char *var);
+int AddVariableToResolveList(ResolvedVariablesList *list, const char *var);
 void CleanVariableResolveList(ResolvedVariablesList *var_list);
 
 #endif /* __UTIL_VAR_H__ */