From: Victor Julien Date: Fri, 10 Jun 2016 10:32:25 +0000 (+0200) Subject: detect: optimize rule address parsing X-Git-Tag: suricata-3.2beta1~338 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=215d0d54c7771fdc735bf05302716ebf041e678b;p=thirdparty%2Fsuricata.git detect: optimize rule address parsing Many rules have the same address vars, so instead of parsing them each time use a hash to store the string and the parsed result. Rules now reference the stored result in the hash table. --- diff --git a/src/detect-engine-address.c b/src/detect-engine-address.c index 8c07d08fc8..60964cd9c8 100644 --- a/src/detect-engine-address.c +++ b/src/detect-engine-address.c @@ -815,7 +815,7 @@ error: */ static int DetectAddressParse2(const DetectEngineCtx *de_ctx, DetectAddressHead *gh, DetectAddressHead *ghn, - char *s, int negate, ResolvedVariablesList *var_list) + const char *s, int negate, ResolvedVariablesList *var_list) { size_t x = 0; size_t u = 0; @@ -1023,6 +1023,8 @@ static int DetectAddressParse2(const DetectEngineCtx *de_ctx, if (DetectAddressParse2(de_ctx, gh, ghn, temp_rule_var_address, (negate + n_set) % 2, var_list) < 0) { SCLogDebug("DetectAddressParse2 hates us"); + if (temp_rule_var_address != rule_var_address) + SCFree(temp_rule_var_address); goto error; } d_set = 0; @@ -1345,6 +1347,98 @@ int DetectAddressTestConfVars(void) return -1; } +#include "util-hash-lookup3.h" + +typedef struct DetectAddressMap_ { + char *string; + DetectAddressHead *address; +} DetectAddressMap; + +static uint32_t DetectAddressMapHashFunc(HashListTable *ht, void *data, uint16_t datalen) +{ + const DetectAddressMap *map = (DetectAddressMap *)data; + uint32_t hash = 0; + + hash = hashlittle_safe(map->string, strlen(map->string), 0); + hash %= ht->array_size; + + return hash; +} + +static char DetectAddressMapCompareFunc(void *data1, uint16_t len1, void *data2, + uint16_t len2) +{ + DetectAddressMap *map1 = (DetectAddressMap *)data1; + DetectAddressMap *map2 = (DetectAddressMap *)data2; + + + int r = (strcmp(map1->string, map2->string) == 0); + return r; +} + +static void DetectAddressMapFreeFunc(void *data) +{ + DetectAddressMap *map = (DetectAddressMap *)data; + if (map != NULL) { + DetectAddressHeadFree(map->address); + SCFree(map->string); + } + SCFree(map); +} + +int DetectAddressMapInit(DetectEngineCtx *de_ctx) +{ + de_ctx->address_table = HashListTableInit(4096, DetectAddressMapHashFunc, + DetectAddressMapCompareFunc, + DetectAddressMapFreeFunc); + if (de_ctx->address_table == NULL) + return -1; + + return 0; +} + +void DetectAddressMapFree(DetectEngineCtx *de_ctx) +{ + if (de_ctx->address_table == NULL) + return; + + HashListTableFree(de_ctx->address_table); + de_ctx->address_table = NULL; + return; +} + +int DetectAddressMapAdd(DetectEngineCtx *de_ctx, const char *string, + DetectAddressHead *address) +{ + DetectAddressMap *map = SCCalloc(1, sizeof(*map)); + if (map == NULL) + return -1; + + map->string = SCStrdup(string); + if (map->string == NULL) { + SCFree(map); + return -1; + } + map->address = address; + + BUG_ON(HashListTableAdd(de_ctx->address_table, (void *)map, 0) != 0); + return 0; +} + +const DetectAddressHead *DetectAddressMapLookup(DetectEngineCtx *de_ctx, + const char *string) +{ + DetectAddressMap map = { (char *)string, NULL }; + + const DetectAddressMap *res = HashListTableLookup(de_ctx->address_table, + &map, 0); + if (res == NULL) + return NULL; + else { + return (const DetectAddressHead *)res->address; + } +} + /** * \brief Parses an address group sent as a character string and updates the * DetectAddressHead sent as the argument with the relevant address @@ -1358,7 +1452,7 @@ int DetectAddressTestConfVars(void) * \retval -1 On failure. */ int DetectAddressParse(const DetectEngineCtx *de_ctx, - DetectAddressHead *gh, char *str) + DetectAddressHead *gh, const char *str) { int r; DetectAddressHead *ghn = NULL; @@ -1401,6 +1495,31 @@ error: return -1; } +const DetectAddressHead *DetectParseAddress(DetectEngineCtx *de_ctx, + const char *string) +{ + const DetectAddressHead *h = DetectAddressMapLookup(de_ctx, string); + if (h != NULL) { + SCLogDebug("found: %s :: %p", string, h); + return h; + } + + SCLogDebug("%s not found", string); + + DetectAddressHead *head = DetectAddressHeadInit(); + if (head == NULL) + return NULL; + + if (DetectAddressParse(de_ctx, head, string) == -1) + { + DetectAddressHeadFree(head); + return NULL; + } + + DetectAddressMapAdd((DetectEngineCtx *)de_ctx, string, head); + return head; +} + /** * \brief Returns a new instance of DetectAddressHead. * diff --git a/src/detect-engine-address.h b/src/detect-engine-address.h index 3b84f22577..dcbde2e427 100644 --- a/src/detect-engine-address.h +++ b/src/detect-engine-address.h @@ -32,7 +32,7 @@ void DetectAddressHeadFree(DetectAddressHead *); void DetectAddressHeadCleanup(DetectAddressHead *); int DetectAddressParseString(DetectAddress *, char *); -int DetectAddressParse(const DetectEngineCtx *, DetectAddressHead *, char *); +int DetectAddressParse(const DetectEngineCtx *, DetectAddressHead *, const char *); DetectAddress *DetectAddressInit(void); void DetectAddressFree(DetectAddress *); @@ -59,4 +59,9 @@ int DetectAddressTestConfVars(void); void DetectAddressTests(void); +int DetectAddressMapInit(DetectEngineCtx *de_ctx); +void DetectAddressMapFree(DetectEngineCtx *de_ctx); +const DetectAddressHead *DetectParseAddress(DetectEngineCtx *de_ctx, + const char *string); + #endif /* __DETECT_ADDRESS_H__ */ diff --git a/src/detect-engine.c b/src/detect-engine.c index e108258d9c..d7aee2242a 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -854,6 +854,7 @@ static DetectEngineCtx *DetectEngineCtxInitReal(int minimal, const char *prefix) ThresholdHashInit(de_ctx); VariableNameInitHash(de_ctx); DetectParseDupSigHashInit(de_ctx); + DetectAddressMapInit(de_ctx); /* init iprep... ignore errors for now */ (void)SRepInit(de_ctx); @@ -964,6 +965,8 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx) DetectEngineCtxFreeThreadKeywordData(de_ctx); SRepDestroy(de_ctx); + DetectAddressMapFree(de_ctx); + /* if we have a config prefix, remove the config from the tree */ if (strlen(de_ctx->config_prefix) > 0) { /* remove config */ diff --git a/src/detect-parse.c b/src/detect-parse.c index f78b6033c7..9e970e3896 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -624,7 +624,7 @@ error: * * \retval 0 ok, -1 error */ -int SigParseAddress(const DetectEngineCtx *de_ctx, +static int SigParseAddress(DetectEngineCtx *de_ctx, Signature *s, const char *addrstr, char flag) { SCLogDebug("Address Group \"%s\" to be parsed now", addrstr); @@ -634,13 +634,15 @@ int SigParseAddress(const DetectEngineCtx *de_ctx, if (strcasecmp(addrstr, "any") == 0) s->flags |= SIG_FLAG_SRC_ANY; - if (DetectAddressParse(de_ctx, &s->src, (char *)addrstr) < 0) + s->src = DetectParseAddress(de_ctx, addrstr); + if (s->src == NULL) goto error; } else { if (strcasecmp(addrstr, "any") == 0) s->flags |= SIG_FLAG_DST_ANY; - if (DetectAddressParse(de_ctx, &s->dst, (char *)addrstr) < 0) + s->dst = DetectParseAddress(de_ctx, addrstr); + if (s->dst == NULL) goto error; } @@ -808,7 +810,7 @@ int SigParseAction(Signature *s, const char *action) * \internal * \brief split a signature string into a few blocks for further parsing */ -static int SigParseBasics(const DetectEngineCtx *de_ctx, +static int SigParseBasics(DetectEngineCtx *de_ctx, Signature *s, const char *sigstr, SignatureParser *parser, uint8_t addrs_direction) { #define MAX_SUBSTRINGS 30 @@ -1018,9 +1020,6 @@ void SigFree(Signature *s) } SigMatchFreeArrays(s); - DetectAddressHeadCleanup(&s->src); - DetectAddressHeadCleanup(&s->dst); - if (s->sp != NULL) { DetectPortCleanupList(s->sp); } @@ -1060,7 +1059,7 @@ static void SigBuildAddressMatchArray(Signature *s) /* source addresses */ uint16_t cnt = 0; uint16_t idx = 0; - DetectAddress *da = s->src.ipv4_head; + DetectAddress *da = s->src->ipv4_head; for ( ; da != NULL; da = da->next) { cnt++; } @@ -1070,7 +1069,7 @@ static void SigBuildAddressMatchArray(Signature *s) exit(EXIT_FAILURE); } - for (da = s->src.ipv4_head; da != NULL; da = da->next) { + for (da = s->src->ipv4_head; da != NULL; da = da->next) { s->addr_src_match4[idx].ip = ntohl(da->ip.addr_data32[0]); s->addr_src_match4[idx].ip2 = ntohl(da->ip2.addr_data32[0]); idx++; @@ -1081,7 +1080,7 @@ static void SigBuildAddressMatchArray(Signature *s) /* destination addresses */ cnt = 0; idx = 0; - da = s->dst.ipv4_head; + da = s->dst->ipv4_head; for ( ; da != NULL; da = da->next) { cnt++; } @@ -1091,7 +1090,7 @@ static void SigBuildAddressMatchArray(Signature *s) exit(EXIT_FAILURE); } - for (da = s->dst.ipv4_head; da != NULL; da = da->next) { + for (da = s->dst->ipv4_head; da != NULL; da = da->next) { s->addr_dst_match4[idx].ip = ntohl(da->ip.addr_data32[0]); s->addr_dst_match4[idx].ip2 = ntohl(da->ip2.addr_data32[0]); idx++; @@ -1102,7 +1101,7 @@ static void SigBuildAddressMatchArray(Signature *s) /* source addresses IPv6 */ cnt = 0; idx = 0; - da = s->src.ipv6_head; + da = s->src->ipv6_head; for ( ; da != NULL; da = da->next) { cnt++; } @@ -1112,7 +1111,7 @@ static void SigBuildAddressMatchArray(Signature *s) exit(EXIT_FAILURE); } - for (da = s->src.ipv6_head; da != NULL; da = da->next) { + for (da = s->src->ipv6_head; da != NULL; da = da->next) { s->addr_src_match6[idx].ip[0] = ntohl(da->ip.addr_data32[0]); s->addr_src_match6[idx].ip[1] = ntohl(da->ip.addr_data32[1]); s->addr_src_match6[idx].ip[2] = ntohl(da->ip.addr_data32[2]); @@ -1129,7 +1128,7 @@ static void SigBuildAddressMatchArray(Signature *s) /* destination addresses IPv6 */ cnt = 0; idx = 0; - da = s->dst.ipv6_head; + da = s->dst->ipv6_head; for ( ; da != NULL; da = da->next) { cnt++; } @@ -1139,7 +1138,7 @@ static void SigBuildAddressMatchArray(Signature *s) exit(EXIT_FAILURE); } - for (da = s->dst.ipv6_head; da != NULL; da = da->next) { + for (da = s->dst->ipv6_head; da != NULL; da = da->next) { s->addr_dst_match6[idx].ip[0] = ntohl(da->ip.addr_data32[0]); s->addr_dst_match6[idx].ip[1] = ntohl(da->ip.addr_data32[1]); s->addr_dst_match6[idx].ip[2] = ntohl(da->ip.addr_data32[2]); diff --git a/src/detect.h b/src/detect.h index 807dcafc2c..e0f555674b 100644 --- a/src/detect.h +++ b/src/detect.h @@ -425,7 +425,7 @@ typedef struct Signature_ { DetectReference *references; /** address settings for this signature */ - DetectAddressHead src, dst; + const DetectAddressHead *src, *dst; /* used at init to determine max dsize */ SigMatch *dsize_sm; @@ -677,6 +677,9 @@ typedef struct DetectEngineCtx_ { DetectPort *tcp_whitelist; DetectPort *udp_whitelist; + /** table for storing the string representation with the parsers result */ + HashListTable *address_table; + } DetectEngineCtx; /* Engine groups profiles (low, medium, high, custom) */ diff --git a/src/util-var.c b/src/util-var.c index 40d4f7f31e..dd7bbc132d 100644 --- a/src/util-var.c +++ b/src/util-var.c @@ -130,7 +130,7 @@ void GenericVarRemove(GenericVar **list, GenericVar *gv) } // Checks if a variable is already in a resolve list and if it's not, adds it. -int AddVariableToResolveList(ResolvedVariablesList *list, char *var) +int AddVariableToResolveList(ResolvedVariablesList *list, const char *var) { ResolvedVariable *p_item; diff --git a/src/util-var.h b/src/util-var.h index dc284725ff..b95b59fd2b 100644 --- a/src/util-var.h +++ b/src/util-var.h @@ -70,7 +70,7 @@ void GenericVarFree(GenericVar *); void GenericVarAppend(GenericVar **, GenericVar *); void GenericVarRemove(GenericVar **, GenericVar *); -int AddVariableToResolveList(ResolvedVariablesList *list, char *var); +int AddVariableToResolveList(ResolvedVariablesList *list, const char *var); void CleanVariableResolveList(ResolvedVariablesList *var_list); #endif /* __UTIL_VAR_H__ */