From: Victor Julien Date: Fri, 19 Feb 2021 10:57:57 +0000 (+0100) Subject: detect: move init only array to init data X-Git-Tag: suricata-7.0.0-beta1~1487 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=86681c9d7c063cf28a73009d4839e55e69f05969;p=thirdparty%2Fsuricata.git detect: move init only array to init data --- diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index a7b5dac265..9b9c0da7c1 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -36,6 +36,7 @@ #include "detect-flowbits.h" #include "util-profiling.h" +#include "util-validate.h" void SigCleanSignatures(DetectEngineCtx *de_ctx) { @@ -651,6 +652,10 @@ static json_t *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigG uint32_t alproto_mpm_bufs[ALPROTO_MAX][max_buffer_type_id]; memset(alproto_mpm_bufs, 0, sizeof(alproto_mpm_bufs)); + DEBUG_VALIDATE_BUG_ON(sgh->init == NULL); + if (sgh->init == NULL) + return NULL; + json_t *js = json_object(); if (unlikely(js == NULL)) return NULL; @@ -661,8 +666,8 @@ static json_t *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigG const Signature *s; uint32_t x; - for (x = 0; x < sgh->sig_cnt; x++) { - s = sgh->match_array[x]; + for (x = 0; x < sgh->init->sig_cnt; x++) { + s = sgh->init->match_array[x]; if (s == NULL) continue; @@ -794,7 +799,7 @@ static json_t *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigG json_object_set_new(js, "rules", js_array); json_t *stats = json_object(); - json_object_set_new(stats, "total", json_integer(sgh->sig_cnt)); + json_object_set_new(stats, "total", json_integer(sgh->init->sig_cnt)); json_t *types = json_object(); json_object_set_new(types, "mpm", json_integer(mpm_cnt)); @@ -870,8 +875,7 @@ static json_t *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigG } json_object_set_new(js, "stats", stats); - if (sgh->init) - json_object_set_new(js, "whitelist", json_integer(sgh->init->whitelist)); + json_object_set_new(js, "whitelist", json_integer(sgh->init->whitelist)); return js; } @@ -928,16 +932,15 @@ static void RulesDumpGrouping(const DetectEngineCtx *de_ctx, } else if (p == IPPROTO_ICMP || p == IPPROTO_ICMPV6) { const char *name = (p == IPPROTO_ICMP) ? "icmpv4" : "icmpv6"; json_t *o = json_object(); - json_t *ts = json_object(); - json_t *tc = json_object(); - if (de_ctx->flow_gh[1].sgh[p]) { + json_t *ts = json_object(); json_t *group_ts = RulesGroupPrintSghStats( de_ctx, de_ctx->flow_gh[1].sgh[p], add_rules, add_mpm_stats); json_object_set_new(ts, "rulegroup", group_ts); json_object_set_new(o, "toserver", ts); } if (de_ctx->flow_gh[0].sgh[p]) { + json_t *tc = json_object(); json_t *group_tc = RulesGroupPrintSghStats( de_ctx, de_ctx->flow_gh[0].sgh[p], add_rules, add_mpm_stats); json_object_set_new(tc, "rulegroup", group_tc); @@ -1480,32 +1483,32 @@ static int PortGroupWhitelist(const DetectPort *a) int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b) { if (PortGroupWhitelist(a) && !PortGroupWhitelist(b)) { - SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", - a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), - b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); + SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", a->port, a->port2, + a->sh->init->sig_cnt, PortGroupWhitelist(a), b->port, b->port2, + b->sh->init->sig_cnt, PortGroupWhitelist(b)); return 1; } else if (!PortGroupWhitelist(a) && PortGroupWhitelist(b)) { - SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)", - a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), - b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); + SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)", a->port, a->port2, + a->sh->init->sig_cnt, PortGroupWhitelist(a), b->port, b->port2, + b->sh->init->sig_cnt, PortGroupWhitelist(b)); return 0; } else if (PortGroupWhitelist(a) > PortGroupWhitelist(b)) { - SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", - a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), - b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); + SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", a->port, a->port2, + a->sh->init->sig_cnt, PortGroupWhitelist(a), b->port, b->port2, + b->sh->init->sig_cnt, PortGroupWhitelist(b)); return 1; } else if (PortGroupWhitelist(a) == PortGroupWhitelist(b)) { - if (a->sh->sig_cnt > b->sh->sig_cnt) { - SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", - a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), - b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); + if (a->sh->init->sig_cnt > b->sh->init->sig_cnt) { + SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", a->port, + a->port2, a->sh->init->sig_cnt, PortGroupWhitelist(a), b->port, b->port2, + b->sh->init->sig_cnt, PortGroupWhitelist(b)); return 1; } } - SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)", - a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), - b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); + SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)", a->port, a->port2, + a->sh->init->sig_cnt, PortGroupWhitelist(a), b->port, b->port2, b->sh->init->sig_cnt, + PortGroupWhitelist(b)); return 0; } @@ -1821,9 +1824,6 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) SigGroupHeadBuildNonPrefilterArray(de_ctx, sgh); - SigGroupHeadInitDataFree(sgh->init); - sgh->init = NULL; - sgh->id = idx; cnt++; } @@ -1836,10 +1836,6 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) * signature not decode event only. */ } - /* cleanup the hashes now since we won't need them - * after the initialization phase. */ - SigGroupHeadHashFree(de_ctx); - int dump_grouping = 0; (void)ConfGetBool("detect.profiling.grouping.dump-to-disk", &dump_grouping); @@ -1852,6 +1848,17 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) RulesDumpGrouping(de_ctx, add_rules, add_mpm_stats); } + for (uint32_t idx = 0; idx < de_ctx->sgh_array_cnt; idx++) { + SigGroupHead *sgh = de_ctx->sgh_array[idx]; + if (sgh == NULL) + continue; + SigGroupHeadInitDataFree(sgh->init); + sgh->init = NULL; + } + /* cleanup the hashes now since we won't need them + * after the initialization phase. */ + SigGroupHeadHashFree(de_ctx); + #ifdef PROFILING SCProfilingSghInitCounters(de_ctx); #endif diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index 2ea28e7775..96bb001151 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -1344,8 +1344,8 @@ MpmStore *MpmStorePrepareBuffer(DetectEngineCtx *de_ctx, SigGroupHead *sgh, break; } - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; @@ -1436,8 +1436,8 @@ static MpmStore *MpmStorePrepareBufferAppLayer(DetectEngineCtx *de_ctx, am->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient", am->sm_list); - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; @@ -1513,8 +1513,8 @@ static MpmStore *MpmStorePrepareBufferPkt(DetectEngineCtx *de_ctx, SCLogDebug("handling %s for list %d", am->name, am->sm_list); - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; @@ -1576,8 +1576,8 @@ static void SetRawReassemblyFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh) const Signature *s = NULL; uint32_t sig; - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; diff --git a/src/detect-engine-prefilter-common.c b/src/detect-engine-prefilter-common.c index 9bc8810fd1..15df839b0c 100644 --- a/src/detect-engine-prefilter-common.c +++ b/src/detect-engine-prefilter-common.c @@ -118,8 +118,8 @@ SetupEngineForPacketHeader(DetectEngineCtx *de_ctx, SigGroupHead *sgh, return -1; } - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; if (s->init_data->prefilter_sm == NULL || s->init_data->prefilter_sm->type != sm_type) @@ -230,8 +230,8 @@ SetupEngineForPacketHeaderPrefilterPacketU8HashCtx(DetectEngineCtx *de_ctx, return 0; } - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; if (s->init_data->prefilter_sm == NULL || s->init_data->prefilter_sm->type != sm_type) @@ -357,8 +357,8 @@ static int PrefilterSetupPacketHeaderCommon(DetectEngineCtx *de_ctx, if (hash_table == NULL) return -1; - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; if (s->init_data->prefilter_sm == NULL || s->init_data->prefilter_sm->type != sm_type) diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 5e7c55cdae..b1140a35a2 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -57,6 +57,10 @@ int SigGroupHeadClearSigs(SigGroupHead *); void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid) { + if (sghid->match_array != NULL) { + SCFree(sghid->match_array); + sghid->match_array = NULL; + } if (sghid->sig_array != NULL) { SCFree(sghid->sig_array); sghid->sig_array = NULL; @@ -158,11 +162,6 @@ void SigGroupHeadFree(const DetectEngineCtx *de_ctx, SigGroupHead *sgh) SCLogDebug("sgh %p", sgh); - if (sgh->match_array != NULL) { - SCFree(sgh->match_array); - sgh->match_array = NULL; - } - if (sgh->non_pf_other_store_array != NULL) { SCFree(sgh->non_pf_other_store_array); sgh->non_pf_other_store_array = NULL; @@ -175,8 +174,6 @@ void SigGroupHeadFree(const DetectEngineCtx *de_ctx, SigGroupHead *sgh) sgh->non_pf_syn_store_cnt = 0; } - sgh->sig_cnt = 0; - if (sgh->init != NULL) { SigGroupHeadInitDataFree(sgh->init); sgh->init = NULL; @@ -372,7 +369,7 @@ int SigGroupHeadClearSigs(SigGroupHead *sgh) if (sgh->init->sig_array != NULL) memset(sgh->init->sig_array, 0, sgh->init->sig_size); - sgh->sig_cnt = 0; + sgh->init->sig_cnt = 0; return 0; } @@ -426,10 +423,10 @@ void SigGroupHeadSetSigCnt(SigGroupHead *sgh, uint32_t max_idx) { uint32_t sig; - sgh->sig_cnt = 0; + sgh->init->sig_cnt = 0; for (sig = 0; sig < max_idx + 1; sig++) { if (sgh->init->sig_array[sig / 8] & (1 << (sig % 8))) - sgh->sig_cnt++; + sgh->init->sig_cnt++; } return; @@ -494,13 +491,13 @@ int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh, if (sgh == NULL) return 0; - BUG_ON(sgh->match_array != NULL); + BUG_ON(sgh->init->match_array != NULL); - sgh->match_array = SCMalloc(sgh->sig_cnt * sizeof(Signature *)); - if (sgh->match_array == NULL) + sgh->init->match_array = SCMalloc(sgh->init->sig_cnt * sizeof(Signature *)); + if (sgh->init->match_array == NULL) return -1; - memset(sgh->match_array,0, sgh->sig_cnt * sizeof(Signature *)); + memset(sgh->init->match_array, 0, sgh->init->sig_cnt * sizeof(Signature *)); for (sig = 0; sig < max_idx + 1; sig++) { if (!(sgh->init->sig_array[(sig / 8)] & (1 << (sig % 8))) ) @@ -510,7 +507,7 @@ int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh, if (s == NULL) continue; - sgh->match_array[idx] = s; + sgh->init->match_array[idx] = s; idx++; } @@ -532,8 +529,8 @@ void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh) if (sgh == NULL) return; - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; @@ -560,8 +557,8 @@ void SigGroupHeadSetFilesizeFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh) if (sgh == NULL) return; - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; @@ -588,8 +585,8 @@ void SigGroupHeadSetFileHashFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh) if (sgh == NULL) return; - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; @@ -629,8 +626,8 @@ void SigGroupHeadSetFilestoreCount(DetectEngineCtx *de_ctx, SigGroupHead *sgh) if (sgh == NULL) return; - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; @@ -657,8 +654,8 @@ int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sg BUG_ON(sgh->non_pf_other_store_array != NULL); - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; @@ -688,8 +685,8 @@ int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sg memset(sgh->non_pf_syn_store_array, 0, non_pf_syn * sizeof(SignatureNonPrefilterStore)); } - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; + for (sig = 0; sig < sgh->init->sig_cnt; sig++) { + s = sgh->init->match_array[sig]; if (s == NULL) continue; @@ -865,7 +862,7 @@ static int SigGroupHeadTest06(void) SigGroupHeadSetSigCnt(sh, 4); - result &= (sh->sig_cnt == 3); + result &= (sh->init->sig_cnt == 3); result &= (SigGroupHeadContainsSigId(de_ctx, sh, 1) == 1); result &= (SigGroupHeadContainsSigId(de_ctx, sh, 2) == 0); result &= (SigGroupHeadContainsSigId(de_ctx, sh, 3) == 1); @@ -949,7 +946,7 @@ static int SigGroupHeadTest07(void) SigGroupHeadSetSigCnt(sh, 4); - result &= (sh->sig_cnt == 3); + result &= (sh->init->sig_cnt == 3); result &= (SigGroupHeadContainsSigId(de_ctx, sh, 1) == 1); result &= (SigGroupHeadContainsSigId(de_ctx, sh, 2) == 0); result &= (SigGroupHeadContainsSigId(de_ctx, sh, 3) == 1); @@ -958,7 +955,7 @@ static int SigGroupHeadTest07(void) SigGroupHeadClearSigs(sh); - result &= (sh->sig_cnt == 0); + result &= (sh->init->sig_cnt == 0); result &= (SigGroupHeadContainsSigId(de_ctx, sh, 1) == 0); result &= (SigGroupHeadContainsSigId(de_ctx, sh, 2) == 0); result &= (SigGroupHeadContainsSigId(de_ctx, sh, 3) == 0); @@ -1041,7 +1038,7 @@ static int SigGroupHeadTest08(void) SigGroupHeadSetSigCnt(src_sh, 4); - result &= (src_sh->sig_cnt == 3); + result &= (src_sh->init->sig_cnt == 3); result &= (SigGroupHeadContainsSigId(de_ctx, src_sh, 1) == 1); result &= (SigGroupHeadContainsSigId(de_ctx, src_sh, 2) == 0); result &= (SigGroupHeadContainsSigId(de_ctx, src_sh, 3) == 1); @@ -1052,7 +1049,7 @@ static int SigGroupHeadTest08(void) SigGroupHeadSetSigCnt(dst_sh, 4); - result &= (dst_sh->sig_cnt == 3); + result &= (dst_sh->init->sig_cnt == 3); result &= (SigGroupHeadContainsSigId(de_ctx, dst_sh, 1) == 1); result &= (SigGroupHeadContainsSigId(de_ctx, dst_sh, 2) == 0); result &= (SigGroupHeadContainsSigId(de_ctx, dst_sh, 3) == 1); @@ -1136,9 +1133,9 @@ static int SigGroupHeadTest09(void) SigGroupHeadSetSigCnt(sh, 4); SigGroupHeadBuildMatchArray(de_ctx, sh, 4); - result &= (sh->match_array[0] == de_ctx->sig_list); - result &= (sh->match_array[1] == de_ctx->sig_list->next->next); - result &= (sh->match_array[2] == de_ctx->sig_list->next->next->next->next); + result &= (sh->init->match_array[0] == de_ctx->sig_list); + result &= (sh->init->match_array[1] == de_ctx->sig_list->next->next); + result &= (sh->init->match_array[2] == de_ctx->sig_list->next->next->next->next); SigGroupHeadFree(de_ctx, sh); diff --git a/src/detect.h b/src/detect.h index a9464bf578..ce29e83e47 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1333,6 +1333,12 @@ typedef struct SigGroupHeadInitData_ { PrefilterEngineList *payload_engines; PrefilterEngineList *tx_engines; + /** number of sigs in this group */ + SigIntId sig_cnt; + + /** Array with sig ptrs... size is sig_cnt * sizeof(Signature *) */ + Signature **match_array; + /* port ptr */ struct DetectPort_ *port; } SigGroupHeadInitData; @@ -1342,9 +1348,6 @@ typedef struct SigGroupHead_ { uint32_t flags; /* coccinelle: SigGroupHead:flags:SIG_GROUP_HEAD_ */ - /* number of sigs in this head */ - SigIntId sig_cnt; - /* non prefilter list excluding SYN rules */ uint32_t non_pf_other_store_cnt; uint32_t non_pf_syn_store_cnt; @@ -1362,9 +1365,6 @@ typedef struct SigGroupHead_ { PrefilterEngine *payload_engines; PrefilterEngine *tx_engines; - /** Array with sig ptrs... size is sig_cnt * sizeof(Signature *) */ - Signature **match_array; - /* ptr to our init data we only use at... init :) */ SigGroupHeadInitData *init;