From: Shivani Bhardwaj Date: Thu, 1 Feb 2024 11:14:33 +0000 (+0530) Subject: detect/engine: set max sig ID per SGH X-Git-Tag: suricata-8.0.0-beta1~1800 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=395c74d81eec59b07181c0015c6760845a9494e7;p=thirdparty%2Fsuricata.git detect/engine: set max sig ID per SGH Present scenario ---------------- Currently, as a part of setting signature count per SGH, a max_idx is passed which could be as high as the highest signature number (internal ID). Issue ----- Not every SGH needs to evaluate all the signatures while setting the signature count or while creating the match_array. In a nonideal scenario, when say, there are 2 SGHs and one SGH has 2 signatures and the other one has 60k, given the current scheme of evaluating max_idx, the max_idx will be set to 60k, and this shall later be passed on to SigGroupHeadSetSigCnt or SigGroupHeadBuildMatchArra which shall traverse over all the 60k sigs for either SGHs. Other info ---------- This is a very fast operation as the internal arithmetic is done bitwise. Patch ----- The functions SigGroupHeadSetSigCnt and SigGroupHeadBuildMatchArray can be optimized by storing the max signature id (internal) per SGH (which also seemed to be the initial intention as per fn comments). As a result of this, the sig_array is only walked up until the max sig id of that respective SGH. --- diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index 604164b21c..b99776bd13 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -1009,8 +1009,8 @@ static int RulesGroupByProto(DetectEngineCtx *de_ctx) if (lookup_sgh == NULL) { SCLogDebug("proto group %d sgh %p is the original", p, sgh_ts[p]); - SigGroupHeadSetSigCnt(sgh_ts[p], max_idx); - SigGroupHeadBuildMatchArray(de_ctx, sgh_ts[p], max_idx); + SigGroupHeadSetSigCnt(sgh_ts[p], 0); + SigGroupHeadBuildMatchArray(de_ctx, sgh_ts[p], 0); SigGroupHeadHashAdd(de_ctx, sgh_ts[p]); SigGroupHeadStore(de_ctx, sgh_ts[p]); @@ -1041,8 +1041,8 @@ static int RulesGroupByProto(DetectEngineCtx *de_ctx) if (lookup_sgh == NULL) { SCLogDebug("proto group %d sgh %p is the original", p, sgh_tc[p]); - SigGroupHeadSetSigCnt(sgh_tc[p], max_idx); - SigGroupHeadBuildMatchArray(de_ctx, sgh_tc[p], max_idx); + SigGroupHeadSetSigCnt(sgh_tc[p], 0); + SigGroupHeadBuildMatchArray(de_ctx, sgh_tc[p], 0); SigGroupHeadHashAdd(de_ctx, sgh_tc[p]); SigGroupHeadStore(de_ctx, sgh_tc[p]); @@ -1129,7 +1129,8 @@ static int RuleSetWhitelist(Signature *s) return wl; } -int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx); +int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, + uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *)); int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b); static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, uint8_t ipproto, uint32_t direction) @@ -1223,7 +1224,7 @@ static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, uint8_t ipproto, u DetectPort *newlist = NULL; uint16_t groupmax = (direction == SIG_FLAG_TOCLIENT) ? de_ctx->max_uniq_toclient_groups : de_ctx->max_uniq_toserver_groups; - CreateGroupedPortList(de_ctx, list, &newlist, groupmax, CreateGroupedPortListCmpCnt, max_idx); + CreateGroupedPortList(de_ctx, list, &newlist, groupmax, CreateGroupedPortListCmpCnt); list = newlist; /* step 4: deduplicate the SGH's */ @@ -1243,8 +1244,8 @@ static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, uint8_t ipproto, u if (lookup_sgh == NULL) { SCLogDebug("port group %p sgh %p is the original", iter, iter->sh); - SigGroupHeadSetSigCnt(iter->sh, max_idx); - SigGroupHeadBuildMatchArray(de_ctx, iter->sh, max_idx); + SigGroupHeadSetSigCnt(iter->sh, 0); + SigGroupHeadBuildMatchArray(de_ctx, iter->sh, 0); SigGroupHeadSetProtoAndDirection(iter->sh, ipproto, direction); SigGroupHeadHashAdd(de_ctx, iter->sh); SigGroupHeadStore(de_ctx, iter->sh); @@ -1541,7 +1542,8 @@ int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b) * The joingr is meant to be a catch all. * */ -int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx) +int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, + uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *)) { DetectPort *tmplist = NULL, *joingr = NULL; char insert = 0; @@ -1560,8 +1562,7 @@ int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, Detect list->next = NULL; groups++; - - SigGroupHeadSetSigCnt(list->sh, max_idx); + SigGroupHeadSetSigCnt(list->sh, 0); /* insert it */ DetectPort *tmpgr = tmplist, *prevtmpgr = NULL; diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 4fadaac5c0..9bc992cb89 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -347,7 +347,7 @@ int SigGroupHeadAppendSig(const DetectEngineCtx *de_ctx, SigGroupHead **sgh, /* enable the sig in the bitarray */ (*sgh)->init->sig_array[s->num / 8] |= 1 << (s->num % 8); - + (*sgh)->init->max_sig_id = MAX(s->num, (*sgh)->init->max_sig_id); return 0; error: @@ -405,6 +405,8 @@ int SigGroupHeadCopySigs(DetectEngineCtx *de_ctx, SigGroupHead *src, SigGroupHea if (src->init->score) (*dst)->init->score = MAX((*dst)->init->score, src->init->score); + if (src->init->max_sig_id) + (*dst)->init->max_sig_id = MAX((*dst)->init->max_sig_id, src->init->max_sig_id); return 0; error: @@ -422,9 +424,9 @@ error: void SigGroupHeadSetSigCnt(SigGroupHead *sgh, uint32_t max_idx) { uint32_t sig; - + sgh->init->max_sig_id = MAX(max_idx, sgh->init->max_sig_id); sgh->init->sig_cnt = 0; - for (sig = 0; sig < max_idx + 1; sig++) { + for (sig = 0; sig < sgh->init->max_sig_id + 1; sig++) { if (sgh->init->sig_array[sig / 8] & (1 << (sig % 8))) sgh->init->sig_cnt++; } @@ -492,12 +494,13 @@ int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh, return 0; BUG_ON(sgh->init->match_array != NULL); + sgh->init->max_sig_id = MAX(sgh->init->max_sig_id, max_idx); sgh->init->match_array = SCCalloc(sgh->init->sig_cnt, sizeof(Signature *)); if (sgh->init->match_array == NULL) return -1; - for (sig = 0; sig < max_idx + 1; sig++) { + for (sig = 0; sig < sgh->init->max_sig_id + 1; sig++) { if (!(sgh->init->sig_array[(sig / 8)] & (1 << (sig % 8))) ) continue; diff --git a/src/detect.h b/src/detect.h index e7cc0dfe93..0707d8a5b2 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1419,6 +1419,7 @@ typedef struct SigGroupHeadInitData_ { uint8_t protos[256]; /**< proto(s) this sgh is for */ uint32_t direction; /**< set to SIG_FLAG_TOSERVER, SIG_FLAG_TOCLIENT or both */ int score; /**< try to make this group a unique one */ + uint32_t max_sig_id; /**< max signature idx for this sgh */ MpmCtx **app_mpms; MpmCtx **pkt_mpms;