detect: add mask check prefilter for non mpm list

Add mask array for non_mpm sigs, so that we can exclude many sigs before
we merge sort.

Shows 50% less non mpm sigs inspected on average.

diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c
index 3b8a9ed3bf6a6fab380e6d18e4988c26668c3311..5f3a9f14ab8a37befc6c033048ba04786485b254 100644 (file)
--- a/src/detect-engine-siggroup.c
+++ b/src/detect-engine-siggroup.c
@@ -198,6 +198,12 @@ void SigGroupHeadFree(SigGroupHead *sgh)
         sgh->non_mpm_id_cnt = 0;
     }
 
+    if (sgh->non_mpm_mask_array != NULL) {
+        SCFree(sgh->non_mpm_mask_array);
+        sgh->non_mpm_mask_array = NULL;
+        sgh->non_mpm_mask_cnt = 0;
+    }
+
     sgh->sig_cnt = 0;
 
     if (sgh->init != NULL) {
@@ -1700,6 +1706,7 @@ int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         return 0;
 
     BUG_ON(sgh->non_mpm_id_array != NULL);
+    BUG_ON(sgh->non_mpm_mask_array != NULL);
 
     for (sig = 0; sig < sgh->sig_cnt; sig++) {
         s = sgh->match_array[sig];
@@ -1721,6 +1728,10 @@ int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
     BUG_ON(sgh->non_mpm_id_array == NULL);
     memset(sgh->non_mpm_id_array, 0, non_mpm * sizeof(SigIntId));
 
+    sgh->non_mpm_mask_array = SCMalloc(non_mpm * sizeof(SignatureMask));
+    BUG_ON(sgh->non_mpm_mask_array == NULL);
+    memset(sgh->non_mpm_mask_array, 0, non_mpm * sizeof(SignatureMask));
+
     for (sig = 0; sig < sgh->sig_cnt; sig++) {
         s = sgh->match_array[sig];
         if (s == NULL)
@@ -1729,9 +1740,11 @@ int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         if (s->mpm_sm == NULL) {
             BUG_ON(sgh->non_mpm_id_cnt >= non_mpm);
             sgh->non_mpm_id_array[sgh->non_mpm_id_cnt++] = s->num;
+            sgh->non_mpm_mask_array[sgh->non_mpm_mask_cnt++] = s->mask;
         } else if (s->flags & (SIG_FLAG_MPM_PACKET_NEG|SIG_FLAG_MPM_STREAM_NEG|SIG_FLAG_MPM_APPLAYER_NEG)) {
             BUG_ON(sgh->non_mpm_id_cnt >= non_mpm);
             sgh->non_mpm_id_array[sgh->non_mpm_id_cnt++] = s->num;
+            sgh->non_mpm_mask_array[sgh->non_mpm_mask_cnt++] = s->mask;
         }
     }
     return 0;

diff --git a/src/detect-engine.c b/src/detect-engine.c
index f2247b5d6b86fd59ee7eccc0a233ec2181a607b2..8f8cc2095dad5ba4209db3c4fce39575fc92ca95 100644 (file)
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -1361,6 +1361,8 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
                                                       SC_PERF_TYPE_UINT64, "NULL");
     uint16_t counter_nonmpm_list = SCPerfTVRegisterAvgCounter("detect.nonmpm_list", tv,
                                                       SC_PERF_TYPE_UINT64, "NULL");
+    uint16_t counter_fnonmpm_list = SCPerfTVRegisterAvgCounter("detect.fnonmpm_list", tv,
+                                                      SC_PERF_TYPE_UINT64, "NULL");
     uint16_t counter_match_list = SCPerfTVRegisterAvgCounter("detect.match_list", tv,
                                                       SC_PERF_TYPE_UINT64, "NULL");
 #endif
@@ -1377,6 +1379,9 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
     det_ctx->tv = tv;
     det_ctx->de_ctx = de_ctx;
 
+    det_ctx->non_mpm_id_array =  SCCalloc(32000, sizeof(SigIntId)); // TODO proper size or dynamicly grow
+    BUG_ON(det_ctx->non_mpm_id_array == NULL);
+
     if (ThreadCtxDoInit(de_ctx, det_ctx) != TM_ECODE_OK)
         return TM_ECODE_FAILED;
 
@@ -1385,6 +1390,7 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
 #ifdef PROFILING
     det_ctx->counter_mpm_list = counter_mpm_list;
     det_ctx->counter_nonmpm_list = counter_nonmpm_list;
+    det_ctx->counter_fnonmpm_list = counter_fnonmpm_list;
     det_ctx->counter_match_list = counter_match_list;
 #endif
 
@@ -1458,6 +1464,9 @@ TmEcode DetectEngineThreadCtxDeinit(ThreadVars *tv, void *data)
         PmqFree(&det_ctx->smsg_pmq[i]);
     }
 
+    if (det_ctx->non_mpm_id_array != NULL)
+        SCFree(det_ctx->non_mpm_id_array);
+
     if (det_ctx->de_state_sig_array != NULL)
         SCFree(det_ctx->de_state_sig_array);
     if (det_ctx->match_array != NULL)

diff --git a/src/detect.c b/src/detect.c
index 99779083141575435f1daf73b53eda6c56ad8ca0..1b561ff912d68951383f6abd4ab3f99cdeee8e02 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -663,17 +663,19 @@ end:
 }
 
 static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
-                                            DetectEngineThreadCtx *det_ctx,
-                                            SigGroupHead *sgh)
+                                            DetectEngineThreadCtx *det_ctx)
+//                                            SigGroupHead *sgh)
 {
     SigIntId mpm, nonmpm;
     det_ctx->match_array_cnt = 0;
     SigIntId *mpm_ptr = det_ctx->pmq.rule_id_array;
-    SigIntId *nonmpm_ptr = sgh->non_mpm_id_array;
+    SigIntId *nonmpm_ptr = det_ctx->non_mpm_id_array;
+    //SigIntId *nonmpm_ptr = sgh->non_mpm_id_array;
     uint32_t m_cnt = det_ctx->pmq.rule_id_array_cnt;
-    uint32_t n_cnt = sgh->non_mpm_id_cnt;
+    //uint32_t n_cnt = sgh->non_mpm_id_cnt;
+    uint32_t n_cnt = det_ctx->non_mpm_id_cnt;
     SCLogDebug("PMQ rule id array count %d", det_ctx->pmq.rule_id_array_cnt);
-    SCLogDebug("SGH non-MPM id count %d", sgh->non_mpm_id_cnt);
+//    SCLogDebug("SGH non-MPM id count %d", sgh->non_mpm_id_cnt);
     SigIntId *final_ptr;
     uint32_t final_cnt;
     SigIntId id;
@@ -752,7 +754,7 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
 
     det_ctx->match_array_cnt = match_array - det_ctx->match_array;
 
-    BUG_ON((det_ctx->pmq.rule_id_array_cnt + sgh->non_mpm_id_cnt) < det_ctx->match_array_cnt);
+    BUG_ON((det_ctx->pmq.rule_id_array_cnt + det_ctx->non_mpm_id_cnt) < det_ctx->match_array_cnt);
 }
 
 /* Return true is the list is sorted smallest to largest */
@@ -1330,6 +1332,17 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
     SignatureMask mask = 0;
     PacketCreateMask(p, &mask, alproto, has_state, smsg, app_decoder_events);
 
+    /* prefilter non_mpm list against the mask of the packet */
+    det_ctx->non_mpm_id_cnt = 0;
+    uint32_t x = 0;
+    for (x = 0; x < det_ctx->sgh->non_mpm_id_cnt; x++) {
+        /* only if the mask matches this rule can possibly match,
+         * so build the non_mpm array only for match candidates */
+        if ((det_ctx->sgh->non_mpm_mask_array[x] & mask) == det_ctx->sgh->non_mpm_mask_array[x]) {
+            det_ctx->non_mpm_id_array[det_ctx->non_mpm_id_cnt++] = det_ctx->sgh->non_mpm_id_array[x];
+        }
+    }
+
     /* run the mpm for each type */
     PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM);
     DetectMpmPrefilter(de_ctx, det_ctx, smsg, p, flags, alproto, has_state, &sms_runflags);
@@ -1340,11 +1353,14 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
                              (uint64_t)det_ctx->pmq.rule_id_array_cnt);
         SCPerfCounterAddUI64(det_ctx->counter_nonmpm_list, th_v->sc_perf_pca,
                              (uint64_t)det_ctx->sgh->non_mpm_id_cnt);
+        /* non mpm sigs after mask prefilter */
+        SCPerfCounterAddUI64(det_ctx->counter_fnonmpm_list,
+                th_v->sc_perf_pca, (uint64_t)det_ctx->non_mpm_id_cnt);
     }
 #endif
 
     PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PREFILTER);
-    DetectPrefilterMergeSort(de_ctx, det_ctx, det_ctx->sgh);
+    DetectPrefilterMergeSort(de_ctx, det_ctx);
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PREFILTER);
 
     PACKET_PROFILING_DETECT_START(p, PROF_DETECT_RULES);

diff --git a/src/detect.h b/src/detect.h
index 2ed8b5a3c0235ddd0536bda4b88919860a7b36b6..9ded0077e1309190e2dc1bcb1ac4dc23a742fea4 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -762,6 +762,9 @@ typedef struct DetectionEngineThreadCtx_ {
     /* the thread to which this detection engine thread belongs */
     ThreadVars *tv;
 
+    SigIntId *non_mpm_id_array;
+    uint32_t non_mpm_id_cnt; // size is cnt * sizeof(uint32_t)
+
     /* detection engine variables */
 
     /** offset into the payload of the last match by:
@@ -791,9 +794,10 @@ typedef struct DetectionEngineThreadCtx_ {
 
     /** id for alert counter */
     uint16_t counter_alerts;
-#ifdef COLLECT_SIGMATCH_LIST_STATS
+#ifdef PROFILING
     uint16_t counter_mpm_list;
     uint16_t counter_nonmpm_list;
+    uint16_t counter_fnonmpm_list;
     uint16_t counter_match_list;
 #endif
 
@@ -964,7 +968,10 @@ typedef struct SigGroupHead_ {
 #endif
 
     SigIntId *non_mpm_id_array;
-    uint32_t non_mpm_id_cnt; // size is cnt * sizeof(uint32_t)
+    uint32_t non_mpm_id_cnt; // size is cnt * sizeof(SigIntId)
+
+    SignatureMask *non_mpm_mask_array;
+    uint32_t non_mpm_mask_cnt; // size is cnt * sizeof(SignatureMask)
 
     /* pattern matcher instances */
     MpmCtx *mpm_proto_other_ctx;