detect: split non-mpm list into syn/nosyn

Since SYN inspecting rules are expensive, this patch splits the
'non-mpm' list (i.e. the rules that are always considered) into
a 'syn' and 'non-syn' list. The SYN list is only inspected if the
packet has the SYN flag set, otherwise the non-syn list is used.

The syn-list contains _all_ rules. The non-syn list contains all
minus the rules requiring the SYN bit in a packet.

diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c
index 1ec12111e7503a9d8a4daad759213bd2f4b935a7..87cfe4b1b042a9e83db505aed0b1a775603f8310 100644 (file)
--- a/src/detect-engine-siggroup.c
+++ b/src/detect-engine-siggroup.c
@@ -39,6 +39,7 @@
 
 #include "detect-content.h"
 #include "detect-uricontent.h"
+#include "detect-flags.h"
 
 #include "util-hash.h"
 #include "util-hashlist.h"
@@ -180,10 +181,16 @@ void SigGroupHeadFree(SigGroupHead *sgh)
         sgh->match_array = NULL;
     }
 
-    if (sgh->non_mpm_store_array != NULL) {
-        SCFree(sgh->non_mpm_store_array);
-        sgh->non_mpm_store_array = NULL;
-        sgh->non_mpm_store_cnt = 0;
+    if (sgh->non_mpm_other_store_array != NULL) {
+        SCFree(sgh->non_mpm_other_store_array);
+        sgh->non_mpm_other_store_array = NULL;
+        sgh->non_mpm_other_store_cnt = 0;
+    }
+
+    if (sgh->non_mpm_syn_store_array != NULL) {
+        SCFree(sgh->non_mpm_syn_store_array);
+        sgh->non_mpm_syn_store_array = NULL;
+        sgh->non_mpm_syn_store_cnt = 0;
     }
 
     sgh->sig_cnt = 0;
@@ -957,53 +964,64 @@ int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
     Signature *s = NULL;
     uint32_t sig = 0;
     uint32_t non_mpm = 0;
+    uint32_t non_mpm_syn = 0;
 
     if (sgh == NULL)
         return 0;
 
-    BUG_ON(sgh->non_mpm_store_array != NULL);
+    BUG_ON(sgh->non_mpm_other_store_array != NULL);
 
     for (sig = 0; sig < sgh->sig_cnt; sig++) {
         s = sgh->match_array[sig];
         if (s == NULL)
             continue;
 
-        if (s->mpm_sm == NULL)
-            non_mpm++;
-        else if (s->flags & (SIG_FLAG_MPM_PACKET_NEG|SIG_FLAG_MPM_STREAM_NEG|SIG_FLAG_MPM_APPLAYER_NEG))
-            non_mpm++;
+        if (s->mpm_sm == NULL || (s->flags & (SIG_FLAG_MPM_PACKET_NEG|SIG_FLAG_MPM_STREAM_NEG|SIG_FLAG_MPM_APPLAYER_NEG))) {
+            if (!(DetectFlagsSignatureNeedsSynPackets(s))) {
+                non_mpm++;
+            }
+            non_mpm_syn++;
+        }
     }
 
-    if (non_mpm == 0) {
-        sgh->non_mpm_store_array = NULL;
+    if (non_mpm == 0 && non_mpm_syn == 0) {
+        sgh->non_mpm_other_store_array = NULL;
+        sgh->non_mpm_syn_store_array = NULL;
         return 0;
     }
 
-    sgh->non_mpm_store_array = SCMalloc(non_mpm * sizeof(SignatureNonMpmStore));
-    BUG_ON(sgh->non_mpm_store_array == NULL);
-    memset(sgh->non_mpm_store_array, 0, non_mpm * sizeof(SignatureNonMpmStore));
+    sgh->non_mpm_other_store_array = SCMalloc(non_mpm * sizeof(SignatureNonMpmStore));
+    BUG_ON(sgh->non_mpm_other_store_array == NULL);
+    memset(sgh->non_mpm_other_store_array, 0, non_mpm * sizeof(SignatureNonMpmStore));
+
+    sgh->non_mpm_syn_store_array = SCMalloc(non_mpm_syn * sizeof(SignatureNonMpmStore));
+    BUG_ON(sgh->non_mpm_syn_store_array == NULL);
+    memset(sgh->non_mpm_syn_store_array, 0, non_mpm_syn * sizeof(SignatureNonMpmStore));
 
     for (sig = 0; sig < sgh->sig_cnt; sig++) {
         s = sgh->match_array[sig];
         if (s == NULL)
             continue;
 
-        if (s->mpm_sm == NULL) {
-            BUG_ON(sgh->non_mpm_store_cnt >= non_mpm);
-            sgh->non_mpm_store_array[sgh->non_mpm_store_cnt].id = s->num;
-            sgh->non_mpm_store_array[sgh->non_mpm_store_cnt].mask = s->mask;
-            sgh->non_mpm_store_cnt++;
-        } else if (s->flags & (SIG_FLAG_MPM_PACKET_NEG|SIG_FLAG_MPM_STREAM_NEG|SIG_FLAG_MPM_APPLAYER_NEG)) {
-            BUG_ON(sgh->non_mpm_store_cnt >= non_mpm);
-            sgh->non_mpm_store_array[sgh->non_mpm_store_cnt].id = s->num;
-            sgh->non_mpm_store_array[sgh->non_mpm_store_cnt].mask = s->mask;
-            sgh->non_mpm_store_cnt++;
+        if (s->mpm_sm == NULL || (s->flags & (SIG_FLAG_MPM_PACKET_NEG|SIG_FLAG_MPM_STREAM_NEG|SIG_FLAG_MPM_APPLAYER_NEG))) {
+            if (!(DetectFlagsSignatureNeedsSynPackets(s))) {
+                BUG_ON(sgh->non_mpm_other_store_cnt >= non_mpm);
+                sgh->non_mpm_other_store_array[sgh->non_mpm_other_store_cnt].id = s->num;
+                sgh->non_mpm_other_store_array[sgh->non_mpm_other_store_cnt].mask = s->mask;
+                sgh->non_mpm_other_store_cnt++;
+            }
+
+            BUG_ON(sgh->non_mpm_syn_store_cnt >= non_mpm_syn);
+            sgh->non_mpm_syn_store_array[sgh->non_mpm_syn_store_cnt].id = s->num;
+            sgh->non_mpm_syn_store_array[sgh->non_mpm_syn_store_cnt].mask = s->mask;
+            sgh->non_mpm_syn_store_cnt++;
         }
     }
 
     /* track highest cnt for any sgh in our de_ctx */
-    if (sgh->non_mpm_store_cnt > de_ctx->non_mpm_store_cnt_max)
-        de_ctx->non_mpm_store_cnt_max = sgh->non_mpm_store_cnt;
+    uint32_t max = MAX(sgh->non_mpm_other_store_cnt, sgh->non_mpm_syn_store_cnt);
+    if (max > de_ctx->non_mpm_store_cnt_max)
+        de_ctx->non_mpm_store_cnt_max = max;
 
     return 0;
 }

diff --git a/src/detect.c b/src/detect.c
index 4a63c4d39f05b82c3089d4837212c9f16c663532..107d0e4f2690dcb1ca735839ee4f628c76400b3d 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -1213,16 +1213,34 @@ static void AlertDebugLogModeSyncFlowbitsNamesToPacketStruct(Packet *p, DetectEn
 static inline void DetectPrefilterBuildNonMpmList(DetectEngineThreadCtx *det_ctx, SignatureMask mask)
 {
     uint32_t x = 0;
-    for (x = 0; x < det_ctx->sgh->non_mpm_store_cnt; x++) {
+    for (x = 0; x < det_ctx->non_mpm_store_cnt; x++) {
         /* only if the mask matches this rule can possibly match,
          * so build the non_mpm array only for match candidates */
-        SignatureMask rule_mask = det_ctx->sgh->non_mpm_store_array[x].mask;
+        SignatureMask rule_mask = det_ctx->non_mpm_store_ptr[x].mask;
         if ((rule_mask & mask) == rule_mask) {
-            det_ctx->non_mpm_id_array[det_ctx->non_mpm_id_cnt++] = det_ctx->sgh->non_mpm_store_array[x].id;
+            det_ctx->non_mpm_id_array[det_ctx->non_mpm_id_cnt++] = det_ctx->non_mpm_store_ptr[x].id;
         }
     }
 }
 
+/** \internal
+ *  \brief select non-mpm list
+ *  Based on the packet properties, select the non-mpm list to use */
+static inline void DetectPrefilterSetNonMpmList(const Packet *p, DetectEngineThreadCtx *det_ctx)
+{
+    if ((p->proto == IPPROTO_TCP) && (p->tcph != NULL) && (p->tcph->th_flags & TH_SYN)) {
+        det_ctx->non_mpm_store_ptr = det_ctx->sgh->non_mpm_syn_store_array;
+        det_ctx->non_mpm_store_cnt = det_ctx->sgh->non_mpm_syn_store_cnt;
+    } else {
+        det_ctx->non_mpm_store_ptr = det_ctx->sgh->non_mpm_other_store_array;
+        det_ctx->non_mpm_store_cnt = det_ctx->sgh->non_mpm_other_store_cnt;
+    }
+    SCLogDebug("sgh non_mpm ptr %p cnt %u (syn %p/%u, other %p/%u)",
+            det_ctx->non_mpm_store_ptr, det_ctx->non_mpm_store_cnt,
+            det_ctx->sgh->non_mpm_syn_store_array, det_ctx->sgh->non_mpm_syn_store_cnt,
+            det_ctx->sgh->non_mpm_other_store_array, det_ctx->sgh->non_mpm_other_store_cnt);
+}
+
 /**
  *  \brief Signature match function
  *
@@ -1429,6 +1447,8 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
         goto end;
     }
 
+    DetectPrefilterSetNonMpmList(p, det_ctx);
+
     PACKET_PROFILING_DETECT_START(p, PROF_DETECT_STATEFUL);
     /* stateful app layer detection */
     if ((p->flags & PKT_HAS_FLOW) && has_state) {
@@ -1452,7 +1472,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
     /* build and prefilter non_mpm list against the mask of the packet */
     PACKET_PROFILING_DETECT_START(p, PROF_DETECT_NONMPMLIST);
     det_ctx->non_mpm_id_cnt = 0;
-    if (likely(det_ctx->sgh->non_mpm_store_cnt > 0)) {
+    if (likely(det_ctx->non_mpm_store_cnt > 0)) {
         DetectPrefilterBuildNonMpmList(det_ctx, mask);
     }
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_NONMPMLIST);
@@ -1466,7 +1486,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
         StatsAddUI64(th_v, det_ctx->counter_mpm_list,
                              (uint64_t)det_ctx->pmq.rule_id_array_cnt);
         StatsAddUI64(th_v, det_ctx->counter_nonmpm_list,
-                             (uint64_t)det_ctx->sgh->non_mpm_store_cnt);
+                             (uint64_t)det_ctx->non_mpm_store_cnt);
         /* non mpm sigs after mask prefilter */
         StatsAddUI64(th_v, det_ctx->counter_fnonmpm_list,
                              (uint64_t)det_ctx->non_mpm_id_cnt);

diff --git a/src/detect.h b/src/detect.h
index afef28f547452e49fb001d6d7077f26352168bea..9d83a793a6efe8dd220f75bc5d893f894c004a96 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -736,6 +736,11 @@ typedef struct FiledataReassembledBody_ {
 /** \todo review how many we actually need here */
 #define DETECT_SMSG_PMQ_NUM 256
 
+typedef struct SignatureNonMpmStore_ {
+    SigIntId id;
+    SignatureMask mask;
+} SignatureNonMpmStore;
+
 /**
   * Detection engine thread data.
   */
@@ -832,6 +837,10 @@ typedef struct DetectEngineThreadCtx_ {
     uint8_t *de_state_sig_array;
 
     struct SigGroupHead_ *sgh;
+
+    SignatureNonMpmStore *non_mpm_store_ptr;
+    uint32_t non_mpm_store_cnt;
+
     /** pointer to the current mpm ctx that is stored
      *  in a rule group head -- can be either a content
      *  or uricontent ctx. */
@@ -983,11 +992,6 @@ typedef struct SigGroupHeadInitData_ {
     struct DetectPort_ *port;
 } SigGroupHeadInitData;
 
-typedef struct SignatureNonMpmStore_ {
-    SigIntId id;
-    SignatureMask mask;
-} SignatureNonMpmStore;
-
 /** \brief Container for matching data for a signature group */
 typedef struct SigGroupHead_ {
     uint32_t flags;
@@ -1003,8 +1007,12 @@ typedef struct SigGroupHead_ {
     SignatureMask *mask_array;
 #endif
 
-    SignatureNonMpmStore *non_mpm_store_array; // size is non_mpm_store_cnt * sizeof(SignatureNonMpmStore)
-    uint32_t non_mpm_store_cnt;
+    /* non mpm list excluding SYN rules */
+    SignatureNonMpmStore *non_mpm_other_store_array; // size is non_mpm_store_cnt * sizeof(SignatureNonMpmStore)
+    uint32_t non_mpm_other_store_cnt;
+    /* non mpm list including SYN rules */
+    SignatureNonMpmStore *non_mpm_syn_store_array; // size is non_mpm_syn_store_cnt * sizeof(SignatureNonMpmStore)
+    uint32_t non_mpm_syn_store_cnt;
 
     /* pattern matcher instances */
     const MpmCtx *mpm_proto_other_ctx;