mpm: improve negated mpm

The idea is: if mpm is negated, it's both on mpm and nonmpm sid lists
and we can kick it out in that case during the merge sort.

It only works for patterns that are 'independent'. This means that the
rule doesn't need to only match if the negated mpm pattern is limited
to the first 10 bytes for example.

Or more generally, an negated mpm pattern that has depth, offset,
distance or within settings can't be handled this way. These patterns
are not added to the mpm at all, but just to to non-mpm list. This
makes sense as they will *always* need manual inspection.

Similarly, a pattern that is 'chopped' always needs validation. This
is because in this case we only inspect a part of the final pattern.

diff --git a/src/detect-content.h b/src/detect-content.h
index 00c1ea3920d8b233efac97c8a3be16655cb3a07f..51300e7ea912887f33968fc61d55698424c78e2f 100644 (file)
--- a/src/detect-content.h
+++ b/src/detect-content.h
@@ -61,6 +61,18 @@
                                         ((c)->flags & DETECT_CONTENT_DEPTH) || \
                                         ((c)->flags & DETECT_CONTENT_OFFSET) ))
 
+/* if a pattern has no depth/offset limits, no relative specifiers and isn't
+ * chopped for the mpm, we can take the mpm and consider this pattern a match
+ * w/o futher inspection. Warning: this may still mean other patterns depend
+ * on this pattern that force match validation anyway. */
+#define DETECT_CONTENT_MPM_IS_CONCLUSIVE(c) \
+                                    !( ((c)->flags & DETECT_CONTENT_DISTANCE) || \
+                                       ((c)->flags & DETECT_CONTENT_WITHIN)   || \
+                                       ((c)->flags & DETECT_CONTENT_DEPTH)    || \
+                                       ((c)->flags & DETECT_CONTENT_OFFSET)   || \
+                                       ((c)->flags & DETECT_CONTENT_FAST_PATTERN_CHOP))
+
+
 #include "util-spm-bm.h"
 
 typedef struct DetectContentData_ {

diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
index 7295813e0c32e9238c1686dcf99dd6a807f3a3af..1faa3fe19274ff5ac4837c5d73e8698a207e9af0 100644 (file)
--- a/src/detect-engine-mpm.c
+++ b/src/detect-engine-mpm.c
@@ -1376,8 +1376,23 @@ void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
 
             const DetectContentData *cd = (DetectContentData *)s->mpm_sm->ctx;
 
-            PopulateMpmHelperAddPatternToPktCtx(ms->mpm_ctx,
-                    cd, s, 0, (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP));
+            int skip = 0;
+            /* negated logic: if mpm match can't be used to be sure about this
+             * pattern, we have to inspect the rule fully regardless of mpm
+             * match. So in this case there is no point of adding it at all.
+             * The non-mpm list entry for the sig will make sure the sig is
+             * inspected. */
+            if ((cd->flags & DETECT_CONTENT_NEGATED) &&
+                !(DETECT_CONTENT_MPM_IS_CONCLUSIVE(cd)))
+            {
+                skip = 1;
+                SCLogDebug("not adding negated mpm as it's not 'single'");
+            }
+
+            if (!skip) {
+                PopulateMpmHelperAddPatternToPktCtx(ms->mpm_ctx,
+                        cd, s, 0, (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP));
+            }
         }
     }
 

diff --git a/src/detect.c b/src/detect.c
index e975f6d31ba4892e7f88763ee0cbc6a7bba4be51..404dac4d53095b140e2d106ecf1c868e418123c6 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -765,7 +765,7 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
         goto final;
     }
     while (1) {
-        if (mpm <= nonmpm) {
+        if (mpm < nonmpm) {
             /* Take from mpm list */
             id = mpm;
 
@@ -778,12 +778,12 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
             if (unlikely(--m_cnt == 0)) {
                 /* mpm list is now empty */
                 final_ptr = nonmpm_ptr;
-                 final_cnt = n_cnt;
-                 goto final;
+                final_cnt = n_cnt;
+                goto final;
              }
              mpm_ptr++;
              mpm = *mpm_ptr;
-         } else {
+         } else if (mpm > nonmpm) {
              id = nonmpm;
 
              s = sig_array[id];
@@ -799,6 +799,38 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
              }
              nonmpm_ptr++;
              nonmpm = *nonmpm_ptr;
+
+        } else { /* implied mpm == nonmpm */
+            /* special case: if on both lists, it's a negated mpm pattern */
+
+            /* mpm list may have dups, so skip past them here */
+            while (--m_cnt != 0) {
+                mpm_ptr++;
+                mpm = *mpm_ptr;
+                if (mpm != nonmpm)
+                    break;
+            }
+            /* if mpm is done, update nonmpm_ptrs and jump to final */
+            if (unlikely(m_cnt == 0)) {
+                n_cnt--;
+
+                /* mpm list is now empty */
+                final_ptr = ++nonmpm_ptr;
+                final_cnt = n_cnt;
+                goto final;
+            }
+            /* otherwise, if nonmpm is done jump to final for mpm
+             * mpm ptrs alrady updated */
+            if (unlikely(--n_cnt == 0)) {
+                final_ptr = mpm_ptr;
+                final_cnt = m_cnt;
+                goto final;
+            }
+
+            /* not at end of the lists, update nonmpm. Mpm already
+             * updated in while loop above. */
+            nonmpm_ptr++;
+            nonmpm = *nonmpm_ptr;
         }
     }