From: Victor Julien <victor@inliniac.net>
Date: Sat, 4 Mar 2017 12:40:39 +0000 (+0100)
Subject: detect: don't rescan when just distance is used
X-Git-Tag: suricata-4.0.0-beta1~111
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7eda6beade1b9227a5ad36c50610858c67155530;p=thirdparty%2Fsuricata.git

detect: don't rescan when just distance is used

Content inspection optimization: when just distance is used without
within we don't need to search recursively.

E.g. content:"a"; content:"b"; distance:1; will scan the buffer for
'a' and when it finds 'a' it will scan the remainder for 'b'. Until
now, the failure to find 'b' would lead to looking for the next 'a'
and then for 'b' after that. However, we already inspected the
entire buffer for 'b', so we know this will fail.
---

diff --git a/src/detect-byte-extract.c b/src/detect-byte-extract.c
index 0b0f473477..6a19310500 100644
--- a/src/detect-byte-extract.c
+++ b/src/detect-byte-extract.c
@@ -2918,7 +2918,7 @@ static int DetectByteExtractTest48(void)
     if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
         cd->flags != (DETECT_CONTENT_DISTANCE_BE |
                       DETECT_CONTENT_DISTANCE |
-                      DETECT_CONTENT_RELATIVE_NEXT) ||
+                      DETECT_CONTENT_DISTANCE_NEXT) ||
         cd->distance != bed1->local_id ||
         cd->depth != 0 ||
         cd->offset != 0) {
@@ -3147,7 +3147,7 @@ static int DetectByteExtractTest50(void)
     if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
         cd->flags != (DETECT_CONTENT_WITHIN_BE |
                       DETECT_CONTENT_WITHIN|
-                      DETECT_CONTENT_RELATIVE_NEXT) ||
+                      DETECT_CONTENT_WITHIN_NEXT) ||
         cd->within != bed1->local_id ||
         cd->depth != 0 ||
         cd->offset != 0 ||
diff --git a/src/detect-content.h b/src/detect-content.h
index 70d17961e2..2182b1bacd 100644
--- a/src/detect-content.h
+++ b/src/detect-content.h
@@ -39,8 +39,7 @@
 /** content is negated */
 #define DETECT_CONTENT_NEGATED           BIT_U32(9)
 
-/** a relative match to this content is next, used in matching phase */
-#define DETECT_CONTENT_RELATIVE_NEXT     BIT_U32(10)
+// bit 10 unused
 
 /* BE - byte extract */
 #define DETECT_CONTENT_OFFSET_BE         BIT_U32(11)
@@ -55,6 +54,10 @@
  * the inspection phase */
 #define DETECT_CONTENT_NO_DOUBLE_INSPECTION_REQUIRED BIT_U32(16)
 
+#define DETECT_CONTENT_WITHIN_NEXT      BIT_U32(17)
+#define DETECT_CONTENT_DISTANCE_NEXT    BIT_U32(18)
+/** a relative match to this content is next, used in matching phase */
+#define DETECT_CONTENT_RELATIVE_NEXT    (DETECT_CONTENT_WITHIN_NEXT|DETECT_CONTENT_DISTANCE_NEXT)
 
 #define DETECT_CONTENT_IS_SINGLE(c) (!( ((c)->flags & DETECT_CONTENT_DISTANCE) || \
                                         ((c)->flags & DETECT_CONTENT_WITHIN) || \
diff --git a/src/detect-distance.c b/src/detect-distance.c
index 10f6528477..869a145026 100644
--- a/src/detect-distance.c
+++ b/src/detect-distance.c
@@ -142,7 +142,11 @@ static int DetectDistanceSetup (DetectEngineCtx *de_ctx, Signature *s,
                        "only content");
             goto end;
         }
-        prev_cd->flags |= DETECT_CONTENT_RELATIVE_NEXT;
+        if ((cd->flags & DETECT_CONTENT_NEGATED) == 0) {
+            prev_cd->flags |= DETECT_CONTENT_DISTANCE_NEXT;
+        } else {
+            prev_cd->flags |= DETECT_CONTENT_RELATIVE_NEXT;
+        }
     } else if (prev_pm->type == DETECT_PCRE) {
         DetectPcreData *pd = (DetectPcreData *)prev_pm->ctx;
         pd->flags |= DETECT_PCRE_RELATIVE_NEXT;
diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c
index 58131d02d8..bea5157e53 100644
--- a/src/detect-engine-content-inspection.c
+++ b/src/detect-engine-content-inspection.c
@@ -340,7 +340,7 @@ int DetectEngineContentInspection(DetectEngineCtx *de_ctx, DetectEngineThreadCtx
                 }
 
                 /* no match and no reason to look for another instance */
-                if ((cd->flags & DETECT_CONTENT_RELATIVE_NEXT) == 0) {
+                if ((cd->flags & DETECT_CONTENT_WITHIN_NEXT) == 0) {
                     SCLogDebug("'next sm' does not depend on me, so we can give up");
                     det_ctx->discontinue_matching = 1;
                     goto no_match;
diff --git a/src/detect-within.c b/src/detect-within.c
index b7921a3138..772fb039d7 100644
--- a/src/detect-within.c
+++ b/src/detect-within.c
@@ -152,7 +152,7 @@ static int DetectWithinSetup(DetectEngineCtx *de_ctx, Signature *s, char *within
                        "only content");
             goto end;
         }
-        prev_cd->flags |= DETECT_CONTENT_RELATIVE_NEXT;
+        prev_cd->flags |= DETECT_CONTENT_WITHIN_NEXT;
     } else if (prev_pm->type == DETECT_PCRE) {
         DetectPcreData *pd = (DetectPcreData *)prev_pm->ctx;
         pd->flags |= DETECT_PCRE_RELATIVE_NEXT;
diff --git a/src/tests/detect-engine-content-inspection.c b/src/tests/detect-engine-content-inspection.c
index eaf52a8305..b633e255f2 100644
--- a/src/tests/detect-engine-content-inspection.c
+++ b/src/tests/detect-engine-content-inspection.c
@@ -83,7 +83,7 @@ static int DetectEngineContentInspectionTest03(void) {
     TEST_RUN("ababc", 5, "content:\"a\"; content:\"b\"; content:\"d\";", false, 3);
 
     TEST_RUN("ababc", 5, "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0;", true, 3);
-    TEST_RUN("ababc", 5, "content:\"a\"; content:\"b\"; distance:0; content:\"d\"; distance:0;", false, 6); // TODO should be 3?
+    TEST_RUN("ababc", 5, "content:\"a\"; content:\"b\"; distance:0; content:\"d\"; distance:0;", false, 3);
 
     TEST_RUN("ababc", 5, "content:\"a\"; content:\"b\"; distance:0; within:1; content:\"d\"; distance:0; within:1;", false, 5);
 
@@ -155,15 +155,15 @@ static int DetectEngineContentInspectionTest07(void) {
     TEST_RUN("abcabcabcabcabcabcabcabcabcabcd", 31, "content:\"a\"; content:\"b\"; within:1; distance:0; content:\"c\"; distance:0; within:1; content:\"d\"; within:1; distance:0; ", true, 31);
     TEST_RUN("abcabcabcabcabcabcabcabcabcabcx", 31, "content:\"a\"; content:\"b\"; within:1; distance:0; content:\"c\"; distance:0; within:1; content:\"d\"; within:1; distance:0; ", false, 31);
 
-    TEST_RUN("abcabcabcabcabcabcabcabcabcabcx", 31, "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0; content:\"d\"; distance:0; ", false, 286); // TODO should be 4?
-    TEST_RUN("abcabcabcabcabcabcabcabcabcabcx", 31, "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0; pcre:\"/^d/R\"; ", false, 286); // TODO should be 4?
-    TEST_RUN("abcabcabcabcabcabcabcabcabcabcx", 31, "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0; isdataat:!1,relative; ", false, 286); // TODO should be 4?
+    TEST_RUN("abcabcabcabcabcabcabcabcabcabcx", 31, "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0; content:\"d\"; distance:0; ", false, 4);
+    TEST_RUN("abcabcabcabcabcabcabcabcabcabcx", 31, "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0; pcre:\"/^d/R\"; ", false, 13);
+    TEST_RUN("abcabcabcabcabcabcabcabcabcabcx", 31, "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0; isdataat:!1,relative; ", false, 13); // TODO should be 4?
     TEST_RUN("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdx", 41,
-            "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0; content:\"d\"; distance:0; content:\"e\"; distance:0; ", false, 1001); // TODO should be 5?
+            "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0; content:\"d\"; distance:0; content:\"e\"; distance:0; ", false, 5);
     TEST_RUN("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdx", 41,
-            "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0; content:\"d\"; distance:0; pcre:\"/^e/R\"; ", false, 1001); // TODO should be 5?
+            "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0; content:\"d\"; distance:0; pcre:\"/^e/R\"; ", false, 14); // TODO should be 5?
     TEST_RUN("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdx", 41,
-            "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0; content:\"d\"; distance:0; isdataat:!1,relative; ", false, 1001); // TODO should be 5?
+            "content:\"a\"; content:\"b\"; distance:0; content:\"c\"; distance:0; content:\"d\"; distance:0; isdataat:!1,relative; ", false, 14); // TODO should be 5?
 
     TEST_RUN("abcabcabcabcabcabcabcabcabcabcd", 31, "content:\"a\"; content:\"b\"; within:1; distance:0; content:\"c\"; distance:0; within:1; pcre:\"/d/\";", true, 4);
     TEST_RUN("abcabcabcabcabcabcabcabcabcabcd", 31, "content:\"a\"; content:\"b\"; within:1; distance:0; content:\"c\"; distance:0; within:1; pcre:\"/d/R\";", true, 4);
@@ -175,6 +175,18 @@ static int DetectEngineContentInspectionTest07(void) {
     TEST_FOOTER;
 }
 
+/** \test mix in negation */
+static int DetectEngineContentInspectionTest08(void) {
+    TEST_HEADER;
+    TEST_RUN("ababc", 5, "content:\"a\"; content:\"b\"; content:!\"d\";", true, 3);
+    TEST_RUN("ababc", 5, "content:\"a\"; content:\"b\"; content:!\"c\";", false, 3);
+
+    TEST_RUN("ababc", 5, "content:\"a\"; content:\"b\"; distance:0; within:1; content:!\"a\"; distance:0; within:1;", true, 5);
+    TEST_RUN("ababc", 5, "content:\"a\"; content:\"b\"; distance:0; within:1; content:!\"a\"; distance:0; ", true, 5);
+
+    TEST_FOOTER;
+}
+
 void DetectEngineContentInspectionRegisterTests(void)
 {
     UtRegisterTest("DetectEngineContentInspectionTest01",
@@ -191,6 +203,8 @@ void DetectEngineContentInspectionRegisterTests(void)
                    DetectEngineContentInspectionTest06);
     UtRegisterTest("DetectEngineContentInspectionTest07",
                    DetectEngineContentInspectionTest07);
+    UtRegisterTest("DetectEngineContentInspectionTest08",
+                   DetectEngineContentInspectionTest08);
 }
 
 #undef TEST_HEADER