When assigning Pattern IDs pids, check Case flags

This fixes bug 1110. When assigning PIDs, use the NO_CASE flag when comparing
for duplicates. The state of the flag must be the same, but also use the same
type of comparisons when checking for duplicates.

Previously, "foo":CS would match with "foo":CI when it should not.
and "foo":CI would not match "FoO":CI when it should. Both of those
cases are fixed with this change.

This then allows simplifying the use of pid in MPMs because now if they
pids match, then so do the flags, so checking the flags is not required.

diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
index 85a4d92e2ccddf7b5726a3cada65f4dea23e025c..53cc756280b99d339eda2e412940ff5b910de285 100644 (file)
--- a/src/detect-engine-mpm.c
+++ b/src/detect-engine-mpm.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2014 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -2813,6 +2813,10 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
     uint32_t content_total_size = 0;
     Signature *s = NULL;
 
+    /* Count the amount of memory needed to store all the structures
+     * and the content of those structures. This will over estimate the
+     * true size, since duplicates are removed below, but counted here.
+     */
     for (s = de_ctx->sig_list; s != NULL; s = s->next) {
         s->mpm_sm = RetrieveFPForSigV2(s);
         if (s->mpm_sm != NULL) {
@@ -2846,25 +2850,44 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
                 content = cd->content;
                 content_len = cd->content_len;
             }
+            uint32_t flags = cd->flags & DETECT_CONTENT_NOCASE;
+            /* Check for content already found on the same list */
             for (; dup != struct_offset; dup++) {
-                if (dup->content_len != content_len ||
-                    dup->sm_list != sm_list ||
-                    SCMemcmp(dup->content, content, dup->content_len) != 0) {
+                if (dup->content_len != content_len)
                     continue;
+                if (dup->sm_list != sm_list)
+                    continue;
+                if (dup->flags != flags)
+                    continue;
+                /* Check for pattern matching a duplicate. Use case insensitive matching
+                 * for case insensitive patterns. */
+                if (flags & DETECT_CONTENT_NOCASE) {
+                    if (SCMemcmpLowercase(dup->content, content, content_len) != 0)
+                        continue;
+                } else {
+                    // Case sensitive matching
+                    if (SCMemcmp(dup->content, content, content_len) != 0)
+                        continue;
                 }
-
+                /* Found a match with a previous pattern. */
                 break;
             }
             if (dup != struct_offset) {
+              /* Exited for-loop before the end, so found an existing match.
+               * Use its ID. */
                 cd->id = dup->id;
                 continue;
             }
 
+            /* Not found, so new content. Give it a new ID and add it to the array.
+             * Copy the content at the end of the content array. */
             struct_offset->id = max_id++;
             cd->id = struct_offset->id;
             struct_offset->content_len = content_len;
             struct_offset->sm_list = sm_list;
             struct_offset->content = content_offset;
+            struct_offset->flags = flags;
+
             content_offset += content_len;
             memcpy(struct_offset->content, content, content_len);
 

diff --git a/src/util-mpm-ac-bs.c b/src/util-mpm-ac-bs.c
index 471e850c16ec1a891b5af5204cdafff311dc92bd..e451bed3271484d42d52dbfb589787c6116a344f 100644 (file)
--- a/src/util-mpm-ac-bs.c
+++ b/src/util-mpm-ac-bs.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2010 Open Information Security Foundation
+/* Copyright (C) 2007-2014 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -131,33 +131,6 @@ static void SCACBSGetConfig()
     return;
 }
 
-/**
- * \internal
- * \brief Compares 2 patterns.  We use it for the hashing process during the
- *        the initial pattern insertion time, to cull duplicate sigs.
- *
- * \param p      Pointer to the first pattern(SCACBSPattern).
- * \param pat    Pointer to the second pattern(raw pattern array).
- * \param patlen Pattern length.
- * \param flags  Flags.  We don't need this.
- *
- * \retval hash A 32 bit unsigned hash.
- */
-static inline int SCACBSCmpPattern(SCACBSPattern *p, uint8_t *pat, uint16_t patlen,
-                                 char flags)
-{
-    if (p->len != patlen)
-        return 0;
-
-    if (p->flags != flags)
-        return 0;
-
-    if (memcmp(p->cs, pat, patlen) != 0)
-        return 0;
-
-    return 1;
-}
-
 /**
  * \internal
  * \brief Creates a hash of the pattern.  We use it for the hashing process
@@ -195,14 +168,13 @@ static inline SCACBSPattern *SCACBSInitHashLookup(SCACBSCtx *ctx, uint8_t *pat,
 {
     uint32_t hash = SCACBSInitHashRaw(pat, patlen);
 
-    if (ctx->init_hash == NULL || ctx->init_hash[hash] == NULL) {
+    if (ctx->init_hash == NULL) {
         return NULL;
     }
 
     SCACBSPattern *t = ctx->init_hash[hash];
     for ( ; t != NULL; t = t->next) {
-        //if (SCACBSCmpPattern(t, pat, patlen, flags) == 1)
-        if (t->flags == flags && t->id == pid)
+        if (t->id == pid)
             return t;
     }
 

diff --git a/src/util-mpm-ac-gfbs.c b/src/util-mpm-ac-gfbs.c
index a0707109213349c4c01ababe0a9703e65b6f4f73..5a9aade147ad96d1bfb2767fd0b9f7847dbf6c7a 100644 (file)
--- a/src/util-mpm-ac-gfbs.c
+++ b/src/util-mpm-ac-gfbs.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2010 Open Information Security Foundation
+/* Copyright (C) 2007-2014 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -125,33 +125,6 @@ static void SCACGfbsGetConfig()
     return;
 }
 
-/**
- * \internal
- * \brief Compares 2 patterns.  We use it for the hashing process during the
- *        the initial pattern insertion time, to cull duplicate sigs.
- *
- * \param p      Pointer to the first pattern(SCACGfbsPattern).
- * \param pat    Pointer to the second pattern(raw pattern array).
- * \param patlen Pattern length.
- * \param flags  Flags.  We don't need this.
- *
- * \retval hash A 32 bit unsigned hash.
- */
-static inline int SCACGfbsCmpPattern(SCACGfbsPattern *p, uint8_t *pat, uint16_t patlen,
-                                     char flags)
-{
-    if (p->len != patlen)
-        return 0;
-
-    if (p->flags != flags)
-        return 0;
-
-    if (memcmp(p->cs, pat, patlen) != 0)
-        return 0;
-
-    return 1;
-}
-
 /**
  * \internal
  * \brief Creates a hash of the pattern.  We use it for the hashing process
@@ -189,14 +162,12 @@ static inline SCACGfbsPattern *SCACGfbsInitHashLookup(SCACGfbsCtx *ctx, uint8_t
 {
     uint32_t hash = SCACGfbsInitHashRaw(pat, patlen);
 
-    if (ctx->init_hash == NULL || ctx->init_hash[hash] == NULL) {
+    if (ctx->init_hash == NULL)
         return NULL;
-    }
 
     SCACGfbsPattern *t = ctx->init_hash[hash];
     for ( ; t != NULL; t = t->next) {
-        //if (SCACGfbsCmpPattern(t, pat, patlen, flags) == 1)
-        if (t->flags == flags && t->id == pid)
+        if (t->id == pid)
             return t;
     }
 

diff --git a/src/util-mpm-ac-tile.c b/src/util-mpm-ac-tile.c
index b724f5a4457e4fda67b67b75ff74adfcf99fd124..8f2a59f05d58d7dce1f91b3a9d96d74ba18db283 100644 (file)
--- a/src/util-mpm-ac-tile.c
+++ b/src/util-mpm-ac-tile.c
@@ -196,8 +196,16 @@ static inline int SCACTileCmpPattern(SCACTilePattern *p, uint8_t *pat,
     if (p->flags != flags)
         return 0;
 
-    if (memcmp(p->cs, pat, patlen) != 0)
-        return 0;
+    if (flags & MPM_PATTERN_FLAG_NOCASE) {
+        // Case insensitive
+        if (SCMemcmpLowercase(p->cs, pat, patlen) != 0)
+            return 0;
+
+    } else {
+        // Case sensitive
+        if (SCMemcmp(p->cs, pat, patlen) != 0)
+            return 0;
+    }
 
     return 1;
 }
@@ -241,15 +249,15 @@ static inline SCACTilePattern *SCACTileInitHashLookup(SCACTileCtx *ctx,
 {
     uint32_t hash = SCACTileInitHashRaw(pat, patlen);
 
-    if (ctx->init_hash == NULL || ctx->init_hash[hash] == NULL) {
+    if (ctx->init_hash == NULL) {
         return NULL;
     }
 
     SCACTilePattern *t = ctx->init_hash[hash];
     for ( ; t != NULL; t = t->next) {
-        //if (SCACTileCmpPattern(t, pat, patlen, flags) == 1)
-        if (t->flags == flags && t->id == pid)
+        if (t->id == pid) {
             return t;
+        }
     }
 
     return NULL;

diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c
index f45970dc0ba5dd9293816bb1290ad54c40681ffa..09eab9029be605ac466a2b0f805f0ef106fbbbd7 100644 (file)
--- a/src/util-mpm-ac.c
+++ b/src/util-mpm-ac.c
@@ -118,33 +118,6 @@ static void SCACGetConfig()
     return;
 }
 
-/**
- * \internal
- * \brief Compares 2 patterns.  We use it for the hashing process during the
- *        the initial pattern insertion time, to cull duplicate sigs.
- *
- * \param p      Pointer to the first pattern(SCACPattern).
- * \param pat    Pointer to the second pattern(raw pattern array).
- * \param patlen Pattern length.
- * \param flags  Flags.  We don't need this.
- *
- * \retval hash A 32 bit unsigned hash.
- */
-static inline int SCACCmpPattern(SCACPattern *p, uint8_t *pat, uint16_t patlen,
-                                 char flags)
-{
-    if (p->len != patlen)
-        return 0;
-
-    if (p->flags != flags)
-        return 0;
-
-    if (memcmp(p->cs, pat, patlen) != 0)
-        return 0;
-
-    return 1;
-}
-
 /**
  * \internal
  * \brief Creates a hash of the pattern.  We use it for the hashing process
@@ -182,14 +155,13 @@ static inline SCACPattern *SCACInitHashLookup(SCACCtx *ctx, uint8_t *pat,
 {
     uint32_t hash = SCACInitHashRaw(pat, patlen);
 
-    if (ctx->init_hash == NULL || ctx->init_hash[hash] == NULL) {
+    if (ctx->init_hash == NULL) {
         return NULL;
     }
 
     SCACPattern *t = ctx->init_hash[hash];
     for ( ; t != NULL; t = t->next) {
-        //if (SCACCmpPattern(t, pat, patlen, flags) == 1)
-        if (t->flags == flags && t->id == pid)
+        if (t->id == pid)
             return t;
     }
 

diff --git a/src/util-mpm-b2g.c b/src/util-mpm-b2g.c
index cbe15b8513fb9de18832d66fd61c1f7857dbded9..970268a2cfbf95e8b1fd5e716baa0fff4cc63c20 100644 (file)
--- a/src/util-mpm-b2g.c
+++ b/src/util-mpm-b2g.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2010 Open Information Security Foundation
+/* Copyright (C) 2007-2014 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -233,7 +233,7 @@ static inline B2gPattern *B2gInitHashLookup(B2gCtx *ctx, uint8_t *pat, uint16_t
     B2gPattern *t = ctx->init_hash[hash];
     for ( ; t != NULL; t = t->next) {
         //if (B2gCmpPattern(t,pat,patlen,flags) == 1)
-        if (t->flags == flags && t->id == pid)
+        if (t->id == pid)
             return t;
     }