detect: pattern id assignment through hash table

Only consider active part of the pattern for mpm (so consider chop).

Move data structure to hash list table over the custom array logic.

diff --git a/src/detect-engine-analyzer.c b/src/detect-engine-analyzer.c
index dce2f8061a8e4bd222553caec2f613c916d9006f..c2ffdef0b3a0bd00837ff2275d55e7e22984c43c 100644 (file)
--- a/src/detect-engine-analyzer.c
+++ b/src/detect-engine-analyzer.c
@@ -683,15 +683,11 @@ static void DumpMatches(RuleAnalyzer *ctx, JsonBuilder *js, const SigMatchData *
     jb_close(js);
 }
 
-static void EngineAnalysisRulePatterns(const DetectEngineCtx *de_ctx, const Signature *s);
-
 SCMutex g_rules_analyzer_write_m = SCMUTEX_INITIALIZER;
 void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s)
 {
     SCEnter();
 
-    EngineAnalysisRulePatterns(de_ctx, s);
-
     RuleAnalyzer ctx = { NULL, NULL, NULL };
 
     ctx.js = jb_new_object();
@@ -977,98 +973,6 @@ void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s)
     SCReturn;
 }
 
-struct PatternItem {
-    const DetectContentData *cd;
-    int sm_list;
-    uint32_t cnt;
-    uint32_t mpm;
-};
-
-static inline uint32_t ContentFlagsForHash(const DetectContentData *cd)
-{
-    return cd->flags & (DETECT_CONTENT_NOCASE | DETECT_CONTENT_OFFSET | DETECT_CONTENT_DEPTH |
-                               DETECT_CONTENT_NEGATED | DETECT_CONTENT_ENDS_WITH);
-}
-
-/** \internal
- *  \brief The hash function for Pattern
- *
- *  \param ht      Pointer to the hash table.
- *  \param data    Pointer to the Pattern.
- *  \param datalen Not used in our case.
- *
- *  \retval hash The generated hash value.
- */
-static uint32_t PatternHashFunc(HashListTable *ht, void *data, uint16_t datalen)
-{
-    struct PatternItem *p = (struct PatternItem *)data;
-    uint32_t hash = p->sm_list + ContentFlagsForHash(p->cd);
-
-    for (uint32_t b = 0; b < p->cd->content_len; b++)
-        hash += p->cd->content[b];
-
-    return hash % ht->array_size;
-}
-
-/**
- * \brief The Compare function for Pattern
- *
- * \param data1 Pointer to the first Pattern.
- * \param len1  Not used.
- * \param data2 Pointer to the second Pattern.
- * \param len2  Not used.
- *
- * \retval 1 If the 2 Patterns sent as args match.
- * \retval 0 If the 2 Patterns sent as args do not match.
- */
-static char PatternCompareFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
-{
-    const struct PatternItem *p1 = (struct PatternItem *)data1;
-    const struct PatternItem *p2 = (struct PatternItem *)data2;
-
-    if (p1->sm_list != p2->sm_list)
-        return 0;
-
-    if (ContentFlagsForHash(p1->cd) != ContentFlagsForHash(p2->cd))
-        return 0;
-
-    if (p1->cd->content_len != p2->cd->content_len)
-        return 0;
-
-    if (memcmp(p1->cd->content, p2->cd->content, p1->cd->content_len) != 0) {
-        return 0;
-    }
-
-    return 1;
-}
-
-static void PatternFreeFunc(void *ptr)
-{
-    SCFree(ptr);
-}
-
-/**
- * \brief Initializes the Pattern mpm hash table to be used by the detection
- *        engine context.
- *
- * \param de_ctx Pointer to the detection engine context.
- *
- * \retval  0 On success.
- * \retval -1 On failure.
- */
-static int PatternInit(DetectEngineCtx *de_ctx)
-{
-    de_ctx->pattern_hash_table =
-            HashListTableInit(4096, PatternHashFunc, PatternCompareFunc, PatternFreeFunc);
-    if (de_ctx->pattern_hash_table == NULL)
-        goto error;
-
-    return 0;
-
-error:
-    return -1;
-}
-
 void DumpPatterns(DetectEngineCtx *de_ctx)
 {
     if (de_ctx->pattern_hash_table == NULL)
@@ -1083,7 +987,7 @@ void DumpPatterns(DetectEngineCtx *de_ctx)
     for (HashListTableBucket *htb = HashListTableGetListHead(de_ctx->pattern_hash_table);
             htb != NULL; htb = HashListTableGetListNext(htb)) {
         char str[1024] = "";
-        struct PatternItem *p = HashListTableGetListData(htb);
+        DetectPatternTracker *p = HashListTableGetListData(htb);
         DetectContentPatternPrettyPrint(p->cd, str, sizeof(str));
 
         JsonBuilder *jb = arrays[p->sm_list];
@@ -1148,156 +1052,6 @@ void DumpPatterns(DetectEngineCtx *de_ctx)
     de_ctx->pattern_hash_table = NULL;
 }
 
-static void EngineAnalysisRulePatterns(const DetectEngineCtx *de_ctx, const Signature *s)
-{
-    SCEnter();
-
-    if (de_ctx->pattern_hash_table == NULL)
-        PatternInit((DetectEngineCtx *)de_ctx); // TODO hack const
-
-    if (s->sm_arrays[DETECT_SM_LIST_PMATCH]) {
-        SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_PMATCH];
-        do {
-            switch (smd->type) {
-                case DETECT_CONTENT: {
-                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;
-                    struct PatternItem lookup = {
-                        .cd = cd, .sm_list = DETECT_SM_LIST_PMATCH, .cnt = 0, .mpm = 0
-                    };
-                    struct PatternItem *res =
-                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
-                    if (res) {
-                        res->cnt++;
-                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
-                    } else {
-                        struct PatternItem *add = SCCalloc(1, sizeof(*add));
-                        BUG_ON(add == NULL);
-                        add->cd = cd;
-                        add->sm_list = DETECT_SM_LIST_PMATCH;
-                        add->cnt = 1;
-                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
-                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
-                    }
-                    break;
-                }
-            }
-            if (smd->is_last)
-                break;
-            smd++;
-        } while (1);
-    }
-
-    const DetectEngineAppInspectionEngine *app = s->app_inspect;
-    for (; app != NULL; app = app->next) {
-        SigMatchData *smd = app->smd;
-        do {
-            switch (smd->type) {
-                case DETECT_CONTENT: {
-                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;
-
-                    struct PatternItem lookup = {
-                        .cd = cd, .sm_list = app->sm_list, .cnt = 0, .mpm = 0
-                    };
-                    struct PatternItem *res =
-                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
-                    if (res) {
-                        res->cnt++;
-                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
-                    } else {
-                        struct PatternItem *add = SCCalloc(1, sizeof(*add));
-                        BUG_ON(add == NULL);
-                        add->cd = cd;
-                        add->sm_list = app->sm_list;
-                        add->cnt = 1;
-                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
-                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
-                    }
-                    break;
-                }
-            }
-            if (smd->is_last)
-                break;
-            smd++;
-        } while (1);
-    }
-    const DetectEnginePktInspectionEngine *pkt = s->pkt_inspect;
-    for (; pkt != NULL; pkt = pkt->next) {
-        SigMatchData *smd = pkt->smd;
-        do {
-            if (smd == NULL) {
-                BUG_ON(!(pkt->sm_list < DETECT_SM_LIST_DYNAMIC_START));
-                smd = s->sm_arrays[pkt->sm_list];
-            }
-            switch (smd->type) {
-                case DETECT_CONTENT: {
-                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;
-
-                    struct PatternItem lookup = {
-                        .cd = cd, .sm_list = pkt->sm_list, .cnt = 0, .mpm = 0
-                    };
-                    struct PatternItem *res =
-                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
-                    if (res) {
-                        res->cnt++;
-                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
-                    } else {
-                        struct PatternItem *add = SCCalloc(1, sizeof(*add));
-                        BUG_ON(add == NULL);
-                        add->cd = cd;
-                        add->sm_list = pkt->sm_list;
-                        add->cnt = 1;
-                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
-                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
-                    }
-                    break;
-                }
-            }
-            if (smd->is_last)
-                break;
-            smd++;
-        } while (1);
-    }
-    const DetectEngineFrameInspectionEngine *frame = s->frame_inspect;
-    for (; frame != NULL; frame = frame->next) {
-        SigMatchData *smd = frame->smd;
-        do {
-            if (smd == NULL) {
-                BUG_ON(!(frame->sm_list < DETECT_SM_LIST_DYNAMIC_START));
-                smd = s->sm_arrays[frame->sm_list];
-            }
-            switch (smd->type) {
-                case DETECT_CONTENT: {
-                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;
-
-                    struct PatternItem lookup = {
-                        .cd = cd, .sm_list = frame->sm_list, .cnt = 0, .mpm = 0
-                    };
-                    struct PatternItem *res =
-                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
-                    if (res) {
-                        res->cnt++;
-                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
-                    } else {
-                        struct PatternItem *add = SCCalloc(1, sizeof(*add));
-                        BUG_ON(add == NULL);
-                        add->cd = cd;
-                        add->sm_list = frame->sm_list;
-                        add->cnt = 1;
-                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
-                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
-                    }
-                    break;
-                }
-            }
-            if (smd->is_last)
-                break;
-            smd++;
-        } while (1);
-    }
-
-    SCReturn;
-}
-
 static void EngineAnalysisItemsReset(void)
 {
     for (size_t i = 0; i < ARRAY_SIZE(analyzer_items); i++) {

diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c
index 0c969f02337741098ad99677a257277882420d61..34357a2f3d1539e49987d59cc5fa0ef3ff9a86c2 100644 (file)
--- a/src/detect-engine-build.c
+++ b/src/detect-engine-build.c
@@ -1901,6 +1901,7 @@ static int SigMatchPrepare(DetectEngineCtx *de_ctx)
         DetectEnginePktInspectionSetup(s);
 
         if (rule_engine_analysis_set) {
+            EngineAnalysisAddAllRulePatterns(de_ctx, s);
             EngineAnalysisRules2(de_ctx, s);
         }
         /* free lists. Ctx' are xferred to sm_arrays so won't get freed */

diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
index c5cce5559d568af1ae3ae9f81ddb53d78bc876eb..c467dc26b8160c03de29a5e175bd247dc7b1608a 100644 (file)
--- a/src/detect-engine-mpm.c
+++ b/src/detect-engine-mpm.c
@@ -61,6 +61,7 @@
 #include "util-debug.h"
 #include "util-print.h"
 #include "util-validate.h"
+#include "util-hash-string.h"
 
 const char *builtin_mpms[] = {
     "toserver TCP packet",
@@ -2101,14 +2102,133 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
     return 0;
 }
 
-typedef struct DetectFPAndItsId_ {
-    PatIntId id;
-    uint16_t content_len;
-    uint32_t flags;
-    int sm_list;
+static inline uint32_t ContentFlagsForHash(const DetectContentData *cd)
+{
+    return cd->flags & (DETECT_CONTENT_NOCASE | DETECT_CONTENT_OFFSET | DETECT_CONTENT_DEPTH |
+                               DETECT_CONTENT_NEGATED | DETECT_CONTENT_ENDS_WITH);
+}
+
+/** \internal
+ *  \brief The hash function for Pattern. Hashes pattern after chop is applied.
+ *
+ *  \param ht      Pointer to the hash table.
+ *  \param data    Pointer to the Pattern.
+ *  \param datalen Not used in our case.
+ *
+ *  \retval hash The generated hash value.
+ */
+static uint32_t PatternChopHashFunc(HashListTable *ht, void *data, uint16_t datalen)
+{
+    const DetectPatternTracker *p = (DetectPatternTracker *)data;
+    uint32_t hash = p->sm_list + ContentFlagsForHash(p->cd);
+    uint16_t content_len = p->cd->content_len;
+    const uint8_t *content = p->cd->content;
+    if (p->cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
+        content += p->cd->fp_chop_offset;
+        content_len = p->cd->fp_chop_len;
+    }
+    hash += StringHashDjb2(content, content_len);
+    return hash % ht->array_size;
+}
+
+/** \internal
+ *  \brief The hash function for Pattern. Ignores chop.
+ *
+ *  \param ht      Pointer to the hash table.
+ *  \param data    Pointer to the Pattern.
+ *  \param datalen Not used in our case.
+ *
+ *  \retval hash The generated hash value.
+ */
+static uint32_t PatternNoChopHashFunc(HashListTable *ht, void *data, uint16_t datalen)
+{
+    const DetectPatternTracker *p = (DetectPatternTracker *)data;
+    uint32_t hash = p->sm_list + ContentFlagsForHash(p->cd);
+    hash += StringHashDjb2(p->cd->content, p->cd->content_len);
+    return hash % ht->array_size;
+}
+
+/**
+ * \brief The Compare function for Pattern. Compares patterns after chop is applied.
+ *
+ * \param data1 Pointer to the first Pattern.
+ * \param len1  Not used.
+ * \param data2 Pointer to the second Pattern.
+ * \param len2  Not used.
+ *
+ * \retval 1 If the 2 Patterns sent as args match.
+ * \retval 0 If the 2 Patterns sent as args do not match.
+ */
+static char PatternChopCompareFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
+{
+    const DetectPatternTracker *p1 = (DetectPatternTracker *)data1;
+    const DetectPatternTracker *p2 = (DetectPatternTracker *)data2;
+
+    if (p1->sm_list != p2->sm_list)
+        return 0;
+
+    if (ContentFlagsForHash(p1->cd) != ContentFlagsForHash(p2->cd))
+        return 0;
+
+    uint16_t p1_content_len = p1->cd->content_len;
+    uint8_t *p1_content = p1->cd->content;
+    if (p1->cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
+        p1_content += p1->cd->fp_chop_offset;
+        p1_content_len = p1->cd->fp_chop_len;
+    }
+    uint16_t p2_content_len = p2->cd->content_len;
+    uint8_t *p2_content = p2->cd->content;
+    if (p2->cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
+        p2_content += p2->cd->fp_chop_offset;
+        p2_content_len = p2->cd->fp_chop_len;
+    }
+
+    if (p1_content_len != p2_content_len)
+        return 0;
+
+    if (memcmp(p1_content, p2_content, p1_content_len) != 0) {
+        return 0;
+    }
+
+    return 1;
+}
+
+/**
+ * \brief The Compare function for Pattern. Ignores chop settings
+ *
+ * \param data1 Pointer to the first Pattern.
+ * \param len1  Not used.
+ * \param data2 Pointer to the second Pattern.
+ * \param len2  Not used.
+ *
+ * \retval 1 If the 2 Patterns sent as args match.
+ * \retval 0 If the 2 Patterns sent as args do not match.
+ */
+static char PatternNoChopCompareFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
+{
+    const DetectPatternTracker *p1 = (DetectPatternTracker *)data1;
+    const DetectPatternTracker *p2 = (DetectPatternTracker *)data2;
+
+    if (p1->sm_list != p2->sm_list)
+        return 0;
+
+    if (ContentFlagsForHash(p1->cd) != ContentFlagsForHash(p2->cd))
+        return 0;
+
+    if (p1->cd->content_len != p2->cd->content_len)
+        return 0;
+
+    if (memcmp(p1->cd->content, p2->cd->content, p1->cd->content_len) != 0) {
+        return 0;
+    }
 
-    uint8_t *content;
-} DetectFPAndItsId;
+    return 1;
+}
+
+static void PatternFreeFunc(void *ptr)
+{
+    SCFree(ptr);
+}
 
 /**
  * \brief Figured out the FP and their respective content ids for all the
@@ -2121,116 +2241,212 @@ typedef struct DetectFPAndItsId_ {
  */
 int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
 {
-    uint32_t struct_total_size = 0;
-    uint32_t content_total_size = 0;
-    Signature *s = NULL;
-
-    /* Count the amount of memory needed to store all the structures
-     * and the content of those structures. This will over estimate the
-     * true size, since duplicates are removed below, but counted here.
-     */
-    for (s = de_ctx->sig_list; s != NULL; s = s->next) {
+    uint32_t cnt = 0;
+    for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
         if (s->flags & SIG_FLAG_PREFILTER)
             continue;
 
         RetrieveFPForSig(de_ctx, s);
         if (s->init_data->mpm_sm != NULL) {
-            DetectContentData *cd = (DetectContentData *)s->init_data->mpm_sm->ctx;
-            struct_total_size += sizeof(DetectFPAndItsId);
-            content_total_size += cd->content_len;
-
             s->flags |= SIG_FLAG_PREFILTER;
+            cnt++;
         }
     }
-    /* no rules */
-    if (struct_total_size + content_total_size == 0)
+    /* no mpm rules */
+    if (cnt == 0)
         return 0;
 
-    /* array hash buffer - I've run out of ideas to name it */
-    uint8_t *ahb = SCMalloc(sizeof(uint8_t) * (struct_total_size + content_total_size));
-    if (unlikely(ahb == NULL))
-        return -1;
-
-    uint8_t *content = NULL;
-    uint16_t content_len = 0;
+    HashListTable *ht =
+            HashListTableInit(4096, PatternChopHashFunc, PatternChopCompareFunc, PatternFreeFunc);
+    BUG_ON(ht == NULL);
     PatIntId max_id = 0;
-    DetectFPAndItsId *struct_offset = (DetectFPAndItsId *)ahb;
-    uint8_t *content_offset = ahb + struct_total_size;
 
-    for (s = de_ctx->sig_list; s != NULL; s = s->next) {
-        if (s->init_data->mpm_sm != NULL) {
-            int sm_list = s->init_data->mpm_sm_list;
-            BUG_ON(sm_list == -1);
-
-            DetectContentData *cd = (DetectContentData *)s->init_data->mpm_sm->ctx;
-            DetectFPAndItsId *dup = (DetectFPAndItsId *)ahb;
-            if (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
-                content = cd->content + cd->fp_chop_offset;
-                content_len = cd->fp_chop_len;
-            } else {
-                content = cd->content;
-                content_len = cd->content_len;
-            }
-            uint32_t flags = cd->flags & DETECT_CONTENT_NOCASE;
-            /* Check for content already found on the same list */
-            for (; dup != struct_offset; dup++) {
-                if (dup->content_len != content_len)
-                    continue;
-                if (dup->sm_list != sm_list)
-                    continue;
-                if (dup->flags != flags)
-                    continue;
-                /* Check for pattern matching a duplicate. Use case insensitive matching
-                 * for case insensitive patterns. */
-                if (flags & DETECT_CONTENT_NOCASE) {
-                    if (SCMemcmpLowercase(dup->content, content, content_len) != 0)
-                        continue;
-                } else {
-                    /* Case sensitive matching */
-                    if (SCMemcmp(dup->content, content, content_len) != 0)
-                        continue;
-                }
-                /* Found a match with a previous pattern. */
-                break;
-            }
-            if (dup != struct_offset) {
-                /* Exited for-loop before the end, so found an existing match.
-                 * Use its ID. */
-                cd->id = dup->id;
-                continue;
-            }
+    for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
+        if (s->init_data->mpm_sm == NULL)
+            continue;
 
-            /* Not found, so new content. Give it a new ID and add it
-             * to the array.  Copy the content at the end of the
-             * content array.
-             */
-            struct_offset->id = max_id++;
-            cd->id = struct_offset->id;
-            struct_offset->content_len = content_len;
-            struct_offset->sm_list = sm_list;
-            struct_offset->content = content_offset;
-            struct_offset->flags = flags;
-
-            content_offset += content_len;
-
-            if (flags & DETECT_CONTENT_NOCASE) {
-                /* Need to store case-insensitive patterns as lower case
-                 * because SCMemcmpLowercase() above assumes that all
-                 * patterns are stored lower case so that it doesn't
-                 * need to re-lower its first argument.
-                 */
-                memcpy_tolower(struct_offset->content, content, content_len);
-            } else {
-                memcpy(struct_offset->content, content, content_len);
-            }
+        const int sm_list = s->init_data->mpm_sm_list;
+        BUG_ON(sm_list == -1);
+
+        DetectContentData *cd = (DetectContentData *)s->init_data->mpm_sm->ctx;
+
+        DetectPatternTracker lookup = { .cd = cd, .sm_list = sm_list, .cnt = 0, .mpm = 0 };
+        DetectPatternTracker *res = HashListTableLookup(ht, &lookup, 0);
+        if (res) {
+            res->cnt++;
+            res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
+
+            cd->id = res->cd->id;
+            SCLogDebug("%u: res id %u cnt %u", s->id, res->cd->id, res->cnt);
+        } else {
+            DetectPatternTracker *add = SCCalloc(1, sizeof(*add));
+            BUG_ON(add == NULL);
+            add->cd = cd;
+            add->sm_list = sm_list;
+            add->cnt = 1;
+            add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
+            HashListTableAdd(ht, (void *)add, 0);
 
-            struct_offset++;
-        } /* if (s->mpm_sm != NULL) */
-    } /* for */
+            cd->id = max_id++;
+            SCLogDebug("%u: add id %u cnt %u", s->id, add->cd->id, add->cnt);
+        }
+    }
 
     de_ctx->max_fp_id = max_id;
 
-    SCFree(ahb);
+    HashListTableFree(ht);
 
     return 0;
 }
+
+/** \brief add all patterns on our stats hash
+ *  Used to fill the hash later used by DumpPatterns()
+ *  \note sets up the hash table on first call
+ */
+void EngineAnalysisAddAllRulePatterns(DetectEngineCtx *de_ctx, const Signature *s)
+{
+    if (de_ctx->pattern_hash_table == NULL) {
+        de_ctx->pattern_hash_table = HashListTableInit(
+                4096, PatternNoChopHashFunc, PatternNoChopCompareFunc, PatternFreeFunc);
+        BUG_ON(de_ctx->pattern_hash_table == NULL);
+    }
+    if (s->sm_arrays[DETECT_SM_LIST_PMATCH]) {
+        SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_PMATCH];
+        do {
+            switch (smd->type) {
+                case DETECT_CONTENT: {
+                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;
+                    DetectPatternTracker lookup = {
+                        .cd = cd, .sm_list = DETECT_SM_LIST_PMATCH, .cnt = 0, .mpm = 0
+                    };
+                    DetectPatternTracker *res =
+                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
+                    if (res) {
+                        res->cnt++;
+                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                    } else {
+                        DetectPatternTracker *add = SCCalloc(1, sizeof(*add));
+                        BUG_ON(add == NULL);
+                        add->cd = cd;
+                        add->sm_list = DETECT_SM_LIST_PMATCH;
+                        add->cnt = 1;
+                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
+                    }
+                    break;
+                }
+            }
+            if (smd->is_last)
+                break;
+            smd++;
+        } while (1);
+    }
+
+    const DetectEngineAppInspectionEngine *app = s->app_inspect;
+    for (; app != NULL; app = app->next) {
+        SigMatchData *smd = app->smd;
+        do {
+            switch (smd->type) {
+                case DETECT_CONTENT: {
+                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;
+
+                    DetectPatternTracker lookup = {
+                        .cd = cd, .sm_list = app->sm_list, .cnt = 0, .mpm = 0
+                    };
+                    DetectPatternTracker *res =
+                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
+                    if (res) {
+                        res->cnt++;
+                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                    } else {
+                        DetectPatternTracker *add = SCCalloc(1, sizeof(*add));
+                        BUG_ON(add == NULL);
+                        add->cd = cd;
+                        add->sm_list = app->sm_list;
+                        add->cnt = 1;
+                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
+                    }
+                    break;
+                }
+            }
+            if (smd->is_last)
+                break;
+            smd++;
+        } while (1);
+    }
+    const DetectEnginePktInspectionEngine *pkt = s->pkt_inspect;
+    for (; pkt != NULL; pkt = pkt->next) {
+        SigMatchData *smd = pkt->smd;
+        do {
+            if (smd == NULL) {
+                BUG_ON(!(pkt->sm_list < DETECT_SM_LIST_DYNAMIC_START));
+                smd = s->sm_arrays[pkt->sm_list];
+            }
+            switch (smd->type) {
+                case DETECT_CONTENT: {
+                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;
+
+                    DetectPatternTracker lookup = {
+                        .cd = cd, .sm_list = pkt->sm_list, .cnt = 0, .mpm = 0
+                    };
+                    DetectPatternTracker *res =
+                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
+                    if (res) {
+                        res->cnt++;
+                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                    } else {
+                        DetectPatternTracker *add = SCCalloc(1, sizeof(*add));
+                        BUG_ON(add == NULL);
+                        add->cd = cd;
+                        add->sm_list = pkt->sm_list;
+                        add->cnt = 1;
+                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
+                    }
+                    break;
+                }
+            }
+            if (smd->is_last)
+                break;
+            smd++;
+        } while (1);
+    }
+    const DetectEngineFrameInspectionEngine *frame = s->frame_inspect;
+    for (; frame != NULL; frame = frame->next) {
+        SigMatchData *smd = frame->smd;
+        do {
+            if (smd == NULL) {
+                BUG_ON(!(frame->sm_list < DETECT_SM_LIST_DYNAMIC_START));
+                smd = s->sm_arrays[frame->sm_list];
+            }
+            switch (smd->type) {
+                case DETECT_CONTENT: {
+                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;
+
+                    DetectPatternTracker lookup = {
+                        .cd = cd, .sm_list = frame->sm_list, .cnt = 0, .mpm = 0
+                    };
+                    DetectPatternTracker *res =
+                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
+                    if (res) {
+                        res->cnt++;
+                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                    } else {
+                        DetectPatternTracker *add = SCCalloc(1, sizeof(*add));
+                        BUG_ON(add == NULL);
+                        add->cd = cd;
+                        add->sm_list = frame->sm_list;
+                        add->cnt = 1;
+                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
+                    }
+                    break;
+                }
+            }
+            if (smd->is_last)
+                break;
+            smd++;
+        } while (1);
+    }
+}

diff --git a/src/detect-engine-mpm.h b/src/detect-engine-mpm.h
index 566287f8c11a551e155f12411347d6f816022c3f..a9149be8e1dad45af7fb723c69e6be6bbbfd5a68 100644 (file)
--- a/src/detect-engine-mpm.h
+++ b/src/detect-engine-mpm.h
@@ -144,5 +144,7 @@ struct MpmListIdDataArgs {
     void *txv;
 };
 
+void EngineAnalysisAddAllRulePatterns(DetectEngineCtx *de_ctx, const Signature *s);
+
 #endif /* __DETECT_ENGINE_MPM_H__ */
 

diff --git a/src/detect.h b/src/detect.h
index 0c93277cec4ede219288f7817972d781c2088a86..bb5307c641e6f371fd12da72a42ded20938df138 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -673,6 +673,14 @@ typedef struct DetectBufferMpmRegistery_ {
     struct DetectBufferMpmRegistery_ *next;
 } DetectBufferMpmRegistery;
 
+/* helper structure to track pattern stats and assign pattern id's. */
+typedef struct DetectPatternTracker {
+    const struct DetectContentData_ *cd;
+    int sm_list;
+    uint32_t cnt;
+    uint32_t mpm;
+} DetectPatternTracker;
+
 typedef struct DetectReplaceList_ {
     struct DetectContentData_ *cd;
     uint8_t *found;