From: Victor Julien <victor@inliniac.net>
Date: Tue, 16 Feb 2021 16:43:08 +0000 (+0100)
Subject: detect/analyze: dump patterns facility
X-Git-Tag: suricata-7.0.0-beta1~1491
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a2e37522bb616a64b024d2981a43a89d83fe87c6;p=thirdparty%2Fsuricata.git

detect/analyze: dump patterns facility

Dump all patterns to `patterns.json`, with the pattern, a total count (`cnt`),
count of how many times this pattern is the mpm (`mpm`) and some of the flags.

Patterns are listed per buffer. So payload, http_uri, etc.
---

diff --git a/src/detect-engine-analyzer.c b/src/detect-engine-analyzer.c
index 0cad2dccd4..9491f7fb1d 100644
--- a/src/detect-engine-analyzer.c
+++ b/src/detect-engine-analyzer.c
@@ -677,11 +677,15 @@ static void DumpMatches(RuleAnalyzer *ctx, JsonBuilder *js, const SigMatchData *
     jb_close(js);
 }
 
+static void EngineAnalysisRulePatterns(const DetectEngineCtx *de_ctx, const Signature *s);
+
 SCMutex g_rules_analyzer_write_m = SCMUTEX_INITIALIZER;
 void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s)
 {
     SCEnter();
 
+    EngineAnalysisRulePatterns(de_ctx, s);
+
     RuleAnalyzer ctx = { NULL, NULL, NULL };
 
     ctx.js = jb_new_object();
@@ -956,6 +960,331 @@ void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s)
     SCReturn;
 }
 
+struct PatternItem {
+    const DetectContentData *cd;
+    int sm_list;
+    uint32_t cnt;
+    uint32_t mpm;
+};
+
+static inline uint32_t ContentFlagsForHash(const DetectContentData *cd)
+{
+    return cd->flags & (DETECT_CONTENT_NOCASE | DETECT_CONTENT_OFFSET | DETECT_CONTENT_DEPTH |
+                               DETECT_CONTENT_NEGATED | DETECT_CONTENT_ENDS_WITH);
+}
+
+/** \internal
+ *  \brief The hash function for Pattern
+ *
+ *  \param ht      Pointer to the hash table.
+ *  \param data    Pointer to the Pattern.
+ *  \param datalen Not used in our case.
+ *
+ *  \retval hash The generated hash value.
+ */
+static uint32_t PatternHashFunc(HashListTable *ht, void *data, uint16_t datalen)
+{
+    struct PatternItem *p = (struct PatternItem *)data;
+    uint32_t hash = p->sm_list + ContentFlagsForHash(p->cd);
+
+    for (uint32_t b = 0; b < p->cd->content_len; b++)
+        hash += p->cd->content[b];
+
+    return hash % ht->array_size;
+}
+
+/**
+ * \brief The Compare function for Pattern
+ *
+ * \param data1 Pointer to the first Pattern.
+ * \param len1  Not used.
+ * \param data2 Pointer to the second Pattern.
+ * \param len2  Not used.
+ *
+ * \retval 1 If the 2 Patterns sent as args match.
+ * \retval 0 If the 2 Patterns sent as args do not match.
+ */
+static char PatternCompareFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
+{
+    const struct PatternItem *p1 = (struct PatternItem *)data1;
+    const struct PatternItem *p2 = (struct PatternItem *)data2;
+
+    if (p1->sm_list != p2->sm_list)
+        return 0;
+
+    if (ContentFlagsForHash(p1->cd) != ContentFlagsForHash(p2->cd))
+        return 0;
+
+    if (p1->cd->content_len != p2->cd->content_len)
+        return 0;
+
+    if (memcmp(p1->cd->content, p2->cd->content, p1->cd->content_len) != 0) {
+        return 0;
+    }
+
+    return 1;
+}
+
+static void PatternFreeFunc(void *ptr)
+{
+    SCFree(ptr);
+}
+
+/**
+ * \brief Initializes the Pattern mpm hash table to be used by the detection
+ *        engine context.
+ *
+ * \param de_ctx Pointer to the detection engine context.
+ *
+ * \retval  0 On success.
+ * \retval -1 On failure.
+ */
+static int PatternInit(DetectEngineCtx *de_ctx)
+{
+    de_ctx->pattern_hash_table =
+            HashListTableInit(4096, PatternHashFunc, PatternCompareFunc, PatternFreeFunc);
+    if (de_ctx->pattern_hash_table == NULL)
+        goto error;
+
+    return 0;
+
+error:
+    return -1;
+}
+
+static inline bool NeedsAsHex(uint8_t c)
+{
+    if (!isprint(c))
+        return true;
+
+    switch (c) {
+        case '/':
+        case ';':
+        case ':':
+        case '\\':
+        case ' ':
+        case '|':
+        case '"':
+        case '`':
+        case '\'':
+            return true;
+    }
+    return false;
+}
+
+static void PatternPrettyPrint(const DetectContentData *cd, char *str, size_t str_len)
+{
+    bool hex = false;
+    for (uint16_t i = 0; i < cd->content_len; i++) {
+        if (NeedsAsHex(cd->content[i])) {
+            char hex_str[4];
+            snprintf(hex_str, sizeof(hex_str), "%s%02X", !hex ? "|" : " ", cd->content[i]);
+            strlcat(str, hex_str, str_len);
+            hex = true;
+        } else {
+            char p_str[3];
+            snprintf(p_str, sizeof(p_str), "%s%c", hex ? "|" : "", cd->content[i]);
+            strlcat(str, p_str, str_len);
+            hex = false;
+        }
+    }
+    if (hex) {
+        strlcat(str, "|", str_len);
+    }
+}
+
+void DumpPatterns(DetectEngineCtx *de_ctx)
+{
+    if (de_ctx->buffer_type_map_elements == 0 || de_ctx->pattern_hash_table == NULL)
+        return;
+
+    JsonBuilder *root_jb = jb_new_object();
+    JsonBuilder *arrays[de_ctx->buffer_type_map_elements];
+    memset(&arrays, 0, sizeof(JsonBuilder *) * de_ctx->buffer_type_map_elements);
+
+    jb_open_array(root_jb, "buffers");
+
+    for (HashListTableBucket *htb = HashListTableGetListHead(de_ctx->pattern_hash_table);
+            htb != NULL; htb = HashListTableGetListNext(htb)) {
+        char str[1024] = "";
+        struct PatternItem *p = HashListTableGetListData(htb);
+        PatternPrettyPrint(p->cd, str, sizeof(str));
+
+        JsonBuilder *jb = arrays[p->sm_list];
+        if (arrays[p->sm_list] == NULL) {
+            jb = arrays[p->sm_list] = jb_new_object();
+            const char *name;
+            if (p->sm_list < DETECT_SM_LIST_DYNAMIC_START)
+                name = DetectListToHumanString(p->sm_list);
+            else
+                name = DetectBufferTypeGetNameById(de_ctx, p->sm_list);
+            jb_set_string(jb, "name", name);
+            jb_set_uint(jb, "list_id", p->sm_list);
+
+            jb_open_array(jb, "patterns");
+        }
+
+        jb_start_object(jb);
+        jb_set_string(jb, "pattern", str);
+        jb_set_uint(jb, "patlen", p->cd->content_len);
+        jb_set_uint(jb, "cnt", p->cnt);
+        jb_set_uint(jb, "mpm", p->mpm);
+        jb_open_object(jb, "flags");
+        jb_set_bool(jb, "nocase", p->cd->flags & DETECT_CONTENT_NOCASE);
+        jb_set_bool(jb, "negated", p->cd->flags & DETECT_CONTENT_NEGATED);
+        jb_set_bool(jb, "depth", p->cd->flags & DETECT_CONTENT_DEPTH);
+        jb_set_bool(jb, "offset", p->cd->flags & DETECT_CONTENT_OFFSET);
+        jb_set_bool(jb, "endswith", p->cd->flags & DETECT_CONTENT_ENDS_WITH);
+        jb_close(jb);
+        jb_close(jb);
+    }
+
+    for (uint32_t i = 0; i < de_ctx->buffer_type_map_elements; i++) {
+        JsonBuilder *jb = arrays[i];
+        if (jb == NULL)
+            continue;
+
+        jb_close(jb); // array
+        jb_close(jb); // object
+
+        jb_append_object(root_jb, jb);
+        jb_free(jb);
+    }
+    jb_close(root_jb);
+    jb_close(root_jb);
+
+    const char *filename = "patterns.json";
+    const char *log_dir = ConfigGetLogDirectory();
+    char json_path[PATH_MAX] = "";
+    snprintf(json_path, sizeof(json_path), "%s/%s", log_dir, filename);
+
+    SCMutexLock(&g_rules_analyzer_write_m);
+    FILE *fp = fopen(json_path, "a");
+    if (fp != NULL) {
+        fwrite(jb_ptr(root_jb), jb_len(root_jb), 1, fp);
+        fprintf(fp, "\n");
+        fclose(fp);
+    }
+    SCMutexUnlock(&g_rules_analyzer_write_m);
+    jb_free(root_jb);
+
+    HashListTableFree(de_ctx->pattern_hash_table);
+    de_ctx->pattern_hash_table = NULL;
+}
+
+static void EngineAnalysisRulePatterns(const DetectEngineCtx *de_ctx, const Signature *s)
+{
+    SCEnter();
+
+    if (de_ctx->pattern_hash_table == NULL)
+        PatternInit((DetectEngineCtx *)de_ctx); // TODO hack const
+
+    if (s->sm_arrays[DETECT_SM_LIST_PMATCH]) {
+        SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_PMATCH];
+        do {
+            switch (smd->type) {
+                case DETECT_CONTENT: {
+                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;
+                    struct PatternItem lookup = {
+                        .cd = cd, .sm_list = DETECT_SM_LIST_PMATCH, .cnt = 0, .mpm = 0
+                    };
+                    struct PatternItem *res =
+                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
+                    if (res) {
+                        res->cnt++;
+                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                    } else {
+                        struct PatternItem *add = SCCalloc(1, sizeof(*add));
+                        BUG_ON(add == NULL);
+                        add->cd = cd;
+                        add->sm_list = DETECT_SM_LIST_PMATCH;
+                        add->cnt = 1;
+                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
+                    }
+                    break;
+                }
+            }
+            if (smd->is_last)
+                break;
+            smd++;
+        } while (1);
+    }
+
+    const DetectEngineAppInspectionEngine *app = s->app_inspect;
+    for (; app != NULL; app = app->next) {
+        SigMatchData *smd = app->smd;
+        do {
+            switch (smd->type) {
+                case DETECT_CONTENT: {
+                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;
+
+                    struct PatternItem lookup = {
+                        .cd = cd, .sm_list = app->sm_list, .cnt = 0, .mpm = 0
+                    };
+                    struct PatternItem *res =
+                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
+                    if (res) {
+                        res->cnt++;
+                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                    } else {
+                        struct PatternItem *add = SCCalloc(1, sizeof(*add));
+                        BUG_ON(add == NULL);
+                        add->cd = cd;
+                        add->sm_list = app->sm_list;
+                        add->cnt = 1;
+                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
+                    }
+                    break;
+                }
+            }
+            if (smd->is_last)
+                break;
+            smd++;
+        } while (1);
+    }
+    const DetectEnginePktInspectionEngine *pkt = s->pkt_inspect;
+    for (; pkt != NULL; pkt = pkt->next) {
+        SigMatchData *smd = pkt->smd;
+        do {
+            if (smd == NULL) {
+                BUG_ON(!(pkt->sm_list < DETECT_SM_LIST_DYNAMIC_START));
+                smd = s->sm_arrays[pkt->sm_list];
+            }
+            switch (smd->type) {
+                case DETECT_CONTENT: {
+                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;
+
+                    struct PatternItem lookup = {
+                        .cd = cd, .sm_list = pkt->sm_list, .cnt = 0, .mpm = 0
+                    };
+                    struct PatternItem *res =
+                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
+                    if (res) {
+                        res->cnt++;
+                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                    } else {
+                        struct PatternItem *add = SCCalloc(1, sizeof(*add));
+                        BUG_ON(add == NULL);
+                        add->cd = cd;
+                        add->sm_list = pkt->sm_list;
+                        add->cnt = 1;
+                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
+                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
+                    }
+                    break;
+                }
+            }
+            if (smd->is_last)
+                break;
+            smd++;
+        } while (1);
+    }
+
+    SCReturn;
+}
+
 static void EngineAnalysisItemsReset(void)
 {
     for (size_t i = 0; i < ARRAY_SIZE(analyzer_items); i++) {
diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c
index 5a38f8da80..a7b5dac265 100644
--- a/src/detect-engine-build.c
+++ b/src/detect-engine-build.c
@@ -1912,7 +1912,7 @@ static int SigMatchPrepare(DetectEngineCtx *de_ctx)
         s->init_data = NULL;
     }
 
-
+    DumpPatterns(de_ctx);
     SCReturnInt(0);
 }
 
diff --git a/src/detect.h b/src/detect.h
index 5b5fbcfe56..a9464bf578 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -796,6 +796,7 @@ typedef struct DetectEngineCtx_ {
     HashListTable *sgh_hash_table;
 
     HashListTable *mpm_hash_table;
+    HashListTable *pattern_hash_table;
 
     /* hash table used to cull out duplicate sigs */
     HashListTable *dup_sig_hash_table;
@@ -1500,6 +1501,8 @@ AppLayerDecoderEvents *DetectEngineGetEvents(DetectEngineThreadCtx *det_ctx);
 int DetectEngineGetEventInfo(const char *event_name, int *event_id,
                              AppLayerEventType *event_type);
 
+void DumpPatterns(DetectEngineCtx *de_ctx);
+
 #include "detect-engine-build.h"
 #include "detect-engine-register.h"