profiling: output post-prefilter matches

Dump a json record containing all sigs that need to be inspected after
prefilter. Part of profiling. Only dump if threshold is met, which is
currently set by:

 --set detect.profiling.inspect-logging-threshold=200

A file called packet_inspected_rules.json is created in the default
log dir.

diff --git a/src/Makefile.am b/src/Makefile.am
index c3c7c679c116651d31ab58f983a0ee362c479dfc..4d0490a1f3f3faf97d62f14e47a04ce1bcfce756 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -121,6 +121,7 @@ detect-engine-mpm.c detect-engine-mpm.h \
 detect-engine-payload.c detect-engine-payload.h \
 detect-engine-port.c detect-engine-port.h \
 detect-engine-proto.c detect-engine-proto.h \
+detect-engine-profile.c detect-engine-profile.h \
 detect-engine-siggroup.c detect-engine-siggroup.h \
 detect-engine-sigorder.c detect-engine-sigorder.h \
 detect-engine-state.c detect-engine-state.h \

diff --git a/src/detect-engine-profile.c b/src/detect-engine-profile.c
new file mode 100644 (file)
index 0000000..f512765
--- /dev/null
+++ b/src/detect-engine-profile.c
@@ -0,0 +1,136 @@
+/* Copyright (C) 2016 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ *
+ */
+
+#include "suricata-common.h"
+#include "suricata.h"
+#include "detect.h"
+#include "detect-parse.h"
+#include "detect-content.h"
+#include "output-json.h"
+#include "util-buffer.h"
+#include "util-print.h"
+
+#ifdef PROFILING
+static void DumpFp(const SigMatch *sm, char *pat_orig, uint32_t pat_orig_sz, char *pat_chop, uint32_t pat_chop_sz)
+{
+    int fast_pattern_chop_set = 0;
+    const DetectContentData *cd = (DetectContentData *)sm->ctx;
+
+    if (cd->flags & DETECT_CONTENT_FAST_PATTERN) {
+        if (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
+            fast_pattern_chop_set = 1;
+        }
+    }
+
+    uint32_t off = 0;
+    PrintRawUriBuf(pat_orig, &off, pat_orig_sz, cd->content, cd->content_len);
+
+    if (fast_pattern_chop_set) {
+        off = 0;
+        PrintRawUriBuf(pat_chop, &off, pat_chop_sz, cd->content + cd->fp_chop_offset, cd->fp_chop_len);
+    }
+}
+
+SCMutex g_rule_dump_write_m = SCMUTEX_INITIALIZER;
+void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx, const Packet *p)
+{
+    json_t *js = CreateJSONHeader(p, 0, "inspectedrules");
+    if (js == NULL)
+        return;
+    json_t *ir = json_object();
+    if (ir == NULL)
+        return;
+
+    json_object_set_new(ir, "rule_group_id", json_integer(det_ctx->sgh->id));
+    json_object_set_new(ir, "rule_cnt", json_integer(det_ctx->match_array_cnt));
+
+    json_t *js_array = json_array();
+    uint32_t x;
+    for (x = 0; x < det_ctx->match_array_cnt; x++)
+    {
+        const Signature *s = det_ctx->match_array[x];
+        if (s == NULL)
+            continue;
+
+        json_t *js_sig = json_object();
+        if (unlikely(js == NULL))
+            continue;
+        json_object_set_new(js_sig, "sig_id", json_integer(s->id));
+
+        json_object_set_new(js_sig, "mpm", (s->mpm_sm != NULL) ? json_true() : json_false());
+
+        if (s->mpm_sm != NULL) {
+            char orig[256] = "";
+            char chop[256] = "";
+
+            DumpFp(s->mpm_sm, orig, sizeof(orig), chop, sizeof(chop));
+
+            json_object_set_new(js_sig, "mpm_buffer", json_string(DetectListToHumanString(SigMatchListSMBelongsTo(s, s->mpm_sm))));
+            json_object_set_new(js_sig, "mpm_pattern", json_string(orig));
+
+            if (strlen(chop) > 0) {
+                json_object_set_new(js_sig, "mpm_pattern_chop", json_string(chop));
+            }
+        }
+        json_array_append_new(js_array, js_sig);
+    }
+
+    json_object_set_new(ir, "rules", js_array);
+    json_object_set_new(js, "inspectedrules", ir);
+
+    const char *filename = "packet_inspected_rules.json";
+    const char *log_dir = ConfigGetLogDirectory();
+    char log_path[PATH_MAX] = "";
+    snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename);
+
+    MemBuffer *mbuf = NULL;
+    mbuf = MemBufferCreateNew(4096);
+    BUG_ON(mbuf == NULL);
+
+    OutputJSONMemBufferWrapper wrapper = {
+        .buffer = &mbuf,
+        .expand_by = 4096,
+    };
+
+    int r = json_dump_callback(js, OutputJSONMemBufferCallback, &wrapper,
+            JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII|
+            JSON_ESCAPE_SLASH);
+    if (r != 0) {
+        SCLogWarning(SC_ERR_SOCKET, "unable to serialize JSON object");
+    } else {
+        MemBufferWriteString(mbuf, "\n");
+        SCMutexLock(&g_rule_dump_write_m);
+        FILE *fp = fopen(log_path, "a");
+        if (fp != NULL) {
+            MemBufferPrintToFPAsString(mbuf, fp);
+            fclose(fp);
+            SCMutexUnlock(&g_rule_dump_write_m);
+        }
+    }
+
+    MemBufferFree(mbuf);
+    json_object_clear(js);
+    json_decref(js);
+}
+#endif /* PROFILING */

diff --git a/src/detect-engine-profile.h b/src/detect-engine-profile.h
new file mode 100644 (file)
index 0000000..d7ed1c9
--- /dev/null
+++ b/src/detect-engine-profile.h
@@ -0,0 +1,29 @@
+/* Copyright (C) 2016 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ */
+
+#ifndef _DETECT_ENGINE_PROFILE_H
+#define        _DETECT_ENGINE_PROFILE_H
+
+void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx, const Packet *p);
+
+#endif /* _DETECT_ENGINE_PROFILE_H */

diff --git a/src/detect-engine.c b/src/detect-engine.c
index c293a2ea8122aee0581bcb07fdd07ba77208a1bb..e9e9ecec5aadd441a86cf668b9c224a5f96d9d22 100644 (file)
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -881,6 +881,11 @@ static DetectEngineCtx *DetectEngineCtxInitReal(int minimal, const char *prefix)
 
 #ifdef PROFILING
     SCProfilingKeywordInitCounters(de_ctx);
+    de_ctx->profile_match_logging_threshold = UINT_MAX; // disabled
+
+    intmax_t v = 0;
+    if (ConfGetInt("detect.profiling.inspect-logging-threshold", &v) == 1)
+        de_ctx->profile_match_logging_threshold = (uint32_t)v;
 #endif
 
     SCClassConfLoadClassficationConfigFile(de_ctx, NULL);

diff --git a/src/detect.c b/src/detect.c
index 49bd118c37499373d4edbdbe8ef76b1790aff6f6..bdf85559a1e3317acd567c78913d8f6ab28305b0 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -33,6 +33,7 @@
 
 #include "detect-parse.h"
 #include "detect-engine.h"
+#include "detect-engine-profile.h"
 
 #include "detect-engine-alert.h"
 #include "detect-engine-siggroup.h"
@@ -201,7 +202,6 @@
 #include "util-optimize.h"
 #include "util-path.h"
 #include "util-mpm-ac.h"
-
 #include "runmodes.h"
 
 #include <glob.h>
@@ -1540,6 +1540,11 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
 
     SGH_PROFILING_RECORD(det_ctx, det_ctx->sgh);
 
+#ifdef PROFILING
+    if (match_cnt >= de_ctx->profile_match_logging_threshold)
+        RulesDumpMatchArray(det_ctx, p);
+#endif
+
     uint32_t sflags, next_sflags = 0;
     if (match_cnt) {
         next_s = *match_array++;

diff --git a/src/detect.h b/src/detect.h
index f70493f3cb4ccf41327c0ee26744b13de4a8ca0a..beb4aca097bf6e910b64818888c181ba613a306d 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -661,6 +661,7 @@ typedef struct DetectEngineCtx_ {
     struct SCProfileKeywordDetectCtx_ *profile_keyword_ctx;
     struct SCProfileKeywordDetectCtx_ *profile_keyword_ctx_per_list[DETECT_SM_LIST_MAX];
     struct SCProfileSghDetectCtx_ *profile_sgh_ctx;
+    uint32_t profile_match_logging_threshold;
 #endif
 
     char config_prefix[64];

diff --git a/suricata.yaml.in b/suricata.yaml.in
index 368ee3408c5d96f0777c5ece5bdc7f40895106ec..59c4d8e161f354465e41cb90b9074b36c7b2ecd9 100644 (file)
--- a/suricata.yaml.in
+++ b/suricata.yaml.in
@@ -608,6 +608,13 @@ detect-engine:
   # is started. This will limit the downtime in IPS mode.
   #- delayed-detect: yes
 
+  profiling:
+    # Log the rules that made it past the prefilter stage, per packet
+    # default is off. The threshold setting determines how many rules
+    # must have made it past pre-filter for that rule to trigger the
+    # logging.
+    #inspect-logging-threshold: 200
+
 # Suricata is multi-threaded. Here the threading can be influenced.
 threading:
   # On some cpu's/architectures it is beneficial to tie individual threads