flowbits: analyze and dump to json

Analyze flowbits to find which bits are only checked.

Track whether they are set and checked on the same level of 'statefulness'
for later used.

Dump flowbits to json including the sids that set/check etc the bit.

diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c
index 31e0545f6f5fc3df8a5a733d539240d3b673b2d5..329d43c55757c6d944d518023f65bf0177e5a982 100644 (file)
--- a/src/detect-engine-build.c
+++ b/src/detect-engine-build.c
@@ -1509,6 +1509,8 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
         SCLogConfig("building signature grouping structure, stage 1: "
                "preprocessing rules... complete");
     }
+    DetectFlowbitsAnalyze(de_ctx);
+
     return 0;
 
 error:

diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c
index 56abce0bfc6a7cbf86f29d9e20b96a19a892ef24..f1fe5b25c1d0e333d8b4b5d1ec555c30eff0a09e 100644 (file)
--- a/src/detect-flowbits.c
+++ b/src/detect-flowbits.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2010 Open Information Security Foundation
+/* Copyright (C) 2007-2017 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -247,6 +247,7 @@ int DetectFlowbitSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
         goto error;
 
     cd->idx = VarNameStoreSetupAdd(fb_name, VAR_TYPE_FLOW_BIT);
+    de_ctx->max_fb_id = MAX(cd->idx, de_ctx->max_fb_id);
     cd->cmd = fb_cmd;
 
     SCLogDebug("idx %" PRIu32 ", cmd %s, name %s",
@@ -298,6 +299,359 @@ void DetectFlowbitFree (void *ptr)
     SCFree(fd);
 }
 
+struct FBAnalyze {
+    uint16_t cnts[DETECT_FLOWBITS_CMD_MAX];
+    uint16_t state_cnts[DETECT_FLOWBITS_CMD_MAX];
+
+    uint32_t *set_sids;
+    uint32_t set_sids_idx;
+    uint32_t set_sids_size;
+
+    uint32_t *isset_sids;
+    uint32_t isset_sids_idx;
+    uint32_t isset_sids_size;
+
+    uint32_t *isnotset_sids;
+    uint32_t isnotset_sids_idx;
+    uint32_t isnotset_sids_size;
+
+    uint32_t *unset_sids;
+    uint32_t unset_sids_idx;
+    uint32_t unset_sids_size;
+
+    uint32_t *toggle_sids;
+    uint32_t toggle_sids_idx;
+    uint32_t toggle_sids_size;
+};
+#ifdef PROFILING
+#ifdef HAVE_LIBJANSSON
+static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx,
+        struct FBAnalyze *array, uint32_t elements);
+#endif
+#endif
+
+void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
+{
+    const int nlists = DetectBufferTypeMaxId();
+
+    const uint32_t max_fb_id = de_ctx->max_fb_id;
+    if (max_fb_id == 0)
+        return;
+
+#define MAX_SIDS 8
+    uint32_t array_size = max_fb_id + 1;
+    struct FBAnalyze array[array_size];
+    memset(&array, 0, array_size * sizeof(struct FBAnalyze));
+
+    SCLogDebug("fb analyzer array size: %"PRIu64,
+            (uint64_t)(array_size * sizeof(struct FBAnalyze)));
+
+    /* fill flowbit array, updating counters per sig */
+    for (uint32_t i = 0; i < de_ctx->sig_array_len; i++) {
+        const Signature *s = de_ctx->sig_array[i];
+        bool has_state = false;
+
+        /* see if the signature uses stateful matching */
+        for (int x = DETECT_SM_LIST_DYNAMIC_START; x < nlists; x++) {
+            if (s->init_data->smlists[x] == NULL)
+                continue;
+            has_state = true;
+            break;
+        }
+
+        for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) {
+            switch (sm->type) {
+                case DETECT_FLOWBITS:
+                {
+                    /* figure out the flowbit action */
+                    const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx;
+                    array[fb->idx].cnts[fb->cmd]++;
+                    if (has_state)
+                        array[fb->idx].state_cnts[fb->cmd]++;
+                    if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
+                        if (array[fb->idx].isset_sids_idx >= array[fb->idx].isset_sids_size) {
+                            uint32_t old_size = array[fb->idx].isset_sids_size;
+                            uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
+
+                            void *ptr = SCRealloc(array[fb->idx].isset_sids, new_size * sizeof(uint32_t));
+                            if (ptr == NULL)
+                                goto end;
+                            array[fb->idx].isset_sids_size = new_size;
+                            array[fb->idx].isset_sids = ptr;
+                        }
+
+                        array[fb->idx].isset_sids[array[fb->idx].isset_sids_idx] = s->num;
+                        array[fb->idx].isset_sids_idx++;
+                    } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET){
+                        if (array[fb->idx].isnotset_sids_idx >= array[fb->idx].isnotset_sids_size) {
+                            uint32_t old_size = array[fb->idx].isnotset_sids_size;
+                            uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
+
+                            void *ptr = SCRealloc(array[fb->idx].isnotset_sids, new_size * sizeof(uint32_t));
+                            if (ptr == NULL)
+                                goto end;
+                            array[fb->idx].isnotset_sids_size = new_size;
+                            array[fb->idx].isnotset_sids = ptr;
+                        }
+
+                        array[fb->idx].isnotset_sids[array[fb->idx].isnotset_sids_idx] = s->num;
+                        array[fb->idx].isnotset_sids_idx++;
+                    }
+                }
+            }
+        }
+        for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH] ; sm != NULL; sm = sm->next) {
+            switch (sm->type) {
+                case DETECT_FLOWBITS:
+                {
+                    /* figure out what flowbit action */
+                    const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx;
+                    array[fb->idx].cnts[fb->cmd]++;
+                    if (has_state)
+                        array[fb->idx].state_cnts[fb->cmd]++;
+                    if (fb->cmd == DETECT_FLOWBITS_CMD_SET) {
+                        if (array[fb->idx].set_sids_idx >= array[fb->idx].set_sids_size) {
+                            uint32_t old_size = array[fb->idx].set_sids_size;
+                            uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
+
+                            void *ptr = SCRealloc(array[fb->idx].set_sids, new_size * sizeof(uint32_t));
+                            if (ptr == NULL)
+                                goto end;
+                            array[fb->idx].set_sids_size = new_size;
+                            array[fb->idx].set_sids = ptr;
+                        }
+
+                        array[fb->idx].set_sids[array[fb->idx].set_sids_idx] = s->num;
+                        array[fb->idx].set_sids_idx++;
+                    }
+                    else if (fb->cmd == DETECT_FLOWBITS_CMD_UNSET) {
+                        if (array[fb->idx].unset_sids_idx >= array[fb->idx].unset_sids_size) {
+                            uint32_t old_size = array[fb->idx].unset_sids_size;
+                            uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
+
+                            void *ptr = SCRealloc(array[fb->idx].unset_sids, new_size * sizeof(uint32_t));
+                            if (ptr == NULL)
+                                goto end;
+                            array[fb->idx].unset_sids_size = new_size;
+                            array[fb->idx].unset_sids = ptr;
+                        }
+
+                        array[fb->idx].unset_sids[array[fb->idx].unset_sids_idx] = s->num;
+                        array[fb->idx].unset_sids_idx++;
+                    }
+                    else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) {
+                        if (array[fb->idx].toggle_sids_idx >= array[fb->idx].toggle_sids_size) {
+                            uint32_t old_size = array[fb->idx].toggle_sids_size;
+                            uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
+
+                            void *ptr = SCRealloc(array[fb->idx].toggle_sids, new_size * sizeof(uint32_t));
+                            if (ptr == NULL)
+                                goto end;
+                            array[fb->idx].toggle_sids_size = new_size;
+                            array[fb->idx].toggle_sids = ptr;
+                        }
+
+                        array[fb->idx].toggle_sids[array[fb->idx].toggle_sids_idx] = s->num;
+                        array[fb->idx].toggle_sids_idx++;
+                    }
+                }
+            }
+        }
+    }
+
+    /* walk array to see if all bits make sense */
+    for (uint32_t i = 0; i < array_size; i++) {
+        char *varname = VarNameStoreSetupLookup(i, VAR_TYPE_FLOW_BIT);
+        if (varname == NULL)
+            continue;
+
+        if (array[i].cnts[DETECT_FLOWBITS_CMD_ISSET] &&
+            array[i].cnts[DETECT_FLOWBITS_CMD_TOGGLE] == 0 &&
+            array[i].cnts[DETECT_FLOWBITS_CMD_SET] == 0) {
+
+            const Signature *s = de_ctx->sig_array[array[i].isset_sids[0]];
+            SCLogWarning(SC_WARN_FLOWBIT, "flowbit '%s' is checked but not "
+                    "set. Checked in %u and %u other sigs",
+                    varname, s->id, array[i].isset_sids_idx - 1);
+        }
+        if (array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET] &&
+            array[i].state_cnts[DETECT_FLOWBITS_CMD_SET] == 0)
+        {
+            SCLogDebug("flowbit %s/%u: isset in state, set not in state", varname, i);
+        }
+        if (array[i].cnts[DETECT_FLOWBITS_CMD_ISSET] > 0 &&
+            array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET] == 0 &&
+            array[i].state_cnts[DETECT_FLOWBITS_CMD_SET])
+        {
+            SCLogDebug("flowbit %s/%u: isset not in state, set in state", varname, i);
+        }
+
+        SCLogDebug("ALL flowbit %s/%u: sets %u toggles %u unsets %u isnotsets %u issets %u", varname, i,
+                array[i].cnts[DETECT_FLOWBITS_CMD_SET], array[i].cnts[DETECT_FLOWBITS_CMD_TOGGLE],
+                array[i].cnts[DETECT_FLOWBITS_CMD_UNSET], array[i].cnts[DETECT_FLOWBITS_CMD_ISNOTSET],
+                array[i].cnts[DETECT_FLOWBITS_CMD_ISSET]);
+        SCLogDebug("STATE flowbit %s/%u: sets %u toggles %u unsets %u isnotsets %u issets %u", varname, i,
+                array[i].state_cnts[DETECT_FLOWBITS_CMD_SET], array[i].state_cnts[DETECT_FLOWBITS_CMD_TOGGLE],
+                array[i].state_cnts[DETECT_FLOWBITS_CMD_UNSET], array[i].state_cnts[DETECT_FLOWBITS_CMD_ISNOTSET],
+                array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET]);
+        for (uint32_t x = 0; x < array[i].set_sids_idx; x++) {
+            SCLogDebug("SET flowbit %s/%u: SID %u", varname, i,
+                    de_ctx->sig_array[array[i].set_sids[x]]->id);
+        }
+        for (uint32_t x = 0; x < array[i].isset_sids_idx; x++) {
+            SCLogDebug("GET flowbit %s/%u: SID %u", varname, i,
+                    de_ctx->sig_array[array[i].isset_sids[x]]->id);
+        }
+        SCFree(varname);
+    }
+#ifdef PROFILING
+#ifdef HAVE_LIBJANSSON
+    DetectFlowbitsAnalyzeDump(de_ctx, array, array_size);
+#endif
+#endif
+
+end:
+    for (uint32_t i = 0; i < array_size; i++) {
+        SCFree(array[i].set_sids);
+        SCFree(array[i].unset_sids);
+        SCFree(array[i].isset_sids);
+        SCFree(array[i].isnotset_sids);
+        SCFree(array[i].toggle_sids);
+    }
+}
+
+#ifdef PROFILING
+#ifdef HAVE_LIBJANSSON
+#include "output-json.h"
+#include "util-buffer.h"
+SCMutex g_flowbits_dump_write_m = SCMUTEX_INITIALIZER;
+static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx,
+        struct FBAnalyze *array, uint32_t elements)
+{
+    json_t *js = json_object();
+    if (js == NULL)
+        return;
+
+    json_t *js_array = json_array();
+    uint32_t x;
+    for (x = 0; x < elements; x++)
+    {
+        char *varname = VarNameStoreSetupLookup(x, VAR_TYPE_FLOW_BIT);
+        if (varname == NULL)
+            continue;
+
+        const struct FBAnalyze *e = &array[x];
+
+        json_t *js_fb = json_object();
+        if (unlikely(js_fb != NULL)) {
+            json_object_set_new(js_fb, "name", json_string(varname));
+            json_object_set_new(js_fb, "internal_id", json_integer(x));
+            json_object_set_new(js_fb, "set_cnt", json_integer(e->cnts[DETECT_FLOWBITS_CMD_SET]));
+            json_object_set_new(js_fb, "unset_cnt", json_integer(e->cnts[DETECT_FLOWBITS_CMD_UNSET]));
+            json_object_set_new(js_fb, "toggle_cnt", json_integer(e->cnts[DETECT_FLOWBITS_CMD_TOGGLE]));
+            json_object_set_new(js_fb, "isset_cnt", json_integer(e->cnts[DETECT_FLOWBITS_CMD_ISSET]));
+            json_object_set_new(js_fb, "isnotset_cnt", json_integer(e->cnts[DETECT_FLOWBITS_CMD_ISNOTSET]));
+
+            // sets
+            if (e->cnts[DETECT_FLOWBITS_CMD_SET]) {
+                json_t *js_set_array = json_array();
+                if (js_set_array) {
+                    for(uint32_t i = 0; i < e->set_sids_idx; i++) {
+                        const Signature *s = de_ctx->sig_array[e->set_sids[i]];
+                        json_array_append_new(js_set_array, json_integer(s->id));
+                    }
+                    json_object_set_new(js_fb, "sets", js_set_array);
+                }
+            }
+            // gets
+            if (e->cnts[DETECT_FLOWBITS_CMD_ISSET]) {
+                json_t *js_isset_array = json_array();
+                if (js_isset_array) {
+                    for(uint32_t i = 0; i < e->isset_sids_idx; i++) {
+                        const Signature *s = de_ctx->sig_array[e->isset_sids[i]];
+                        json_array_append_new(js_isset_array, json_integer(s->id));
+                    }
+                    json_object_set_new(js_fb, "isset", js_isset_array);
+                }
+            }
+            // isnotset
+            if (e->cnts[DETECT_FLOWBITS_CMD_ISNOTSET]) {
+                json_t *js_isnotset_array = json_array();
+                if (js_isnotset_array) {
+                    for(uint32_t i = 0; i < e->isnotset_sids_idx; i++) {
+                        const Signature *s = de_ctx->sig_array[e->isnotset_sids[i]];
+                        json_array_append_new(js_isnotset_array, json_integer(s->id));
+                    }
+                    json_object_set_new(js_fb, "isnotset", js_isnotset_array);
+                }
+            }
+            // unset
+            if (e->cnts[DETECT_FLOWBITS_CMD_UNSET]) {
+                json_t *js_unset_array = json_array();
+                if (js_unset_array) {
+                    for(uint32_t i = 0; i < e->unset_sids_idx; i++) {
+                        const Signature *s = de_ctx->sig_array[e->unset_sids[i]];
+                        json_array_append_new(js_unset_array, json_integer(s->id));
+                    }
+                    json_object_set_new(js_fb, "unset", js_unset_array);
+                }
+            }
+            // toggle
+            if (e->cnts[DETECT_FLOWBITS_CMD_TOGGLE]) {
+                json_t *js_toggle_array = json_array();
+                if (js_toggle_array) {
+                    for(uint32_t i = 0; i < e->toggle_sids_idx; i++) {
+                        const Signature *s = de_ctx->sig_array[e->toggle_sids[i]];
+                        json_array_append_new(js_toggle_array, json_integer(s->id));
+                    }
+                    json_object_set_new(js_fb, "toggle", js_toggle_array);
+                }
+            }
+
+            json_array_append_new(js_array, js_fb);
+        }
+        SCFree(varname);
+    }
+
+    json_object_set_new(js, "flowbits", js_array);
+
+    const char *filename = "flowbits.json";
+    const char *log_dir = ConfigGetLogDirectory();
+    char log_path[PATH_MAX] = "";
+    snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename);
+
+    MemBuffer *mbuf = NULL;
+    mbuf = MemBufferCreateNew(4096);
+    BUG_ON(mbuf == NULL);
+
+    OutputJSONMemBufferWrapper wrapper = {
+        .buffer = &mbuf,
+        .expand_by = 4096,
+    };
+
+    int r = json_dump_callback(js, OutputJSONMemBufferCallback, &wrapper,
+            JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII|
+            JSON_ESCAPE_SLASH);
+    if (r != 0) {
+        SCLogWarning(SC_ERR_SOCKET, "unable to serialize JSON object");
+    } else {
+        MemBufferWriteString(mbuf, "\n");
+        SCMutexLock(&g_flowbits_dump_write_m);
+        FILE *fp = fopen(log_path, "w");
+        if (fp != NULL) {
+            MemBufferPrintToFPAsString(mbuf, fp);
+            fclose(fp);
+        }
+        SCMutexUnlock(&g_flowbits_dump_write_m);
+    }
+
+    MemBufferFree(mbuf);
+    json_object_clear(js);
+    json_decref(js);
+}
+#endif /* HAVE_LIBJANSSON */
+#endif /* PROFILING */
+
 #ifdef UNITTESTS
 
 static int FlowBitsTestParse01(void)

diff --git a/src/detect.h b/src/detect.h
index 4ea043d8f29ff8c0126cc98f63b13ab9b0c821f3..a98cfcb15a04d718633269ce429ed8063afe7abb 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -672,6 +672,9 @@ typedef struct DetectEngineCtx_ {
     /* specify the configuration for mpm context factory */
     uint8_t sgh_mpm_context;
 
+    /* max flowbit id that is used */
+    uint32_t max_fb_id;
+
     uint32_t max_fp_id;
 
     MpmCtxFactoryContainer *mpm_ctx_factory_container;
@@ -1247,6 +1250,8 @@ int SigMatchSignaturesRunPostMatch(ThreadVars *tv,
                                    const Signature *s);
 void DetectSignatureApplyActions(Packet *p, const Signature *s, const uint8_t);
 
+void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx);
+
 #include "detect-engine-build.h"
 #include "detect-engine-register.h"
 

diff --git a/src/util-error.c b/src/util-error.c
index 89b994f1fa4f8774acb7c2a66824384086a00d42..3f8209746bb98a2470089001e3691e9297d1f239 100644 (file)
--- a/src/util-error.c
+++ b/src/util-error.c
@@ -346,6 +346,8 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_WARN_RENAMING_FILE);
         CASE_CODE (SC_ERR_PF_RING_VLAN);
         CASE_CODE (SC_ERR_CREATE_DIRECTORY);
+        CASE_CODE (SC_WARN_FLOWBIT);
+
         CASE_CODE (SC_ERR_MAX);
     }
 

diff --git a/src/util-error.h b/src/util-error.h
index b391ee06218237833ff1778d44196c54689b4772..94f0a31f1e8d5732f785a97b0d23e2d529232d4a 100644 (file)
--- a/src/util-error.h
+++ b/src/util-error.h
@@ -336,6 +336,8 @@ typedef enum {
     SC_WARN_RENAMING_FILE,
     SC_ERR_PF_RING_VLAN,
     SC_ERR_CREATE_DIRECTORY,
+    SC_WARN_FLOWBIT,
+
     SC_ERR_MAX,
 } SCError;
 

diff --git a/src/util-var-name.c b/src/util-var-name.c
index 276b8723dc2e9c1fd23b6d6fe73dacdebfaf7804..31923f2c405d59aeb8d7976ae1382cab5c2c9f31 100644 (file)
--- a/src/util-var-name.c
+++ b/src/util-var-name.c
@@ -203,7 +203,6 @@ error:
     return 0;
 }
 
-#if 0
 /** \brief Get a name from the idx.
  *  \param idx index of the variable whose name is to be fetched
  *  \param type variable type
@@ -211,7 +210,7 @@ error:
  *  \retval name of the variable if successful.
  *  \todo no alloc on lookup
  */
-static const char *VariableIdxGetName(VarNameStore *v, uint32_t idx, enum VarTypes type)
+static char *VariableIdxGetName(VarNameStore *v, uint32_t idx, enum VarTypes type)
 {
     VariableName *fn = SCMalloc(sizeof(VariableName));
     if (unlikely(fn == NULL))
@@ -239,7 +238,7 @@ error:
     VariableNameFree(fn);
     return NULL;
 }
-#endif
+
 /** \brief setup staging store. Include current store if there is one.
  */
 int VarNameStoreSetupStaging(uint32_t de_ctx_version)
@@ -327,6 +326,14 @@ uint32_t VarNameStoreSetupAdd(const char *name, const enum VarTypes type)
     return id;
 }
 
+char *VarNameStoreSetupLookup(uint32_t idx, const enum VarTypes type)
+{
+    SCMutexLock(&g_varnamestore_staging_m);
+    char *name = VariableIdxGetName(g_varnamestore_staging, idx, type);
+    SCMutexUnlock(&g_varnamestore_staging_m);
+    return name;
+}
+
 void VarNameStoreActivateStaging(void)
 {
     SCMutexLock(&g_varnamestore_staging_m);

diff --git a/src/util-var-name.h b/src/util-var-name.h
index c620d74eb5a49cf50ee4aa01005f09d37ca8ad91..5b0b18d716c78c9fcb7678ed276f3175facfacdf 100644 (file)
--- a/src/util-var-name.h
+++ b/src/util-var-name.h
@@ -28,6 +28,7 @@ int VarNameStoreSetupStaging(uint32_t de_ctx_version);
 const char *VarNameStoreLookupById(const uint32_t id, const enum VarTypes type);
 uint32_t VarNameStoreLookupByName(const char *name, const enum VarTypes type);
 uint32_t VarNameStoreSetupAdd(const char *name, const enum VarTypes type);
+char *VarNameStoreSetupLookup(uint32_t idx, const enum VarTypes type);
 void VarNameStoreActivateStaging(void);
 void VarNameStoreFreeOld(void);
 void VarNameStoreFree(uint32_t de_ctx_version);