]> git.ipfire.org Git - thirdparty/suricata.git/commitdiff
detect/analyzer: Refactor engine analysis code
authorJeff Lucovsky <jeff@lucovsky.org>
Tue, 3 Sep 2019 21:57:54 +0000 (17:57 -0400)
committerVictor Julien <victor@inliniac.net>
Tue, 19 Nov 2019 16:24:43 +0000 (17:24 +0100)
This commit changes the analysis code to be table driven to better
identify the rule elements covered by the analysis.

src/detect-engine-analyzer.c

index 1dee79223365aee87c8d28faae5dfe748d206b83..27a38b45450f5094cde6491b0c50c72db84a0bfa 100644 (file)
@@ -51,6 +51,80 @@ typedef struct FpPatternStats_ {
     uint64_t tot;
 } FpPatternStats;
 
+/* Details for each buffer being tracked */
+typedef struct DetectEngineAnalyzerItems {
+    int16_t     item_id;
+    bool        item_seen;
+    bool        export_item_seen;
+    bool        check_encoding_match;
+    const char  *item_name;
+    const char  *display_name;
+} DetectEngineAnalyzerItems;
+
+/* Track which items require the item_seen value to be exposed */
+struct ExposedItemSeen {
+    const char  *bufname;
+    bool        *item_seen_ptr;
+};
+
+DetectEngineAnalyzerItems analyzer_items[] = {
+    /* request keywords */
+    { 0, false, false, true,  "http_uri",           "http uri" },
+    { 0, false, false, false, "http_raw_uri",       "http raw uri" },
+    { 0, false, true,  false, "http_method",        "http method" },
+    { 0, false, false, false, "http_request_line",  "http request line" },
+    { 0, false, false, false, "http_client_body",   "http client body" },
+    { 0, false, false, true,  "http_header",        "http header" },
+    { 0, false, false, false, "http_raw_header",    "http raw header" },
+    { 0, false, false, true,  "http_cookie",        "http cookie" },
+    { 0, false, false, false, "http_user_agent",    "http user agent" },
+    { 0, false, false, false, "http_host",          "http host" },
+    { 0, false, false, false, "http_raw_host",      "http raw host" },
+    { 0, false, false, false, "http_accept_enc",    "http accept enc" },
+    { 0, false, false, false, "http_referer",       "http referer" },
+    { 0, false, false, false, "http_content_type",  "http content type" },
+    { 0, false, false, false, "http_header_names",  "http header names" },
+
+    /* response keywords not listed above */
+    { 0, false, false, false, "http_stat_msg",      "http stat msg" },
+    { 0, false, false, false, "http_stat_code",     "http stat code" },
+    { 0, false, true,  false, "file_data",          "http server body"},
+
+    /* missing request keywords */
+    { 0, false, false, false, "http_request_line",  "http request line" },
+    { 0, false, false, false, "http_accept",        "http accept" },
+    { 0, false, false, false, "http_accept_lang",   "http accept lang" },
+    { 0, false, false, false, "http_connection",    "http connection" },
+    { 0, false, false, false, "http_content_len",   "http content len" },
+    { 0, false, false, false, "http_protocol",      "http protocol" },
+    { 0, false, false, false, "http_start",         "http start" },
+
+    /* missing response keywords; some of the missing are listed above*/
+    { 0, false, false, false, "http_response_line", "http response line" },
+    { 0, false, false, false, "http.server",        "http server" },
+    { 0, false, false, false, "http.location",      "http location" },
+};
+
+/*
+ * This array contains the map between the `analyzer_items` array listed above and
+ * the item ids returned by DetectBufferTypeGetByName. Iterating signature's sigmatch
+ * array provides list_ids. The map converts those ids into elements of the
+ * analyzer items array.
+ *
+ * Ultimately, the g_buffer_type_hash is searched for each buffer name. The size of that
+ * hashlist is 256, so that's the value we use here.
+ */
+int16_t analyzer_item_map[256];
+
+/*
+ * Certain values must be directly accessible. This array contains items that are directly
+ * accessed when checking if they've been seen or not.
+ */
+struct ExposedItemSeen exposed_item_seen_list[] = {
+    { .bufname = "http_method"},
+    { .bufname = "file_data"}
+};
+
 static FpPatternStats fp_pattern_stats[DETECT_SM_LIST_MAX];
 
 static void FpPatternStatsAdd(int list, uint16_t patlen)
@@ -859,6 +933,43 @@ void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s)
     SCReturn;
 }
 
+static void EngineAnalysisItemsReset(void)
+{
+    for (size_t i = 0; i < ARRAY_SIZE(analyzer_items); i++) {
+        analyzer_items[i].item_seen = false;
+    }
+}
+
+static void EngineAnalysisItemsInit(void)
+{
+    static bool analyzer_init = false;
+
+    if (analyzer_init) {
+        EngineAnalysisItemsReset();
+        return;
+    }
+
+    memset(analyzer_item_map, -1, sizeof(analyzer_item_map));
+
+    for (size_t i = 0; i < ARRAY_SIZE(analyzer_items); i++) {
+        DetectEngineAnalyzerItems *analyzer_item = &analyzer_items[i];
+
+        analyzer_item->item_id = DetectBufferTypeGetByName(analyzer_item->item_name);
+        analyzer_item->item_seen = false;
+
+        if (analyzer_item->export_item_seen) {
+            for (size_t k = 0; k < ARRAY_SIZE(exposed_item_seen_list); k++) {
+                if (0 == strcmp(exposed_item_seen_list[k].bufname, analyzer_item->item_name))
+                    exposed_item_seen_list[k].item_seen_ptr = &analyzer_item->item_seen;
+            }
+
+        }
+        analyzer_item_map[analyzer_item->item_id] = (int16_t) i;
+    }
+
+    analyzer_init = true;
+}
+
 /**
  * \brief Prints analysis of loaded rules.
  *
@@ -886,32 +997,12 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx,
     uint32_t rule_ipv6_only = 0;
     uint32_t rule_flowbits = 0;
     uint32_t rule_flowint = 0;
-    //uint32_t rule_flowvar = 0;
     uint32_t rule_content_http = 0;
     uint32_t rule_content_offset_depth = 0;
     int32_t list_id = 0;
     uint32_t rule_warning = 0;
-    uint32_t raw_http_buf = 0;
-    uint32_t norm_http_buf = 0;
     uint32_t stream_buf = 0;
     uint32_t packet_buf = 0;
-    uint32_t http_header_buf = 0;
-    uint32_t http_accept_enc_buf = 0;
-    uint32_t http_uri_buf = 0;
-    uint32_t http_method_buf = 0;
-    uint32_t http_cookie_buf = 0;
-    uint32_t http_content_type_buf = 0;
-    uint32_t http_client_body_buf = 0;
-    uint32_t http_server_body_buf = 0;
-    uint32_t http_stat_code_buf = 0;
-    uint32_t http_stat_msg_buf = 0;
-    uint32_t http_raw_header_buf = 0;
-    uint32_t http_raw_uri_buf = 0;
-    uint32_t http_ua_buf = 0;
-    uint32_t http_host_buf = 0;
-    uint32_t http_rawhost_buf = 0;
-    uint32_t http_headernames_buf = 0;
-    uint32_t http_referer_buf = 0;
     uint32_t warn_pcre_no_content = 0;
     uint32_t warn_pcre_http_content = 0;
     uint32_t warn_pcre_http = 0;
@@ -930,23 +1021,10 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx,
     uint32_t warn_no_direction = 0;
     uint32_t warn_both_direction = 0;
 
-    const int filedata_id = DetectBufferTypeGetByName("file_data");
-    const int httpmethod_id = DetectBufferTypeGetByName("http_method");
-    const int httpuri_id = DetectBufferTypeGetByName("http_uri");
-    const int httpuseragent_id = DetectBufferTypeGetByName("http_user_agent");
-    const int httpcontenttype_id = DetectBufferTypeGetByName("http_content_type");
-    const int httpcookie_id = DetectBufferTypeGetByName("http_cookie");
-    const int httpstatcode_id = DetectBufferTypeGetByName("http_stat_code");
-    const int httpstatmsg_id = DetectBufferTypeGetByName("http_stat_msg");
-    const int httpheader_id = DetectBufferTypeGetByName("http_header");
-    const int httpacceptenc_id = DetectBufferTypeGetByName("http_accept_enc");
-    const int httprawheader_id = DetectBufferTypeGetByName("http_raw_header");
-    const int httpclientbody_id = DetectBufferTypeGetByName("http_client_body");
-    const int httprawuri_id = DetectBufferTypeGetByName("http_raw_uri");
-    const int httphost_id = DetectBufferTypeGetByName("http_host");
-    const int httprawhost_id = DetectBufferTypeGetByName("http_raw_host");
-    const int httpreferer_id = DetectBufferTypeGetByName("http_referer");
-    const int httpheadernames_id = DetectBufferTypeGetByName("http_header_names");
+    EngineAnalysisItemsInit();
+
+    bool *http_method_item_seen_ptr = exposed_item_seen_list[0].item_seen_ptr;
+    bool *http_server_body_item_seen_ptr = exposed_item_seen_list[1].item_seen_ptr;
 
     if (s->init_data->init_flags & SIG_FLAG_INIT_BIDIREC) {
         rule_bidirectional = 1;
@@ -969,192 +1047,35 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx,
     for (list_id = 0; list_id < (int)s->init_data->smlists_array_size; list_id++) {
         SigMatch *sm = NULL;
         for (sm = s->init_data->smlists[list_id]; sm != NULL; sm = sm->next) {
+            int16_t item_slot = analyzer_item_map[list_id];
             if (sm->type == DETECT_PCRE) {
-                if (list_id == httpclientbody_id) {
-                    rule_pcre_http += 1;
-                    http_client_body_buf += 1;
-                    raw_http_buf += 1;
-                }
-                else if (list_id == httpuri_id) {
-                    rule_pcre_http += 1;
-                    norm_http_buf += 1;
-                    http_uri_buf += 1;
-                }
-                else if (list_id == httpheader_id) {
-                    rule_pcre_http += 1;
-                    norm_http_buf += 1;
-                    http_header_buf += 1;
-                }
-                else if (list_id == httpacceptenc_id) {
-                    rule_pcre_http += 1;
-                    norm_http_buf += 1;
-                    http_accept_enc_buf += 1;
-                }
-                else if (list_id == httpcookie_id) {
-                    rule_pcre_http += 1;
-                    norm_http_buf += 1;
-                    http_cookie_buf += 1;
-                }
-                else if (list_id == httpcontenttype_id) {
-                    rule_pcre_http += 1;
-                    norm_http_buf += 1;
-                    http_content_type_buf += 1;
-                }
-                else if (list_id == filedata_id) {
-                    rule_pcre_http += 1;
-                    http_server_body_buf += 1;
-                    raw_http_buf += 1;
-                }
-                else if (list_id == httprawheader_id) {
-                    rule_pcre_http += 1;
-                    raw_http_buf += 1;
-                    http_raw_header_buf += 1;
-                }
-                else if (list_id == httpmethod_id) {
-                    rule_pcre_http += 1;
-                    raw_http_buf += 1;
-                    http_method_buf += 1;
-                }
-                else if (list_id == httprawuri_id) {
-                    rule_pcre_http += 1;
-                    raw_http_buf += 1;
-                    http_raw_uri_buf += 1;
+                if (item_slot == -1) {
+                    rule_pcre++;
+                    continue;
                 }
-                else if (list_id == httpstatmsg_id) {
-                    rule_pcre_http += 1;
-                    raw_http_buf += 1;
-                    http_stat_msg_buf += 1;
-                }
-                else if (list_id == httpstatcode_id) {
-                    rule_pcre_http += 1;
-                    raw_http_buf += 1;
-                    http_stat_code_buf += 1;
-                }
-                else if (list_id == httpuseragent_id) {
-                    rule_pcre_http += 1;
-                    norm_http_buf += 1;
-                    http_ua_buf += 1;
-                }
-                else if (list_id == httphost_id) {
-                    rule_pcre_http += 1;
-                    norm_http_buf += 1;
-                    http_host_buf += 1;
-                }
-                else if (list_id == httprawhost_id) {
-                    rule_pcre_http += 1;
-                    raw_http_buf += 1;
-                    http_rawhost_buf += 1;
-                }
-                else if (list_id == httpheadernames_id) {
-                    rule_pcre_http += 1;
-                    raw_http_buf += 1;
-                    http_headernames_buf += 1;
-                }
-                else if (list_id == httpreferer_id) {
-                    rule_pcre_http += 1;
-                    raw_http_buf += 1;
-                    http_referer_buf += 1;
-                }
-                else {
-                    rule_pcre += 1;
+
+                rule_pcre_http++;
+                analyzer_items[item_slot].item_seen = true;
+            } else if (sm->type == DETECT_CONTENT) {
+                if (item_slot == -1) {
+                    rule_content++;
+                    if (list_id == DETECT_SM_LIST_PMATCH) {
+                        DetectContentData *cd = (DetectContentData *)sm->ctx;
+                        if (cd->flags & (DETECT_CONTENT_OFFSET | DETECT_CONTENT_DEPTH)) {
+                            rule_content_offset_depth++;
+                        }
+                    }
+                    continue;
                 }
-            }
-            else if (sm->type == DETECT_CONTENT) {
 
-                if (list_id == httpuri_id
-                          || list_id == httpheader_id
-                          || list_id == httpcookie_id) {
-                    rule_content_http += 1;
-                    norm_http_buf += 1;
+                rule_content_http++;
+                analyzer_items[item_slot].item_seen = true;
+
+                if (analyzer_items[item_slot].check_encoding_match) {
                     DetectContentData *cd = (DetectContentData *)sm->ctx;
                     if (cd != NULL && PerCentEncodingMatch(cd->content, cd->content_len) > 0) {
                         warn_encoding_norm_http_buf += 1;
-                        rule_warning += 1;
-                    }
-                    if (list_id == httpuri_id) {
-                        http_uri_buf += 1;
-                    }
-                    else if (list_id == httpheader_id) {
-                        http_header_buf += 1;
-                    }
-                    else if (list_id == httpcookie_id) {
-                        http_cookie_buf += 1;
-                    }
-                }
-                else if (list_id == httpclientbody_id) {
-                    rule_content_http += 1;
-                    raw_http_buf += 1;
-                    http_client_body_buf += 1;
-                }
-                else if (list_id == httpcontenttype_id) {
-                    rule_content_http += 1;
-                    raw_http_buf += 1;
-                    http_content_type_buf += 1;
-                }
-                else if (list_id == filedata_id) {
-                    rule_content_http += 1;
-                    raw_http_buf += 1;
-                    http_server_body_buf += 1;
-                }
-                else if (list_id == httprawheader_id) {
-                    rule_content_http += 1;
-                    raw_http_buf += 1;
-                    http_raw_header_buf += 1;
-                }
-                else if (list_id == httprawuri_id) {
-                    rule_content_http += 1;
-                    raw_http_buf += 1;
-                    http_raw_uri_buf += 1;
-                }
-                else if (list_id == httpstatmsg_id) {
-                    rule_content_http += 1;
-                    raw_http_buf += 1;
-                    http_stat_msg_buf += 1;
-                }
-                else if (list_id == httpstatcode_id) {
-                    rule_content_http += 1;
-                    raw_http_buf += 1;
-                    http_stat_code_buf += 1;
-                }
-                else if (list_id == httpmethod_id) {
-                    rule_content_http += 1;
-                    raw_http_buf += 1;
-                    http_method_buf += 1;
-                }
-                else if (list_id == httphost_id) {
-                    rule_content_http += 1;
-                    raw_http_buf += 1;
-                    http_host_buf += 1;
-                }
-                else if (list_id == httpuseragent_id) {
-                    rule_content_http += 1;
-                    norm_http_buf += 1;
-                    http_ua_buf += 1;
-                }
-                else if (list_id == httprawhost_id) {
-                    rule_content_http += 1;
-                    raw_http_buf += 1;
-                    http_rawhost_buf += 1;
-                }
-                else if (list_id == httpheadernames_id) {
-                    rule_content_http += 1;
-                    raw_http_buf += 1;
-                    http_headernames_buf += 1;
-                }
-                else if (list_id == httpreferer_id) {
-                    rule_content_http += 1;
-                    raw_http_buf += 1;
-                    http_referer_buf += 1;
-                }
-                else if (list_id == DETECT_SM_LIST_PMATCH) {
-                    rule_content += 1;
-                    DetectContentData *cd = (DetectContentData *)sm->ctx;
-                    if (cd->flags &
-                        (DETECT_CONTENT_OFFSET | DETECT_CONTENT_DEPTH)) {
-                        rule_content_offset_depth++;
                     }
-                } else {
-                    rule_content += 1;
                 }
             }
             else if (sm->type == DETECT_FLOW) {
@@ -1235,12 +1156,13 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx,
         rule_warning += 1;
         warn_direction = 1;
     }
-    if (http_method_buf) {
+
+    if (*http_method_item_seen_ptr) {
         if (rule_flow && rule_flow_toclient) {
             rule_warning += 1;
             warn_method_toclient = 1;
         }
-        if (http_server_body_buf) {
+        if (*http_server_body_item_seen_ptr) {
             rule_warning += 1;
             warn_method_serverbody = 1;
         }
@@ -1284,23 +1206,12 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx,
         if (!rule_flow_nostream && stream_buf && (rule_flow || rule_flowbits || rule_content || rule_pcre)) {
             fprintf(rule_engine_analysis_FD, "    Rule matches on reassembled stream.\n");
         }
-        if (http_uri_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http uri buffer.\n");
-        if (http_header_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http header buffer.\n");
-        if (http_accept_enc_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http accept enc buffer.\n");
-        if (http_cookie_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http cookie buffer.\n");
-        if (http_content_type_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http content type buffer.\n");
-        if (http_raw_uri_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http raw uri buffer.\n");
-        if (http_raw_header_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http raw header buffer.\n");
-        if (http_method_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http method buffer.\n");
-        if (http_server_body_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http server body buffer.\n");
-        if (http_client_body_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http client body buffer.\n");
-        if (http_stat_msg_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http stat msg buffer.\n");
-        if (http_stat_code_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http stat code buffer.\n");
-        if (http_ua_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http user agent buffer.\n");
-        if (http_host_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http host buffer.\n");
-        if (http_rawhost_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http rawhost buffer.\n");
-        if (http_headernames_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http header names buffer.\n");
-        if (http_referer_buf) fprintf(rule_engine_analysis_FD, "    Rule matches on http header referer buffer.\n");
+        for(size_t i = 0; i < ARRAY_SIZE(analyzer_items); i++) {
+            DetectEngineAnalyzerItems *ai = &analyzer_items[i];
+            if (ai->item_seen) {
+                 fprintf(rule_engine_analysis_FD, "    Rule matches on %s buffer.\n", ai->display_name);
+            }
+        }
         if (s->alproto != ALPROTO_UNKNOWN) {
             fprintf(rule_engine_analysis_FD, "    App layer protocol is %s.\n", AppProtoToString(s->alproto));
         }