From ae198add6da9d87b0f30d0bb64f7aafba65efd62 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Tue, 3 Sep 2019 17:57:54 -0400 Subject: [PATCH] detect/analyzer: Refactor engine analysis code This commit changes the analysis code to be table driven to better identify the rule elements covered by the analysis. --- src/detect-engine-analyzer.c | 379 ++++++++++++++--------------------- 1 file changed, 145 insertions(+), 234 deletions(-) diff --git a/src/detect-engine-analyzer.c b/src/detect-engine-analyzer.c index 1dee792233..27a38b4545 100644 --- a/src/detect-engine-analyzer.c +++ b/src/detect-engine-analyzer.c @@ -51,6 +51,80 @@ typedef struct FpPatternStats_ { uint64_t tot; } FpPatternStats; +/* Details for each buffer being tracked */ +typedef struct DetectEngineAnalyzerItems { + int16_t item_id; + bool item_seen; + bool export_item_seen; + bool check_encoding_match; + const char *item_name; + const char *display_name; +} DetectEngineAnalyzerItems; + +/* Track which items require the item_seen value to be exposed */ +struct ExposedItemSeen { + const char *bufname; + bool *item_seen_ptr; +}; + +DetectEngineAnalyzerItems analyzer_items[] = { + /* request keywords */ + { 0, false, false, true, "http_uri", "http uri" }, + { 0, false, false, false, "http_raw_uri", "http raw uri" }, + { 0, false, true, false, "http_method", "http method" }, + { 0, false, false, false, "http_request_line", "http request line" }, + { 0, false, false, false, "http_client_body", "http client body" }, + { 0, false, false, true, "http_header", "http header" }, + { 0, false, false, false, "http_raw_header", "http raw header" }, + { 0, false, false, true, "http_cookie", "http cookie" }, + { 0, false, false, false, "http_user_agent", "http user agent" }, + { 0, false, false, false, "http_host", "http host" }, + { 0, false, false, false, "http_raw_host", "http raw host" }, + { 0, false, false, false, "http_accept_enc", "http accept enc" }, + { 0, false, false, false, "http_referer", "http referer" }, + { 0, false, false, false, "http_content_type", "http content type" }, + { 0, false, false, false, "http_header_names", "http header names" }, + + /* response keywords not listed above */ + { 0, false, false, false, "http_stat_msg", "http stat msg" }, + { 0, false, false, false, "http_stat_code", "http stat code" }, + { 0, false, true, false, "file_data", "http server body"}, + + /* missing request keywords */ + { 0, false, false, false, "http_request_line", "http request line" }, + { 0, false, false, false, "http_accept", "http accept" }, + { 0, false, false, false, "http_accept_lang", "http accept lang" }, + { 0, false, false, false, "http_connection", "http connection" }, + { 0, false, false, false, "http_content_len", "http content len" }, + { 0, false, false, false, "http_protocol", "http protocol" }, + { 0, false, false, false, "http_start", "http start" }, + + /* missing response keywords; some of the missing are listed above*/ + { 0, false, false, false, "http_response_line", "http response line" }, + { 0, false, false, false, "http.server", "http server" }, + { 0, false, false, false, "http.location", "http location" }, +}; + +/* + * This array contains the map between the `analyzer_items` array listed above and + * the item ids returned by DetectBufferTypeGetByName. Iterating signature's sigmatch + * array provides list_ids. The map converts those ids into elements of the + * analyzer items array. + * + * Ultimately, the g_buffer_type_hash is searched for each buffer name. The size of that + * hashlist is 256, so that's the value we use here. + */ +int16_t analyzer_item_map[256]; + +/* + * Certain values must be directly accessible. This array contains items that are directly + * accessed when checking if they've been seen or not. + */ +struct ExposedItemSeen exposed_item_seen_list[] = { + { .bufname = "http_method"}, + { .bufname = "file_data"} +}; + static FpPatternStats fp_pattern_stats[DETECT_SM_LIST_MAX]; static void FpPatternStatsAdd(int list, uint16_t patlen) @@ -859,6 +933,43 @@ void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s) SCReturn; } +static void EngineAnalysisItemsReset(void) +{ + for (size_t i = 0; i < ARRAY_SIZE(analyzer_items); i++) { + analyzer_items[i].item_seen = false; + } +} + +static void EngineAnalysisItemsInit(void) +{ + static bool analyzer_init = false; + + if (analyzer_init) { + EngineAnalysisItemsReset(); + return; + } + + memset(analyzer_item_map, -1, sizeof(analyzer_item_map)); + + for (size_t i = 0; i < ARRAY_SIZE(analyzer_items); i++) { + DetectEngineAnalyzerItems *analyzer_item = &analyzer_items[i]; + + analyzer_item->item_id = DetectBufferTypeGetByName(analyzer_item->item_name); + analyzer_item->item_seen = false; + + if (analyzer_item->export_item_seen) { + for (size_t k = 0; k < ARRAY_SIZE(exposed_item_seen_list); k++) { + if (0 == strcmp(exposed_item_seen_list[k].bufname, analyzer_item->item_name)) + exposed_item_seen_list[k].item_seen_ptr = &analyzer_item->item_seen; + } + + } + analyzer_item_map[analyzer_item->item_id] = (int16_t) i; + } + + analyzer_init = true; +} + /** * \brief Prints analysis of loaded rules. * @@ -886,32 +997,12 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx, uint32_t rule_ipv6_only = 0; uint32_t rule_flowbits = 0; uint32_t rule_flowint = 0; - //uint32_t rule_flowvar = 0; uint32_t rule_content_http = 0; uint32_t rule_content_offset_depth = 0; int32_t list_id = 0; uint32_t rule_warning = 0; - uint32_t raw_http_buf = 0; - uint32_t norm_http_buf = 0; uint32_t stream_buf = 0; uint32_t packet_buf = 0; - uint32_t http_header_buf = 0; - uint32_t http_accept_enc_buf = 0; - uint32_t http_uri_buf = 0; - uint32_t http_method_buf = 0; - uint32_t http_cookie_buf = 0; - uint32_t http_content_type_buf = 0; - uint32_t http_client_body_buf = 0; - uint32_t http_server_body_buf = 0; - uint32_t http_stat_code_buf = 0; - uint32_t http_stat_msg_buf = 0; - uint32_t http_raw_header_buf = 0; - uint32_t http_raw_uri_buf = 0; - uint32_t http_ua_buf = 0; - uint32_t http_host_buf = 0; - uint32_t http_rawhost_buf = 0; - uint32_t http_headernames_buf = 0; - uint32_t http_referer_buf = 0; uint32_t warn_pcre_no_content = 0; uint32_t warn_pcre_http_content = 0; uint32_t warn_pcre_http = 0; @@ -930,23 +1021,10 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx, uint32_t warn_no_direction = 0; uint32_t warn_both_direction = 0; - const int filedata_id = DetectBufferTypeGetByName("file_data"); - const int httpmethod_id = DetectBufferTypeGetByName("http_method"); - const int httpuri_id = DetectBufferTypeGetByName("http_uri"); - const int httpuseragent_id = DetectBufferTypeGetByName("http_user_agent"); - const int httpcontenttype_id = DetectBufferTypeGetByName("http_content_type"); - const int httpcookie_id = DetectBufferTypeGetByName("http_cookie"); - const int httpstatcode_id = DetectBufferTypeGetByName("http_stat_code"); - const int httpstatmsg_id = DetectBufferTypeGetByName("http_stat_msg"); - const int httpheader_id = DetectBufferTypeGetByName("http_header"); - const int httpacceptenc_id = DetectBufferTypeGetByName("http_accept_enc"); - const int httprawheader_id = DetectBufferTypeGetByName("http_raw_header"); - const int httpclientbody_id = DetectBufferTypeGetByName("http_client_body"); - const int httprawuri_id = DetectBufferTypeGetByName("http_raw_uri"); - const int httphost_id = DetectBufferTypeGetByName("http_host"); - const int httprawhost_id = DetectBufferTypeGetByName("http_raw_host"); - const int httpreferer_id = DetectBufferTypeGetByName("http_referer"); - const int httpheadernames_id = DetectBufferTypeGetByName("http_header_names"); + EngineAnalysisItemsInit(); + + bool *http_method_item_seen_ptr = exposed_item_seen_list[0].item_seen_ptr; + bool *http_server_body_item_seen_ptr = exposed_item_seen_list[1].item_seen_ptr; if (s->init_data->init_flags & SIG_FLAG_INIT_BIDIREC) { rule_bidirectional = 1; @@ -969,192 +1047,35 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx, for (list_id = 0; list_id < (int)s->init_data->smlists_array_size; list_id++) { SigMatch *sm = NULL; for (sm = s->init_data->smlists[list_id]; sm != NULL; sm = sm->next) { + int16_t item_slot = analyzer_item_map[list_id]; if (sm->type == DETECT_PCRE) { - if (list_id == httpclientbody_id) { - rule_pcre_http += 1; - http_client_body_buf += 1; - raw_http_buf += 1; - } - else if (list_id == httpuri_id) { - rule_pcre_http += 1; - norm_http_buf += 1; - http_uri_buf += 1; - } - else if (list_id == httpheader_id) { - rule_pcre_http += 1; - norm_http_buf += 1; - http_header_buf += 1; - } - else if (list_id == httpacceptenc_id) { - rule_pcre_http += 1; - norm_http_buf += 1; - http_accept_enc_buf += 1; - } - else if (list_id == httpcookie_id) { - rule_pcre_http += 1; - norm_http_buf += 1; - http_cookie_buf += 1; - } - else if (list_id == httpcontenttype_id) { - rule_pcre_http += 1; - norm_http_buf += 1; - http_content_type_buf += 1; - } - else if (list_id == filedata_id) { - rule_pcre_http += 1; - http_server_body_buf += 1; - raw_http_buf += 1; - } - else if (list_id == httprawheader_id) { - rule_pcre_http += 1; - raw_http_buf += 1; - http_raw_header_buf += 1; - } - else if (list_id == httpmethod_id) { - rule_pcre_http += 1; - raw_http_buf += 1; - http_method_buf += 1; - } - else if (list_id == httprawuri_id) { - rule_pcre_http += 1; - raw_http_buf += 1; - http_raw_uri_buf += 1; + if (item_slot == -1) { + rule_pcre++; + continue; } - else if (list_id == httpstatmsg_id) { - rule_pcre_http += 1; - raw_http_buf += 1; - http_stat_msg_buf += 1; - } - else if (list_id == httpstatcode_id) { - rule_pcre_http += 1; - raw_http_buf += 1; - http_stat_code_buf += 1; - } - else if (list_id == httpuseragent_id) { - rule_pcre_http += 1; - norm_http_buf += 1; - http_ua_buf += 1; - } - else if (list_id == httphost_id) { - rule_pcre_http += 1; - norm_http_buf += 1; - http_host_buf += 1; - } - else if (list_id == httprawhost_id) { - rule_pcre_http += 1; - raw_http_buf += 1; - http_rawhost_buf += 1; - } - else if (list_id == httpheadernames_id) { - rule_pcre_http += 1; - raw_http_buf += 1; - http_headernames_buf += 1; - } - else if (list_id == httpreferer_id) { - rule_pcre_http += 1; - raw_http_buf += 1; - http_referer_buf += 1; - } - else { - rule_pcre += 1; + + rule_pcre_http++; + analyzer_items[item_slot].item_seen = true; + } else if (sm->type == DETECT_CONTENT) { + if (item_slot == -1) { + rule_content++; + if (list_id == DETECT_SM_LIST_PMATCH) { + DetectContentData *cd = (DetectContentData *)sm->ctx; + if (cd->flags & (DETECT_CONTENT_OFFSET | DETECT_CONTENT_DEPTH)) { + rule_content_offset_depth++; + } + } + continue; } - } - else if (sm->type == DETECT_CONTENT) { - if (list_id == httpuri_id - || list_id == httpheader_id - || list_id == httpcookie_id) { - rule_content_http += 1; - norm_http_buf += 1; + rule_content_http++; + analyzer_items[item_slot].item_seen = true; + + if (analyzer_items[item_slot].check_encoding_match) { DetectContentData *cd = (DetectContentData *)sm->ctx; if (cd != NULL && PerCentEncodingMatch(cd->content, cd->content_len) > 0) { warn_encoding_norm_http_buf += 1; - rule_warning += 1; - } - if (list_id == httpuri_id) { - http_uri_buf += 1; - } - else if (list_id == httpheader_id) { - http_header_buf += 1; - } - else if (list_id == httpcookie_id) { - http_cookie_buf += 1; - } - } - else if (list_id == httpclientbody_id) { - rule_content_http += 1; - raw_http_buf += 1; - http_client_body_buf += 1; - } - else if (list_id == httpcontenttype_id) { - rule_content_http += 1; - raw_http_buf += 1; - http_content_type_buf += 1; - } - else if (list_id == filedata_id) { - rule_content_http += 1; - raw_http_buf += 1; - http_server_body_buf += 1; - } - else if (list_id == httprawheader_id) { - rule_content_http += 1; - raw_http_buf += 1; - http_raw_header_buf += 1; - } - else if (list_id == httprawuri_id) { - rule_content_http += 1; - raw_http_buf += 1; - http_raw_uri_buf += 1; - } - else if (list_id == httpstatmsg_id) { - rule_content_http += 1; - raw_http_buf += 1; - http_stat_msg_buf += 1; - } - else if (list_id == httpstatcode_id) { - rule_content_http += 1; - raw_http_buf += 1; - http_stat_code_buf += 1; - } - else if (list_id == httpmethod_id) { - rule_content_http += 1; - raw_http_buf += 1; - http_method_buf += 1; - } - else if (list_id == httphost_id) { - rule_content_http += 1; - raw_http_buf += 1; - http_host_buf += 1; - } - else if (list_id == httpuseragent_id) { - rule_content_http += 1; - norm_http_buf += 1; - http_ua_buf += 1; - } - else if (list_id == httprawhost_id) { - rule_content_http += 1; - raw_http_buf += 1; - http_rawhost_buf += 1; - } - else if (list_id == httpheadernames_id) { - rule_content_http += 1; - raw_http_buf += 1; - http_headernames_buf += 1; - } - else if (list_id == httpreferer_id) { - rule_content_http += 1; - raw_http_buf += 1; - http_referer_buf += 1; - } - else if (list_id == DETECT_SM_LIST_PMATCH) { - rule_content += 1; - DetectContentData *cd = (DetectContentData *)sm->ctx; - if (cd->flags & - (DETECT_CONTENT_OFFSET | DETECT_CONTENT_DEPTH)) { - rule_content_offset_depth++; } - } else { - rule_content += 1; } } else if (sm->type == DETECT_FLOW) { @@ -1235,12 +1156,13 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx, rule_warning += 1; warn_direction = 1; } - if (http_method_buf) { + + if (*http_method_item_seen_ptr) { if (rule_flow && rule_flow_toclient) { rule_warning += 1; warn_method_toclient = 1; } - if (http_server_body_buf) { + if (*http_server_body_item_seen_ptr) { rule_warning += 1; warn_method_serverbody = 1; } @@ -1284,23 +1206,12 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx, if (!rule_flow_nostream && stream_buf && (rule_flow || rule_flowbits || rule_content || rule_pcre)) { fprintf(rule_engine_analysis_FD, " Rule matches on reassembled stream.\n"); } - if (http_uri_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http uri buffer.\n"); - if (http_header_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http header buffer.\n"); - if (http_accept_enc_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http accept enc buffer.\n"); - if (http_cookie_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http cookie buffer.\n"); - if (http_content_type_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http content type buffer.\n"); - if (http_raw_uri_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http raw uri buffer.\n"); - if (http_raw_header_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http raw header buffer.\n"); - if (http_method_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http method buffer.\n"); - if (http_server_body_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http server body buffer.\n"); - if (http_client_body_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http client body buffer.\n"); - if (http_stat_msg_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http stat msg buffer.\n"); - if (http_stat_code_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http stat code buffer.\n"); - if (http_ua_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http user agent buffer.\n"); - if (http_host_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http host buffer.\n"); - if (http_rawhost_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http rawhost buffer.\n"); - if (http_headernames_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http header names buffer.\n"); - if (http_referer_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http header referer buffer.\n"); + for(size_t i = 0; i < ARRAY_SIZE(analyzer_items); i++) { + DetectEngineAnalyzerItems *ai = &analyzer_items[i]; + if (ai->item_seen) { + fprintf(rule_engine_analysis_FD, " Rule matches on %s buffer.\n", ai->display_name); + } + } if (s->alproto != ALPROTO_UNKNOWN) { fprintf(rule_engine_analysis_FD, " App layer protocol is %s.\n", AppProtoToString(s->alproto)); } -- 2.47.2