From: Russ Combs (rucombs) Date: Tue, 23 Jun 2020 00:20:19 +0000 (+0000) Subject: Merge pull request #2250 in SNORT/snort3 from ~RUCOMBS/snort3:aimless to master X-Git-Tag: 3.0.2-1~24 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e520d7479b0c7ab98089599bf7ae5810c00a300d;p=thirdparty%2Fsnort3.git Merge pull request #2250 in SNORT/snort3 from ~RUCOMBS/snort3:aimless to master Squashed commit of the following: commit a88d8e5e36e9c9bdb18261b3fb9994f018d8b76e Author: russ Date: Thu Jun 18 12:50:04 2020 -0400 max_detect: detained inspection disabled pending further work commit e0a6c905a965853d6739177c528d2c3cfd317ff2 Author: russ Date: Wed Jun 17 01:32:16 2020 -0400 snort: fix --dump-rule-meta with ips.states commit df44b9f9fdcc708d23b99e4dd6d4bd250ee73bc5 Author: russ Date: Mon Jun 8 10:45:04 2020 -0400 detection: remove unused code commit 947fb40131cf9b671bd63c9202dca0eac013bd1d Author: russ Date: Sun Jun 7 11:53:19 2020 -0400 regex: convert to same syntax as pcre plus fast_pattern option commit 74fb07f83cb7eca507a9b0708078f1ab0e8f8c21 Author: russ Date: Sat Jun 6 21:10:41 2020 -0400 mpse: remove unused pattern trimming support commit b3c00fbe13508a83763d06dfe583c76ab6af0763 Author: russ Date: Sat Jun 6 00:40:59 2020 -0400 ips: update detection trees for earliest header checks commit 07816e253eeba09240df1abc80386303ddbcc691 Author: russ Date: Fri Jun 5 03:09:28 2020 -0400 ips: refactor fast pattern selection. Enable content, regex, and sd_pattern options to be deduplicated. commit 4e671b312dc3d168b48a48a7c8709eaf5cb125c5 Author: russ Date: Mon Jun 1 08:53:01 2020 -0400 tweaks: updates for efficacy and performance commit 5fc59bd0061a52750b57ff6cdf9e9d23b1da10f4 Author: russ Date: Sat May 30 10:43:41 2020 -0400 appid: use configured search method for multi-pattern matching commit 838255f2b79c8504a96f0f2d2000c83088b024ab Author: russ Date: Wed Jun 3 01:04:03 2020 -0400 ips: add http fast pattern buffers These additional fast pattern buffers are supported: http_raw_uri http_raw_header http_stat_code http_stat_msg http_cookie http_method The current implementation handles them somewhat generically, so other inspectors can provide a method or a stat_code buffer too. A future iteration will make these buffers extensible. commit e8b52034c1735e2fa95911967753eec47f6ded26 Author: russ Date: Sat May 30 19:34:58 2020 -0400 ips: add ips service vs buffer checks; add missing services commit 822d67423914d137399d20a6fc7a462eb138c491 Author: russ Date: Fri May 29 12:20:06 2020 -0400 ips: minimize port group construction for any-any and bidirectional rules commit e719dad994e1e1f65601bf439ef61dae5f904d66 Author: russ Date: Tue May 26 13:07:58 2020 -0400 ips: enable non-service rules when service is detected Do fast pattern searches for port groups after service groups. Also, search_engine.detect_raw_tcp is applied to rules w/ or w/o a fast pattern (previously, erroneously, only fast-pattern rules). In addition, this no longer applies to flows w/o a service inspector. Such flows act as if detect_raw_tcp is true regardless of setting. commit f11be51de012d6b6f290484329675c5bc5a7d077 Author: russ Date: Tue May 19 22:09:48 2020 -0400 snort_defaults.lua: remove unused AIM_SERVERS var --- diff --git a/lua/balanced.lua b/lua/balanced.lua index 97ce4aadd..0495ba5d3 100644 --- a/lua/balanced.lua +++ b/lua/balanced.lua @@ -5,6 +5,8 @@ arp_spoof = nil +detection = { pcre_override = false } + http_inspect.request_depth = 300 http_inspect.response_depth = 500 diff --git a/lua/max_detect.lua b/lua/max_detect.lua index 02cc4cdaa..2510f6910 100644 --- a/lua/max_detect.lua +++ b/lua/max_detect.lua @@ -11,10 +11,14 @@ ftp_server.check_encrypted = true detection = { pcre_match_limit = 3500, - pcre_match_limit_recursion = 3500 + pcre_match_limit_recursion = 3500, + + -- enable for hyperscan for best throughput + -- use multiple packet threads for fast startup + --hyperscan_literals = true, + --pcre_to_regex = true } -http_inspect.detained_inspection = true http_inspect.decompress_pdf = true http_inspect.decompress_swf = true http_inspect.decompress_zip = true @@ -40,8 +44,6 @@ smtp.decompress_zip = true stream_ip.min_frag_length = 100 -stream_tcp.require_3whs = 0 - stream_tcp.small_segments = { count = 3, diff --git a/lua/security.lua b/lua/security.lua index 163b71ab5..c2613d94f 100644 --- a/lua/security.lua +++ b/lua/security.lua @@ -33,8 +33,6 @@ smtp.decompress_zip = true stream_ip.min_frag_length = 100 -stream_tcp.require_3whs = 180 - stream_tcp.small_segments = { count = 3, diff --git a/lua/snort_defaults.lua b/lua/snort_defaults.lua index f68990ea2..9aa365bc8 100644 --- a/lua/snort_defaults.lua +++ b/lua/snort_defaults.lua @@ -58,23 +58,6 @@ SSH_SERVERS = HOME_NET -- List of telnet servers on your network TELNET_SERVERS = HOME_NET --- other variables, these should not be modified -AIM_SERVERS = -[[ -64.12.24.0/23 -64.12.28.0/23 -64.12.161.0/24 -64.12.163.0/24 -64.12.200.0/24 -205.188.3.0/24 -205.188.5.0/24 -205.188.7.0/24 -205.188.9.0/24 -205.188.153.0/24 -205.188.179.0/24 -205.188.248.0/24 -]] - --------------------------------------------------------------------------- -- default ports - used in Talos rules --------------------------------------------------------------------------- diff --git a/src/detection/detection_options.cc b/src/detection/detection_options.cc index 4d76db919..b9e311c7f 100644 --- a/src/detection/detection_options.cc +++ b/src/detection/detection_options.cc @@ -411,18 +411,14 @@ int detection_option_node_evaluate( // No, haven't evaluated this one before... Check it. do { - switch ( node->option_type ) - { - case RULE_OPTION_TYPE_LEAF_NODE: - // Add the match for this otn to the queue. + if ( node->otn ) { - OptTreeNode* otn = (OptTreeNode*)node->option_data; SnortProtocolId snort_protocol_id = p->get_snort_protocol_id(); int check_ports = 1; if ( snort_protocol_id != UNKNOWN_PROTOCOL_ID ) { - const auto& sig_info = otn->sigInfo; + const auto& sig_info = node->otn->sigInfo; for ( const auto& svc : sig_info.services ) { @@ -442,16 +438,15 @@ int detection_option_node_evaluate( } } - bool eval_rtn_result; - - // Don't include RTN time - { - RulePause pause(profile); - eval_rtn_result = fp_eval_rtn(getRuntimeRtnFromOtn(otn), p, check_ports); - } + if ( !fp_eval_rtn(getRuntimeRtnFromOtn(node->otn), p, check_ports) ) + break; + } - if ( eval_rtn_result ) + switch ( node->option_type ) + { + case RULE_OPTION_TYPE_LEAF_NODE: { + OptTreeNode* otn = (OptTreeNode*)node->option_data; bool f_result = true; if ( otn->detection_filter ) @@ -463,7 +458,11 @@ int detection_option_node_evaluate( p->pkth->ts.tv_sec); } - if ( f_result ) + if ( !f_result ) + { + debug_log(detection_trace, TRACE_RULE_EVAL, p, "Header check failed\n"); + } + else { otn->state[get_instance_id()].matches++; @@ -479,14 +478,7 @@ int detection_option_node_evaluate( result = rval = (int)IpsOption::MATCH; } } -#ifdef DEBUG_MSGS - else - debug_log(detection_trace, TRACE_RULE_EVAL, p, - "Header check failed\n"); -#endif - break; - } case RULE_OPTION_TYPE_CONTENT: if ( node->evaluate ) @@ -585,7 +577,6 @@ int detection_option_node_evaluate( } { - RulePause pause(profile); // Passed, check the children. if ( node->num_children ) { diff --git a/src/detection/detection_options.h b/src/detection/detection_options.h index 8cfca7ac6..bcf135fe4 100644 --- a/src/detection/detection_options.h +++ b/src/detection/detection_options.h @@ -89,13 +89,14 @@ struct dot_node_state_t struct detection_option_tree_node_t { eval_func_t evaluate; + detection_option_tree_node_t** children; + void* option_data; + dot_node_state_t* state; + struct OptTreeNode* otn; int is_relative; int num_children; int relative_children; - void* option_data; option_type_t option_type; - detection_option_tree_node_t** children; - dot_node_state_t* state; }; struct detection_option_tree_root_t diff --git a/src/detection/fp_config.cc b/src/detection/fp_config.cc index 40416c2ef..ebeabcc75 100644 --- a/src/detection/fp_config.cc +++ b/src/detection/fp_config.cc @@ -42,7 +42,6 @@ FastPatternConfig::FastPatternConfig() { search_api = MpseManager::get_search_api("ac_bnfa"); assert(search_api); - trim = MpseManager::search_engine_trim(search_api); } @@ -54,7 +53,6 @@ bool FastPatternConfig::set_search_method(const char* method) return false; search_api = api; - trim = MpseManager::search_engine_trim(search_api); return true; } @@ -77,14 +75,6 @@ bool FastPatternConfig::set_offload_search_method(const char* method) return true; } -const char* FastPatternConfig::get_offload_search_method() -{ - if ( !offload_search_api ) - return nullptr; - - return offload_search_api->base.name; -} - void FastPatternConfig::set_max_pattern_len(unsigned int max_len) { if (max_pattern_len != 0) diff --git a/src/detection/fp_config.h b/src/detection/fp_config.h index 80d0934f0..1d26db5a8 100644 --- a/src/detection/fp_config.h +++ b/src/detection/fp_config.h @@ -126,10 +126,7 @@ public: const char* get_search_method(); bool set_offload_search_method(const char*); - const char* get_offload_search_method(); - void set_max_pattern_len(unsigned); - void set_queue_limit(unsigned); unsigned get_queue_limit() const @@ -141,15 +138,6 @@ public: const snort::MpseApi* get_offload_search_api() const { return offload_search_api; } - bool get_trim() const - { return trim; } - - void trimmed() - { num_patterns_trimmed++; } - - int get_num_patterns_trimmed() const - { return num_patterns_trimmed; } - int get_num_patterns_truncated() const { return num_patterns_truncated; } @@ -160,7 +148,6 @@ private: const snort::MpseApi* offload_search_api = nullptr; bool inspect_stream_insert = true; - bool trim; bool split_any_any = false; bool debug_print_fast_pattern = false; bool debug = false; @@ -174,7 +161,6 @@ private: int portlists_flags = 0; int num_patterns_truncated = 0; // due to max_pattern_len - int num_patterns_trimmed = 0; // due to zero byte prefix }; #endif diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc index cca0034a1..cbc8405a2 100644 --- a/src/detection/fp_create.cc +++ b/src/detection/fp_create.cc @@ -37,6 +37,7 @@ #include "framework/mpse.h" #include "framework/mpse_batch.h" #include "hash/ghash.h" +#include "hash/hash_defs.h" #include "hash/xhash.h" #include "log/messages.h" #include "main/snort.h" @@ -68,8 +69,8 @@ static const char* s_group = ""; static void fpDeletePMX(void* data); -static int fpGetFinalPattern(FastPatternConfig*, PatternMatchData*, const char*& ret_pattern, - unsigned& ret_bytes, Mpse::MpseType mpse_type); +static int fpGetFinalPattern( + FastPatternConfig*, PatternMatchData*, const char*& ret_pattern, unsigned& ret_bytes); static void print_nfp_info(const char*, OptTreeNode*); static void print_fp_info(const char*, const OptTreeNode*, const PatternMatchData*, @@ -96,6 +97,51 @@ static int finalize_detection_option_tree(SnortConfig* sc, detection_option_tree return 0; } +static OptTreeNode* fixup_tree( + detection_option_tree_node_t* dot, bool branched, unsigned contents) +{ + if ( dot->num_children == 0 ) + { + if ( !branched and contents ) + return (OptTreeNode*)dot->option_data; + + dot->otn = (OptTreeNode*)dot->option_data; + return nullptr; + } + if ( dot->num_children == 1 ) + { + if ( dot->option_type == RULE_OPTION_TYPE_CONTENT ) + ++contents; + + OptTreeNode* otn = fixup_tree(dot->children[0], false, contents); + + if ( !branched and contents > 1 ) + return otn; + + dot->otn = otn; + return nullptr; + } + for ( int i = 0; i < dot->num_children; ++i ) + fixup_tree(dot->children[i], true, 0); + + return nullptr; +} + +static void fixup_trees(SnortConfig* sc) +{ + if ( !sc->detection_option_tree_hash_table ) + return; + + HashNode* hn = sc->detection_option_tree_hash_table->find_first_node(); + + while ( hn ) + { + detection_option_tree_node_t* node = (detection_option_tree_node_t*)hn->data; + fixup_tree(node, true, 0); + hn = sc->detection_option_tree_hash_table->find_next_node(); + } +} + static bool new_sig(int num_children, detection_option_tree_node_t** nodes, OptTreeNode* otn) { for ( int i = 0; i < num_children; ++i ) @@ -155,7 +201,7 @@ static int otn_create_tree(OptTreeNode* otn, void** existing_tree, Mpse::MpseTyp /* Don't add contents that are only for use in the * fast pattern matcher */ - if ( is_fast_pattern_only(opt_fp, mpse_type) ) + if ( is_fast_pattern_only(otn, opt_fp, mpse_type) ) { opt_fp = opt_fp->next; continue; @@ -382,15 +428,14 @@ static int pmx_create_tree_offload(SnortConfig* sc, void* id, void** existing_tr } static int fpFinishPortGroupRule( - Mpse* mpse, OptTreeNode* otn, PatternMatchData* pmd, - FastPatternConfig* fp, Mpse::MpseType mpse_type, bool get_final_pat) + Mpse* mpse, OptTreeNode* otn, PatternMatchData* pmd, FastPatternConfig* fp, bool get_final_pat) { const char* pattern; unsigned pattern_length; if (get_final_pat) { - if (fpGetFinalPattern(fp, pmd, pattern, pattern_length, mpse_type) == -1) + if (fpGetFinalPattern(fp, pmd, pattern, pattern_length) == -1) return -1; } else @@ -502,10 +547,9 @@ static int fpFinishPortGroup(SnortConfig* sc, PortGroup* pg, FastPatternConfig* } static void fpAddAlternatePatterns( - Mpse* mpse, OptTreeNode* otn, PatternMatchData* pmd, FastPatternConfig* fp, - Mpse::MpseType mpse_type) + Mpse* mpse, OptTreeNode* otn, PatternMatchData* pmd, FastPatternConfig* fp) { - fpFinishPortGroupRule(mpse, otn, pmd, fp, mpse_type, false); + fpFinishPortGroupRule(mpse, otn, pmd, fp, false); } static int fpAddPortGroupRule( @@ -513,7 +557,7 @@ static int fpAddPortGroupRule( { const MpseApi* search_api = nullptr; const MpseApi* offload_search_api = nullptr; - OptFpList* next = nullptr; + OptFpList* ofp = nullptr; bool exclude; // skip builtin rules, continue for text and so rules @@ -527,12 +571,12 @@ static int fpAddPortGroupRule( assert(search_api); bool only_literal = !MpseManager::is_regex_capable(search_api); - PatternMatchVector pmv = get_fp_content(otn, next, srvc, only_literal, exclude); + PatternMatchVector pmv = get_fp_content(otn, ofp, srvc, only_literal, exclude); if ( !pmv.empty() ) { PatternMatchVector pmv_ol; - OptFpList* next_ol = nullptr; + OptFpList* ofp_ol = nullptr; bool add_to_offload = false; bool cont = true; PatternMatchData* ol_pmd = nullptr; @@ -545,7 +589,7 @@ static int fpAddPortGroupRule( { bool exclude_ol; bool only_literal_ol = !MpseManager::is_regex_capable(offload_search_api); - pmv_ol = get_fp_content(otn, next_ol, srvc, only_literal_ol, exclude_ol); + pmv_ol = get_fp_content(otn, ofp_ol, srvc, only_literal_ol, exclude_ol); // If we can get a fast_pattern for the normal search engine but not for the // offload search engine then add rule to the non fast pattern list @@ -561,36 +605,20 @@ static int fpAddPortGroupRule( PatternMatchData* main_pmd = pmv.back(); pmv.pop_back(); - if ( !main_pmd->is_relative() && !main_pmd->is_negated() && main_pmd->fp_only >= 0 && - // FIXIT-L no_case consideration is mpse specific, delegate - !main_pmd->offset && !main_pmd->depth && main_pmd->is_no_case() ) - { - if ( !next || !next->ips_opt || !next->ips_opt->is_relative() ) - main_pmd->fp_only |= (1 << Mpse::MPSE_TYPE_NORMAL); - } - static MpseAgent agent = { pmx_create_tree_normal, add_patrn_to_neg_list, fpDeletePMX, free_detection_option_root, neg_list_free }; - if ( pg->mpsegrp[main_pmd->pm_type] == nullptr ) - { + if ( !pg->mpsegrp[main_pmd->pm_type] ) pg->mpsegrp[main_pmd->pm_type] = new MpseGroup; - if ( pg->mpsegrp[main_pmd->pm_type] == nullptr ) - { - ParseError("Failed to create pattern matcher for %d", main_pmd->pm_type); - return -1; - } - } - if (pg->mpsegrp[main_pmd->pm_type]->normal_mpse == nullptr) + if ( !pg->mpsegrp[main_pmd->pm_type]->normal_mpse ) { if (!pg->mpsegrp[main_pmd->pm_type]->create_normal_mpse(sc, &agent)) { - ParseError("Failed to create normal pattern matcher for %d", - main_pmd->pm_type); + ParseError("Failed to create normal pattern matcher for %d", main_pmd->pm_type); return -1; } @@ -604,14 +632,6 @@ static int fpAddPortGroupRule( ol_pmd = pmv_ol.back(); pmv_ol.pop_back(); - if ( !ol_pmd->is_relative() && !ol_pmd->is_negated() && ol_pmd->fp_only >= 0 && - // FIXIT-L no_case consideration is mpse specific, delegate - !ol_pmd->offset && !ol_pmd->depth && ol_pmd->is_no_case() ) - { - if ( !next_ol || !next_ol->ips_opt || !next_ol->ips_opt->is_relative() ) - ol_pmd->fp_only |= (1 << Mpse::MPSE_TYPE_OFFLOAD); - } - static MpseAgent agent_offload = { pmx_create_tree_offload, add_patrn_to_neg_list, @@ -619,7 +639,7 @@ static int fpAddPortGroupRule( }; // Keep the created mpse alongside the same pm type as the main pmd - if (pg->mpsegrp[main_pmd->pm_type]->offload_mpse == nullptr) + if ( !pg->mpsegrp[main_pmd->pm_type]->offload_mpse ) { if (!pg->mpsegrp[main_pmd->pm_type]->create_offload_mpse(sc, &agent_offload)) { @@ -644,16 +664,19 @@ static int fpAddPortGroupRule( add_nfp_rule = true; // Now add patterns - if (fpFinishPortGroupRule(pg->mpsegrp[main_pmd->pm_type]->normal_mpse, - otn, main_pmd, fp, Mpse::MPSE_TYPE_NORMAL, true) == 0) + if (fpFinishPortGroupRule( + pg->mpsegrp[main_pmd->pm_type]->normal_mpse, otn, main_pmd, fp, true) == 0) { if (main_pmd->pattern_size > otn->longestPatternLen) otn->longestPatternLen = main_pmd->pattern_size; + if ( make_fast_pattern_only(ofp, main_pmd) ) + otn->normal_fp_only = ofp; + // Add Alternative patterns for (auto p : pmv) - fpAddAlternatePatterns(pg->mpsegrp[main_pmd->pm_type]->normal_mpse, - otn, p, fp, Mpse::MPSE_TYPE_NORMAL); + fpAddAlternatePatterns( + pg->mpsegrp[main_pmd->pm_type]->normal_mpse, otn, p, fp); } } @@ -664,16 +687,19 @@ static int fpAddPortGroupRule( add_nfp_rule = true; // Now add patterns - if (fpFinishPortGroupRule(pg->mpsegrp[main_pmd->pm_type]->offload_mpse, - otn, ol_pmd, fp, Mpse::MPSE_TYPE_OFFLOAD, true) == 0) + if (fpFinishPortGroupRule( + pg->mpsegrp[main_pmd->pm_type]->offload_mpse, otn, ol_pmd, fp, true) == 0) { if (ol_pmd->pattern_size > otn->longestPatternLen) otn->longestPatternLen = ol_pmd->pattern_size; + if ( make_fast_pattern_only(ofp_ol, ol_pmd) ) + otn->offload_fp_only = ofp_ol; + // Add Alternative patterns for (auto p : pmv_ol) - fpAddAlternatePatterns(pg->mpsegrp[main_pmd->pm_type]->offload_mpse, - otn, p, fp, Mpse::MPSE_TYPE_OFFLOAD); + fpAddAlternatePatterns( + pg->mpsegrp[main_pmd->pm_type]->offload_mpse, otn, p, fp); } } @@ -891,13 +917,9 @@ static void fpFreeRuleMaps(SnortConfig* sc) } static int fpGetFinalPattern( - FastPatternConfig* fp, PatternMatchData* pmd, - const char*& ret_pattern, unsigned& ret_bytes, Mpse::MpseType mpse_type) + FastPatternConfig* fp, PatternMatchData* pmd, const char*& ret_pattern, unsigned& ret_bytes) { - if ( !fp or !pmd ) - { - return -1; - } + assert(fp and pmd); const char* pattern = pmd->pattern_buf; unsigned bytes = pmd->pattern_size; @@ -916,9 +938,7 @@ static int fpGetFinalPattern( // 3. non-literals like regex - truncation could invalidate the // expression. - assert((mpse_type == Mpse::MPSE_TYPE_NORMAL) or (mpse_type == Mpse::MPSE_TYPE_OFFLOAD)); - - if ( (pmd->fp_only & (1 << mpse_type)) or pmd->is_negated() or !pmd->is_literal() ) + if ( pmd->is_negated() or !pmd->is_literal() ) { ret_pattern = pattern; ret_bytes = bytes; @@ -933,36 +953,6 @@ static int fpGetFinalPattern( pattern = pmd->pattern_buf + pmd->fp_offset; bytes = pmd->fp_length ? pmd->fp_length : pmd->pattern_size - pmd->fp_length; } - else - { - /* Trim leading null bytes for non-deterministic pattern matchers. - * Assuming many packets may have strings of 0x00 bytes in them, - * this should help performance with non-deterministic pattern matchers - * that have a full next state vector at state 0. If no patterns are - * inserted into the state machine that start with 0x00, failstates that - * land us at state 0 will allow us to roll through the 0x00 bytes, - * since the next state is deterministic in state 0 and we won't move - * beyond state 0 as long as the next input char is 0x00 */ - if ( fp->get_trim() ) - { - bytes = flp_trim(pmd->pattern_buf, pmd->pattern_size, &pattern); - - if (bytes < pmd->pattern_size) - { - // The pattern is all '\0' - use the whole pattern. This potentially - // hurts the performance boost gained by stripping leading zeros. - if (bytes == 0) - { - bytes = pmd->pattern_size; - pattern = pmd->pattern_buf; - } - else - { - fp->trimmed(); - } - } - } - } ret_pattern = pattern; ret_bytes = fp->set_max(bytes); @@ -1303,10 +1293,7 @@ static void fpBuildServicePortGroups( assert(snort_protocol_id != UNKNOWN_PROTOCOL_ID); assert((unsigned)snort_protocol_id < sopg.size()); - if(snort_protocol_id == UNKNOWN_PROTOCOL_ID) - continue; - - sopg[ snort_protocol_id ] = pg; + sopg[snort_protocol_id] = pg; } } @@ -1633,6 +1620,8 @@ int fpCreateFastPacketDetection(SnortConfig* sc) if ( c != expected ) ParseError("Failed to compile %u search engines", expected - c); + + fixup_trees(sc); } fp_print_port_groups(port_tables); @@ -1653,9 +1642,6 @@ int fpCreateFastPacketDetection(SnortConfig* sc) if ( fp->get_num_patterns_truncated() ) LogMessage("%25.25s: %-12u\n", "truncated patterns", fp->get_num_patterns_truncated()); - if ( fp->get_num_patterns_trimmed() ) - LogMessage("%25.25s: %-12u\n", "prefix trims", fp->get_num_patterns_trimmed()); - MpseManager::setup_search_engine(fp->get_search_api(), sc); return 0; @@ -1704,8 +1690,6 @@ void get_pattern_info(const PatternMatchData* pmd, opts = "("; if ( pmd->is_fast_pattern() ) opts += " user"; - if ( pmd->fp_only > 0 ) - opts += " only"; if ( pmd->is_negated() ) opts += " negated"; opts += " )"; diff --git a/src/detection/fp_detect.cc b/src/detection/fp_detect.cc index 634047b19..1bbcd8886 100644 --- a/src/detection/fp_detect.cc +++ b/src/detection/fp_detect.cc @@ -874,14 +874,13 @@ static inline int batch_search( return 0; } -static inline int search_buffer( +static inline void search_buffer( Inspector* gadget, InspectionBuffer& buf, InspectionBuffer::Type ibt, Packet* p, PortGroup* pg, PmType pmt, PegCount& cnt) { - if ( gadget->get_fp_buf(ibt, p, buf) ) + if ( MpseGroup* so = pg->mpsegrp[pmt] ) { - // Depending on where we are searching we call the appropriate mpse - if ( MpseGroup* so = pg->mpsegrp[pmt] ) + if ( gadget->get_fp_buf(ibt, p, buf) ) { debug_logf(detection_trace, TRACE_FP_SEARCH, p, "%" PRIu64 " fp %s.%s[%d]\n", p->context->packet_number, @@ -890,19 +889,20 @@ static inline int search_buffer( batch_search(so, p, buf.data, buf.len, cnt); } } - return 0; } -static int fp_search(PortGroup* port_group, Packet* p) +static int fp_search(PortGroup* port_group, Packet* p, bool srvc) { Inspector* gadget = p->flow ? p->flow->gadget : nullptr; InspectionBuffer buf; debug_log(detection_trace, TRACE_RULE_EVAL, p, "Fast pattern search\n"); - if ( p->data and p->dsize ) + // ports search raw packet only + if ( p->dsize ) { - // ports search raw packet only + assert(p->data); + if ( MpseGroup* so = port_group->mpsegrp[PM_TYPE_PKT] ) { if ( uint16_t pattern_match_size = p->get_detect_limit() ) @@ -919,23 +919,42 @@ static int fp_search(PortGroup* port_group, Packet* p) if ( gadget ) { - // service searches PDU buffers and file - if ( search_buffer(gadget, buf, buf.IBT_KEY, p, port_group, PM_TYPE_KEY, pc.key_searches) ) - return 1; - - if ( search_buffer(gadget, buf, buf.IBT_HEADER, p, port_group, PM_TYPE_HEADER, pc.header_searches) ) - return 1; - - if ( search_buffer(gadget, buf, buf.IBT_BODY, p, port_group, PM_TYPE_BODY, pc.body_searches) ) - return 1; - // FIXIT-L PM_TYPE_ALT will never be set unless we add // norm_data keyword or telnet, rpc_decode, smtp keywords // until then we must use the standard packet mpse - if ( search_buffer(gadget, buf, buf.IBT_ALT, p, port_group, PM_TYPE_PKT, pc.alt_searches) ) - return 1; + search_buffer(gadget, buf, buf.IBT_ALT, p, port_group, PM_TYPE_PKT, pc.alt_searches); } + if ( !srvc ) + return 0; + + // service searches PDU buffers and file + if ( gadget ) + { + search_buffer(gadget, buf, buf.IBT_KEY, p, port_group, PM_TYPE_KEY, pc.key_searches); + + search_buffer(gadget, buf, buf.IBT_HEADER, p, port_group, PM_TYPE_HEADER, pc.header_searches); + + search_buffer(gadget, buf, buf.IBT_BODY, p, port_group, PM_TYPE_BODY, pc.body_searches); + + search_buffer( + gadget, buf, buf.IBT_RAW_KEY, p, port_group, PM_TYPE_RAW_KEY, pc.raw_key_searches); + + search_buffer( + gadget, buf, buf.IBT_RAW_HEADER, p, port_group, PM_TYPE_RAW_HEADER, pc.raw_header_searches); + + search_buffer( + gadget, buf, buf.IBT_METHOD, p, port_group, PM_TYPE_METHOD, pc.method_searches); + + search_buffer( + gadget, buf, buf.IBT_STAT_CODE, p, port_group, PM_TYPE_STAT_CODE, pc.stat_code_searches); + + search_buffer( + gadget, buf, buf.IBT_STAT_MSG, p, port_group, PM_TYPE_STAT_MSG, pc.stat_msg_searches); + + search_buffer( + gadget, buf, buf.IBT_COOKIE, p, port_group, PM_TYPE_COOKIE, pc.cookie_searches); + } { // file searches file only if ( MpseGroup* so = port_group->mpsegrp[PM_TYPE_FILE] ) @@ -958,7 +977,7 @@ static int fp_search(PortGroup* port_group, Packet* p) } static inline void eval_fp( - PortGroup* port_group, Packet* p, char ip_rule) + PortGroup* port_group, Packet* p, char ip_rule, bool srvc) { const uint8_t* tmp_payload = nullptr; uint16_t tmp_dsize = 0; @@ -983,11 +1002,8 @@ static inline void eval_fp( if ( DetectionEngine::content_enabled(p) ) { - FastPatternConfig* fp = p->context->conf->fast_pattern_config; - - if ( fp->get_stream_insert() || !(p->packet_flags & PKT_STREAM_INSERT) ) - if ( fp_search(port_group, p) ) - return; + if ( fp_search(port_group, p, srvc) ) + return; } if ( ip_rule ) { @@ -1078,18 +1094,17 @@ static inline void eval_nfp( // for non-content. The otn list search will eventually be redone for // for performance purposes. -static inline int fpEvalHeaderSW(PortGroup* port_group, Packet* p, char ip_rule, FPTask task) +static inline void fpEvalHeaderSW( + PortGroup* port_group, Packet* p, char ip_rule, FPTask task, bool srvc = false) { if ( !p->is_detection_enabled(p->packet_flags & PKT_FROM_CLIENT) ) - return 0; + return; if ( task & FPTask::FP ) - eval_fp(port_group, p, ip_rule); + eval_fp(port_group, p, ip_rule, srvc); if ( task & FPTask::NON_FP ) eval_nfp(port_group, p, ip_rule); - - return 0; } static inline void fpEvalHeaderIp(Packet* p, FPTask task) @@ -1125,6 +1140,7 @@ static inline void fpEvalHeaderIcmp(Packet* p, FPTask task) static inline void fpEvalHeaderTcp(Packet* p, FPTask task) { + PortGroup* src = nullptr, * dst = nullptr, * any = nullptr; if ( !prmFindRuleGroupTcp(p->context->conf->prmTcpRTNX, p->ptrs.dp, p->ptrs.sp, &src, &dst, &any) ) @@ -1157,25 +1173,23 @@ static inline void fpEvalHeaderUdp(Packet* p, FPTask task) fpEvalHeaderSW(any, p, 0, task); } -static inline bool fpEvalHeaderSvc(Packet* p, FPTask task) +static inline void fpEvalHeaderSvc(Packet* p, FPTask task) { - PortGroup* svc = nullptr; - SnortProtocolId snort_protocol_id = p->get_snort_protocol_id(); - if (snort_protocol_id != UNKNOWN_PROTOCOL_ID and snort_protocol_id != INVALID_PROTOCOL_ID) - { - if (p->is_from_server()) /* to cli */ - svc = p->context->conf->sopgTable->get_port_group(false, snort_protocol_id); + if (snort_protocol_id == UNKNOWN_PROTOCOL_ID or snort_protocol_id == INVALID_PROTOCOL_ID) + return; - if (p->is_from_client()) /* to srv */ - svc = p->context->conf->sopgTable->get_port_group(true, snort_protocol_id); - } + PortGroup* svc = nullptr; - if ( svc ) - fpEvalHeaderSW(svc, p, 0, task); + if (p->is_from_server()) + svc = p->context->conf->sopgTable->get_port_group(false, snort_protocol_id); - return svc != nullptr; + else if (p->is_from_client()) + svc = p->context->conf->sopgTable->get_port_group(true, snort_protocol_id); + + if ( svc ) + fpEvalHeaderSW(svc, p, 0, task, true); } static void fpEvalPacketUdp(Packet* p, FPTask task) @@ -1220,28 +1234,45 @@ static void fpEvalPacketUdp(Packet* p, FPTask task) DetectionEngine::set_detects(p, save_detect); } -/* -** the IP protocol is processed. If it is TCP, UDP, or ICMP, we -** process the both that particular ruleset and the IP ruleset -** with in the fpEvalHeader for that protocol. If the protocol -** is not TCP, UDP, or ICMP, we just process the packet against -** the IP rules at the end of the fpEvalPacket routine. Since -** we are using a setwise methodology for snort rules, both the -** network layer rules and the transport layer rules are done -** at the same time. While this is not the best for modularity, -** it is the best for performance, which is what we are working -** on currently. -*/ +static inline bool skip_raw_tcp(const Packet* p) +{ + if ( !(p->packet_flags & PKT_STREAM_INSERT) ) + return false; + + if ( !p->flow or !p->flow->gadget ) + return false; + + if ( !p->context->conf->fast_pattern_config->get_stream_insert() ) + return true; + + return false; +} + +// the IP protocol is processed. If it is TCP, UDP, or ICMP, we +// process the both that particular ruleset and the IP ruleset +// with in the fpEvalHeader for that protocol. If the protocol +// is not TCP, UDP, or ICMP, we just process the packet against +// the IP rules at the end of the fpEvalPacket routine. Since +// we are using a setwise methodology for snort rules, both the +// network layer rules and the transport layer rules are done +// at the same time. While this is not the best for modularity, +// it is the best for performance, which is what we are working +// on currently. + static void fpEvalPacket(Packet* p, FPTask task) { - /* Run UDP rules against the UDP header of Teredo packets */ + // Run UDP rules against the UDP header of Teredo packets // FIXIT-L udph is always inner; need to check for outer + // FIXIT-M UDP tunnel not searching service groups + if ( p->is_udp_tunneled() ) fpEvalPacketUdp(p, task); - if ( p->get_snort_protocol_id() != UNKNOWN_PROTOCOL_ID and fpEvalHeaderSvc(p, task) ) + if ( skip_raw_tcp(p) ) return; + fpEvalHeaderSvc(p, task); + switch (p->type()) { case PktType::IP: diff --git a/src/detection/fp_utils.cc b/src/detection/fp_utils.cc index 8945147c6..2ac646fb3 100644 --- a/src/detection/fp_utils.cc +++ b/src/detection/fp_utils.cc @@ -28,10 +28,9 @@ #include #include -#include "ips_options/ips_flow.h" #include "log/messages.h" #include "main/snort_config.h" -#include "main/thread_config.h" +#include "parser/parse_conf.h" #include "pattern_match_data.h" #include "ports/port_group.h" #include "target_based/snort_protocols.h" @@ -48,31 +47,18 @@ using namespace snort; // private utilities //-------------------------------------------------------------------------- -static void finalize_content(OptFpList* ofl) -{ - PatternMatchData* pmd = get_pmd(ofl, UNKNOWN_PROTOCOL_ID, RULE_WO_DIR); - - if ( !pmd ) - return; - - if ( pmd->is_negated() ) - pmd->last_check = (PmdLastCheck*)snort_calloc( - ThreadConfig::get_instance_max(), sizeof(*pmd->last_check)); -} - -static void clear_fast_pattern_only(OptFpList* ofl) -{ - PatternMatchData* pmd = get_pmd(ofl, UNKNOWN_PROTOCOL_ID, RULE_WO_DIR); - - if ( pmd && pmd->fp_only > 0 ) - pmd->fp_only = 0; -} - static bool pmd_can_be_fp( PatternMatchData* pmd, CursorActionType cat, bool only_literals) { - if ( cat <= CAT_SET_OTHER ) + switch ( cat ) + { + case CAT_NONE: + case CAT_ADJUST: + case CAT_SET_OTHER: return false; + default: + break; + } if ( only_literals and !pmd->is_literal() ) return false; @@ -80,7 +66,7 @@ static bool pmd_can_be_fp( return pmd->can_be_fp(); } -static PmType get_pm_type(CursorActionType cat) +PmType get_pm_type(CursorActionType cat) { switch ( cat ) { @@ -88,6 +74,27 @@ static PmType get_pm_type(CursorActionType cat) case CAT_SET_OTHER: return PM_TYPE_PKT; + case CAT_SET_COOKIE: + return PM_TYPE_COOKIE; + + case CAT_SET_STAT_MSG: + return PM_TYPE_STAT_MSG; + + case CAT_SET_STAT_CODE: + return PM_TYPE_STAT_CODE; + + case CAT_SET_METHOD: + return PM_TYPE_METHOD; + + case CAT_SET_RAW_HEADER: + return PM_TYPE_RAW_HEADER; + + case CAT_SET_RAW_KEY: + return PM_TYPE_RAW_KEY; + + case CAT_SET_FILE: + return PM_TYPE_FILE; + case CAT_SET_BODY: return PM_TYPE_BODY; @@ -97,9 +104,6 @@ static PmType get_pm_type(CursorActionType cat) case CAT_SET_KEY: return PM_TYPE_KEY; - case CAT_SET_FILE: - return PM_TYPE_FILE; - default: break; } @@ -109,123 +113,43 @@ static PmType get_pm_type(CursorActionType cat) static RuleDirection get_dir(OptTreeNode* otn) { - if ( OtnFlowFromServer(otn) ) + if ( otn->to_client() ) return RULE_FROM_SERVER; - if ( OtnFlowFromClient(otn) ) + if ( otn->to_server() ) return RULE_FROM_CLIENT; return RULE_WO_DIR; } -//-------------------------------------------------------------------------- -// public utilities -//-------------------------------------------------------------------------- - -PatternMatchData* get_pmd(OptFpList* ofl, SnortProtocolId snort_protocol_id, RuleDirection direction) +// this will be made extensible when fast patterns are extensible +static const char* get_service(const char* opt) { - if ( !ofl->ips_opt ) - return nullptr; + if ( !strncmp(opt, "http_", 5) ) + return "http"; - return ofl->ips_opt->get_pattern(snort_protocol_id, direction); -} + if ( !strncmp(opt, "cip_", 4) ) // NO FP BUF + return "cip"; -bool is_fast_pattern_only(OptFpList* ofl, Mpse::MpseType mpse_type) -{ - PatternMatchData* pmd = get_pmd(ofl, UNKNOWN_PROTOCOL_ID, RULE_WO_DIR); + if ( !strncmp(opt, "dce_", 4) ) + return "netbios-ssn"; - if ( !pmd ) - return false; + if ( !strncmp(opt, "dnp3_", 5) ) + return "dnp3"; - assert((mpse_type == Mpse::MPSE_TYPE_NORMAL) or (mpse_type == Mpse::MPSE_TYPE_OFFLOAD)); + if ( !strncmp(opt, "gtp_", 4) ) // NO FP BUF + return "gtp"; - return (pmd->fp_only & (1 << mpse_type)); -} + if ( !strncmp(opt, "modbus_", 7) ) + return "modbus"; -bool is_fast_pattern_only(OptFpList* ofl) -{ - PatternMatchData* pmd = get_pmd(ofl, UNKNOWN_PROTOCOL_ID, RULE_WO_DIR); + if ( !strncmp(opt, "s7commplus_", 11) ) + return "s7commplus"; - if ( !pmd ) - return false; + if ( !strncmp(opt, "sip_", 4) ) + return "sip"; - return pmd->fp_only > 0; -} - -/* - * Trim zero byte prefixes, this increases uniqueness - * will not alter regex since they can't contain bald \0 - * - * returns - * length - of trimmed pattern - * buff - ptr to new beginning of trimmed buffer - */ -unsigned flp_trim(const char* p, unsigned plen, const char** buff) -{ - unsigned i; - unsigned size = 0; - - if ( !p ) - return 0; - - for (i=0; iopt_func; fpl; fpl = fpl->next) - { - // a relative option is following a fast_pattern/only and - if ( relative_is_bad_mkay ) - { - if (fpl->isRelative) - { - assert(fp); - clear_fast_pattern_only(fp); - } - } - - // reset the check if one of these are present. - if ( fpl->ips_opt and !fpl->ips_opt->get_pattern(0)) - { - if ( fpl->ips_opt->get_cursor_type() > CAT_NONE ) - relative_is_bad_mkay = false; - } - // set/unset the check on content options. - else - { - if ( is_fast_pattern_only(fpl) ) - { - fp = fpl; - relative_is_bad_mkay = true; - } - else - relative_is_bad_mkay = false; - } - finalize_content(fpl); - } + return nullptr; } //-------------------------------------------------------------------------- @@ -235,25 +159,23 @@ void validate_fast_pattern(OptTreeNode* otn) struct FpSelector { - CursorActionType cat; - PatternMatchData* pmd; - unsigned size; - - FpSelector(CursorActionType, PatternMatchData*); + CursorActionType cat = CAT_NONE; + IpsOption* opt = nullptr; + PatternMatchData* pmd = nullptr; + unsigned size = 0; - FpSelector() - { cat = CAT_NONE; pmd = nullptr; size = 0; } + FpSelector() = default; + FpSelector(CursorActionType, IpsOption*, PatternMatchData*); bool is_better_than(FpSelector&, bool srvc, RuleDirection, bool only_literals = false); }; -FpSelector::FpSelector(CursorActionType c, PatternMatchData* p) +FpSelector::FpSelector(CursorActionType c, IpsOption* o, PatternMatchData* p) { cat = c; + opt = o; pmd = p; - - // FIXIT-M unconditional trim is bad mkay? see fpGetFinalPattern - size = flp_trim(pmd->pattern_buf, pmd->pattern_size, nullptr); + size = p->pattern_size; } bool FpSelector::is_better_than( @@ -262,13 +184,8 @@ bool FpSelector::is_better_than( if ( !pmd_can_be_fp(pmd, cat, only_literals) ) { if ( pmd->is_fast_pattern() ) - { ParseWarning(WARN_RULES, "content ineligible for fast_pattern matcher - ignored"); - // When we have a normal search engine we do not wish to invalidate the user - // indicated fast pattern as this may be a valid fast pattern for use in the offload - // search engine - // pmd->flags &= ~PatternMatchData::FAST_PAT; - } + return false; } @@ -281,7 +198,7 @@ bool FpSelector::is_better_than( { ParseWarning(WARN_RULES, "only one fast_pattern content per rule allowed - using first"); - pmd->flags &= ~PatternMatchData::FAST_PAT; + return false; } return true; @@ -305,13 +222,70 @@ bool FpSelector::is_better_than( // public methods //-------------------------------------------------------------------------- +void validate_services(SnortConfig* sc, OptTreeNode* otn) +{ + std::string svc; + bool file = false; + + for (OptFpList* ofl = otn->opt_func; ofl; ofl = ofl->next) + { + if ( !ofl->ips_opt ) + continue; + + CursorActionType cat = ofl->ips_opt->get_cursor_type(); + + if ( cat <= CAT_ADJUST ) + continue; + + const char* s = ofl->ips_opt->get_name(); + + // special case file_data because it could be any subset of file carving services + if ( !strcmp(s, "file_data") ) + { + file = true; + continue; + } + + s = get_service(s); + + if ( !s ) + continue; + + if ( !svc.empty() and svc != s ) + { + ParseWarning(WARN_RULES, "%u:%u:%u has mixed service buffers (%s and %s)", + otn->sigInfo.gid, otn->sigInfo.sid, otn->sigInfo.rev, svc.c_str(), s); + } + svc = s; + } + if ( otn->sigInfo.services.size() == 1 and !svc.empty() and otn->sigInfo.services[0].service != svc ) + { + ParseWarning(WARN_RULES, "%u:%u:%u has service:%s with %s buffer", + otn->sigInfo.gid, otn->sigInfo.sid, otn->sigInfo.rev, + otn->sigInfo.services[0].service.c_str(), svc.c_str()); + } + if ( otn->sigInfo.services.empty() and !svc.empty() ) + { + ParseWarning(WARN_RULES, "%u:%u:%u has no service with %s buffer", + otn->sigInfo.gid, otn->sigInfo.sid, otn->sigInfo.rev, svc.c_str()); + + add_service_to_otn(sc, otn, svc.c_str()); + } + if ( otn->sigInfo.services.empty() and file ) + { + ParseWarning(WARN_RULES, "%u:%u:%u has no service with file_data", + otn->sigInfo.gid, otn->sigInfo.sid, otn->sigInfo.rev); + + add_service_to_otn(sc, otn, "file"); + } +} + PatternMatchVector get_fp_content( - OptTreeNode* otn, OptFpList*& next, bool srvc, bool only_literals, bool& exclude) + OptTreeNode* otn, OptFpList*& node, bool srvc, bool only_literals, bool& exclude) { CursorActionType curr_cat = CAT_SET_RAW; FpSelector best; bool content = false; - bool fp_only = true; PatternMatchVector pmds; for (OptFpList* ofl = otn->opt_func; ofl; ofl = ofl->next) @@ -322,55 +296,65 @@ PatternMatchVector get_fp_content( CursorActionType cat = ofl->ips_opt->get_cursor_type(); if ( cat > CAT_ADJUST ) - { curr_cat = cat; - fp_only = !ofl->ips_opt->fp_research(); - } RuleDirection dir = get_dir(otn); - PatternMatchData* tmp = get_pmd(ofl, otn->snort_protocol_id, dir); + PatternMatchData* tmp = ofl->ips_opt->get_pattern(otn->snort_protocol_id, dir); if ( !tmp ) continue; content = true; - if ( !fp_only ) - tmp->fp_only = -1; - - tmp->pm_type = get_pm_type(curr_cat); - - FpSelector curr(curr_cat, tmp); + FpSelector curr(curr_cat, ofl->ips_opt, tmp); if ( curr.is_better_than(best, srvc, dir, only_literals) ) { best = curr; - next = ofl->next; - pmds.clear(); - // Add alternate pattern - PatternMatchData* alt_pmd = ofl->ips_opt->get_alternate_pattern(); - if (alt_pmd) - pmds.emplace_back(alt_pmd); - // Add main pattern last - pmds.emplace_back(best.pmd); + node = ofl; } } - if ( best.pmd and best.cat != CAT_SET_RAW and !srvc and !otn->sigInfo.services.empty() ) - { - pmds.clear(); // just include in service group - exclude = true; - } - else - exclude = false; + exclude = best.pmd and (best.cat != CAT_SET_RAW) and !srvc and !otn->sigInfo.services.empty(); if ( content && !best.pmd) ParseWarning(WARN_RULES, "content based rule %u:%u has no eligible fast pattern", otn->sigInfo.gid, otn->sigInfo.sid); + if ( !exclude and best.pmd ) + { + PatternMatchData* alt_pmd = best.opt->get_alternate_pattern(); + if (alt_pmd) + pmds.emplace_back(alt_pmd); + pmds.emplace_back(best.pmd); // add primary pattern last + } return pmds; } +bool make_fast_pattern_only(const OptFpList* ofp, const PatternMatchData* pmd) +{ + // FIXIT-L no_case consideration is mpse specific, delegate + if ( !pmd->is_relative() and !pmd->is_negated() and + !pmd->offset and !pmd->depth and pmd->is_no_case() ) + { + ofp = ofp->next; + if ( !ofp || !ofp->ips_opt || !ofp->ips_opt->is_relative() ) + return true; + } + return false; +} + +bool is_fast_pattern_only(const OptTreeNode* otn, const OptFpList* ofp, Mpse::MpseType mpse_type) +{ + if ( mpse_type == Mpse::MPSE_TYPE_NORMAL and otn->normal_fp_only == ofp ) + return true; + + if ( mpse_type == Mpse::MPSE_TYPE_OFFLOAD and otn->offload_fp_only == ofp ) + return true; + + return false; +} + //-------------------------------------------------------------------------- // mpse compile threads //-------------------------------------------------------------------------- @@ -520,7 +504,7 @@ TEST_CASE("fp_simple", "[FastPatternSelect]") FpSelector test; PatternMatchData pmd; set_pmd(pmd, 0x0, "foo"); - FpSelector left(CAT_SET_RAW, &pmd); + FpSelector left(CAT_SET_RAW, nullptr, &pmd); CHECK(left.is_better_than(test, false, RULE_WO_DIR)); test.size = 1; @@ -531,11 +515,11 @@ TEST_CASE("fp_negated", "[FastPatternSelect]") { PatternMatchData p0; set_pmd(p0, 0x0, "foo"); - FpSelector s0(CAT_SET_RAW, &p0); + FpSelector s0(CAT_SET_RAW, nullptr, &p0); PatternMatchData p1; set_pmd(p1, 0x1, "foo"); - FpSelector s1(CAT_SET_RAW, &p1); + FpSelector s1(CAT_SET_RAW, nullptr, &p1); CHECK(s0.is_better_than(s1, false, RULE_WO_DIR)); CHECK(!s1.is_better_than(s0, false, RULE_WO_DIR)); @@ -545,11 +529,11 @@ TEST_CASE("fp_cat1", "[FastPatternSelect]") { PatternMatchData p0; set_pmd(p0, 0x0, "longer"); - FpSelector s0(CAT_SET_FILE, &p0); + FpSelector s0(CAT_SET_FILE, nullptr, &p0); PatternMatchData p1; set_pmd(p1, 0x0, "short"); - FpSelector s1(CAT_SET_BODY, &p1); + FpSelector s1(CAT_SET_BODY, nullptr, &p1); CHECK(s0.is_better_than(s1, true, RULE_WO_DIR)); } @@ -558,11 +542,11 @@ TEST_CASE("fp_cat2", "[FastPatternSelect]") { PatternMatchData p0; set_pmd(p0, 0x0, "foo"); - FpSelector s0(CAT_SET_RAW, &p0); + FpSelector s0(CAT_SET_RAW, nullptr, &p0); PatternMatchData p1; set_pmd(p1, 0x0, "foo"); - FpSelector s1(CAT_SET_FILE, &p1); + FpSelector s1(CAT_SET_FILE, nullptr, &p1); CHECK(!s0.is_better_than(s1, false, RULE_WO_DIR)); CHECK(!s1.is_better_than(s0, false, RULE_WO_DIR)); @@ -572,11 +556,11 @@ TEST_CASE("fp_cat3", "[FastPatternSelect]") { PatternMatchData p0; set_pmd(p0, 0x0, "foo"); - FpSelector s0(CAT_SET_RAW, &p0); + FpSelector s0(CAT_SET_RAW, nullptr, &p0); PatternMatchData p1; set_pmd(p1, 0x0, "foo"); - FpSelector s1(CAT_SET_FILE, &p1); + FpSelector s1(CAT_SET_FILE, nullptr, &p1); CHECK(!s0.is_better_than(s1, true, RULE_WO_DIR)); } @@ -585,11 +569,11 @@ TEST_CASE("fp_size", "[FastPatternSelect]") { PatternMatchData p0; set_pmd(p0, 0x0, "longer"); - FpSelector s0(CAT_SET_HEADER, &p0); + FpSelector s0(CAT_SET_HEADER, nullptr, &p0); PatternMatchData p1; set_pmd(p1, 0x0, "short"); - FpSelector s1(CAT_SET_HEADER, &p1); + FpSelector s1(CAT_SET_HEADER, nullptr, &p1); CHECK(s0.is_better_than(s1, false, RULE_WO_DIR)); } @@ -598,11 +582,11 @@ TEST_CASE("fp_pkt_key_port", "[FastPatternSelect]") { PatternMatchData p0; set_pmd(p0, 0x0, "short"); - FpSelector s0(CAT_SET_RAW, &p0); + FpSelector s0(CAT_SET_RAW, nullptr, &p0); PatternMatchData p1; set_pmd(p1, 0x0, "longer"); - FpSelector s1(CAT_SET_KEY, &p1); + FpSelector s1(CAT_SET_KEY, nullptr, &p1); CHECK(!s0.is_better_than(s1, false, RULE_WO_DIR)); } @@ -611,11 +595,11 @@ TEST_CASE("fp_pkt_key_port_user", "[FastPatternSelect]") { PatternMatchData p0; set_pmd(p0, 0x10, "short"); - FpSelector s0(CAT_SET_KEY, &p0); + FpSelector s0(CAT_SET_KEY, nullptr, &p0); PatternMatchData p1; set_pmd(p1, 0x0, "longer"); - FpSelector s1(CAT_SET_KEY, &p1); + FpSelector s1(CAT_SET_KEY, nullptr, &p1); CHECK(s0.is_better_than(s1, false, RULE_WO_DIR)); } @@ -624,11 +608,11 @@ TEST_CASE("fp_pkt_key_port_user_user", "[FastPatternSelect]") { PatternMatchData p0; set_pmd(p0, 0x10, "longer"); - FpSelector s0(CAT_SET_KEY, &p0); + FpSelector s0(CAT_SET_KEY, nullptr, &p0); PatternMatchData p1; set_pmd(p1, 0x10, "short"); - FpSelector s1(CAT_SET_KEY, &p1); + FpSelector s1(CAT_SET_KEY, nullptr, &p1); CHECK(!s0.is_better_than(s1, false, RULE_WO_DIR)); } @@ -637,11 +621,11 @@ TEST_CASE("fp_pkt_key_port_user_user2", "[FastPatternSelect]") { PatternMatchData p0; set_pmd(p0, 0x0, "longer"); - FpSelector s0(CAT_SET_KEY, &p0); + FpSelector s0(CAT_SET_KEY, nullptr, &p0); PatternMatchData p1; set_pmd(p1, 0x10, "short"); - FpSelector s1(CAT_SET_KEY, &p1); + FpSelector s1(CAT_SET_KEY, nullptr, &p1); CHECK(!s0.is_better_than(s1, false, RULE_WO_DIR)); } @@ -650,11 +634,11 @@ TEST_CASE("fp_pkt_key_srvc_1", "[FastPatternSelect]") { PatternMatchData p0; set_pmd(p0, 0x0, "short"); - FpSelector s0(CAT_SET_RAW, &p0); + FpSelector s0(CAT_SET_RAW, nullptr, &p0); PatternMatchData p1; set_pmd(p1, 0x0, "longer"); - FpSelector s1(CAT_SET_KEY, &p1); + FpSelector s1(CAT_SET_KEY, nullptr, &p1); CHECK(s1.is_better_than(s0, true, RULE_WO_DIR)); } @@ -663,11 +647,11 @@ TEST_CASE("fp_pkt_key_srvc_2", "[FastPatternSelect]") { PatternMatchData p0; set_pmd(p0, 0x0, "longer"); - FpSelector s0(CAT_SET_RAW, &p0); + FpSelector s0(CAT_SET_RAW, nullptr, &p0); PatternMatchData p1; set_pmd(p1, 0x0, "short"); - FpSelector s1(CAT_SET_KEY, &p1); + FpSelector s1(CAT_SET_KEY, nullptr, &p1); CHECK(s0.is_better_than(s1, true, RULE_WO_DIR)); } @@ -676,11 +660,11 @@ TEST_CASE("fp_pkt_key_srvc_rsp", "[FastPatternSelect]") { PatternMatchData p0; set_pmd(p0, 0x0, "short"); - FpSelector s0(CAT_SET_RAW, &p0); + FpSelector s0(CAT_SET_RAW, nullptr, &p0); PatternMatchData p1; set_pmd(p1, 0x0, "longer"); - FpSelector s1(CAT_SET_KEY, &p1); + FpSelector s1(CAT_SET_KEY, nullptr, &p1); CHECK(!s0.is_better_than(s1, true, RULE_FROM_SERVER)); CHECK(s1.is_better_than(s0, true, RULE_FROM_SERVER)); diff --git a/src/detection/fp_utils.h b/src/detection/fp_utils.h index 7ce0322a0..a95a6b6ef 100644 --- a/src/detection/fp_utils.h +++ b/src/detection/fp_utils.h @@ -25,16 +25,18 @@ #include #include "framework/ips_option.h" #include "framework/mpse.h" +#include "ports/port_group.h" struct OptFpList; struct OptTreeNode; struct PatternMatchData* get_pmd(OptFpList*, SnortProtocolId, snort::RuleDirection); -bool is_fast_pattern_only(OptFpList*, snort::Mpse::MpseType); -bool is_fast_pattern_only(OptFpList*); -void validate_fast_pattern(OptTreeNode*); -unsigned flp_trim(const char* p, unsigned plen, const char** buff); +bool make_fast_pattern_only(const OptFpList*, const PatternMatchData*); +bool is_fast_pattern_only(const OptTreeNode*, const OptFpList*, snort::Mpse::MpseType); + +PmType get_pm_type(snort::CursorActionType); + bool set_fp_content(OptTreeNode*); std::vector get_fp_content( @@ -43,5 +45,7 @@ std::vector get_fp_content( void queue_mpse(snort::Mpse*); unsigned compile_mpses(struct snort::SnortConfig*, bool parallel = false); +void validate_services(struct snort::SnortConfig*, OptTreeNode*); + #endif diff --git a/src/detection/pattern_match_data.h b/src/detection/pattern_match_data.h index c167b861a..9afc88a64 100644 --- a/src/detection/pattern_match_data.h +++ b/src/detection/pattern_match_data.h @@ -70,7 +70,6 @@ struct PatternMatchData uint16_t fp_length; // not used by ips_content - int8_t fp_only; uint8_t pm_type; bool is_unbounded() const diff --git a/src/detection/rules.cc b/src/detection/rules.cc index 8317d500d..1795229bc 100644 --- a/src/detection/rules.cc +++ b/src/detection/rules.cc @@ -30,6 +30,7 @@ #include "hash/xhash.h" #include "main/policy.h" #include "main/snort_config.h" +#include "parser/parse_conf.h" #include "parser/parser.h" #include "sfip/sf_ipvar.h" #include "sfip/sf_vartable.h" @@ -138,5 +139,11 @@ void RuleStateMap::update_rtn(RuleTreeNode* rtn, const RuleState& s) case IpsPolicy::INHERIT_ENABLE: break; } rtn->action = s.action; + + if ( rtn->header ) + { + rtn->header = new RuleHeader(*rtn->header); + rtn->header->action = s.rule_action; + } } diff --git a/src/detection/rules.h b/src/detection/rules.h index 33e4ce618..e0521cfe4 100644 --- a/src/detection/rules.h +++ b/src/detection/rules.h @@ -25,6 +25,7 @@ // FIXIT-L refactor this header #include +#include #include "actions/actions.h" #include "main/policy.h" @@ -81,6 +82,7 @@ struct RuleKey struct RuleState { + std::string rule_action; snort::Actions::Type action; IpsPolicy::Enable enable; }; diff --git a/src/detection/service_map.cc b/src/detection/service_map.cc index 4e7dee794..26d7a575d 100644 --- a/src/detection/service_map.cc +++ b/src/detection/service_map.cc @@ -38,7 +38,6 @@ #include #include "hash/ghash.h" -#include "ips_options/ips_flow.h" #include "log/messages.h" #include "main/snort_config.h" #include "parser/parser.h" @@ -164,10 +163,10 @@ static void ServiceMapAddOtn( { assert(servicename and otn); - if ( !OtnFlowFromClient(otn) ) + if ( !otn->to_server() ) ServiceMapAddOtnRaw(srmm->to_cli, servicename, otn); - if ( !OtnFlowFromServer(otn) ) + if ( !otn->to_client() ) ServiceMapAddOtnRaw(srmm->to_srv, servicename, otn); } diff --git a/src/detection/treenodes.h b/src/detection/treenodes.h index 8451484c9..f259dc7f6 100644 --- a/src/detection/treenodes.h +++ b/src/detection/treenodes.h @@ -27,6 +27,7 @@ #include "detection/rule_option_types.h" #include "main/policy.h" #include "main/snort_types.h" +#include "ports/port_group.h" #include "time/clock_defs.h" namespace snort @@ -169,6 +170,8 @@ struct OptTreeNode static constexpr Flag STATELESS = 0x02; static constexpr Flag RULE_STATE = 0x04; static constexpr Flag META_MATCH = 0x08; + static constexpr Flag TO_CLIENT = 0x10; + static constexpr Flag TO_SERVER = 0x20; /* metadata about signature */ SigInfo sigInfo; @@ -179,6 +182,9 @@ struct OptTreeNode OutputSet* outputFuncs = nullptr; /* per sid enabled output functions */ snort::IpsOption* agent = nullptr; + OptFpList* normal_fp_only = nullptr; + OptFpList* offload_fp_only = nullptr; + struct THD_NODE* detection_filter = nullptr; /* if present, evaluated last, after header checks */ TagData* tag = nullptr; @@ -195,6 +201,8 @@ struct OptTreeNode IpsPolicy::Enable enable; Flag flags = 0; + uint8_t sticky_buf = PM_TYPE_PKT; // parsing only + void set_warned_fp() { flags |= WARNED_FP; } @@ -227,6 +235,20 @@ struct OptTreeNode bool metadata_matched() const { return (flags & META_MATCH) != 0; } + + void set_to_client() + { flags |= TO_CLIENT; } + + bool to_client() + { return (flags & TO_CLIENT) != 0; } + + void set_to_server() + { flags |= TO_SERVER; } + + bool to_server() + { return (flags & TO_SERVER) != 0; } + + void update_fp(snort::IpsOption*); }; typedef int (* RuleOptEvalFunc)(void*, Cursor&, snort::Packet*); diff --git a/src/framework/inspector.h b/src/framework/inspector.h index 2638cb141..b51bd701e 100644 --- a/src/framework/inspector.h +++ b/src/framework/inspector.h @@ -45,7 +45,9 @@ struct InspectionBuffer enum Type { // FIXIT-L file data is tbd - IBT_KEY, IBT_HEADER, IBT_BODY, IBT_FILE, IBT_ALT, IBT_MAX + IBT_KEY, IBT_HEADER, IBT_BODY, IBT_FILE, IBT_ALT, + IBT_RAW_KEY, IBT_RAW_HEADER, IBT_METHOD, IBT_STAT_CODE, + IBT_STAT_MSG, IBT_COOKIE, IBT_MAX }; const uint8_t* data; unsigned len; diff --git a/src/framework/ips_option.h b/src/framework/ips_option.h index 9e514184e..afe1539ed 100644 --- a/src/framework/ips_option.h +++ b/src/framework/ips_option.h @@ -52,6 +52,12 @@ enum CursorActionType CAT_ADJUST, CAT_SET_OTHER, CAT_SET_RAW, + CAT_SET_COOKIE, + CAT_SET_STAT_MSG, + CAT_SET_STAT_CODE, + CAT_SET_METHOD, + CAT_SET_RAW_HEADER, + CAT_SET_RAW_KEY, CAT_SET_FILE, CAT_SET_BODY, CAT_SET_HEADER, @@ -81,7 +87,6 @@ public: // packet threads virtual bool is_relative() { return false; } - virtual bool fp_research() { return false; } virtual bool retry(Cursor&) { return false; } virtual void action(Packet*) { } diff --git a/src/framework/mpse.h b/src/framework/mpse.h index 3b083d737..cd767baa4 100644 --- a/src/framework/mpse.h +++ b/src/framework/mpse.h @@ -125,7 +125,6 @@ typedef void (* MpseDelFunc)(Mpse*); typedef Mpse::MpseRespType (* MpsePollFunc)(MpseBatch*&, Mpse::MpseType); #define MPSE_BASE 0x00 // no optional features -#define MPSE_TRIM 0x01 // should trim leading zero bytes from patterns #define MPSE_REGEX 0x02 // supports regex patterns #define MPSE_ASYNC 0x04 // does asynchronous (lookaside) searches #define MPSE_MTBLD 0x08 // support multithreaded / parallel compilation diff --git a/src/ips_options/CMakeLists.txt b/src/ips_options/CMakeLists.txt index 65a0af88c..6c7ddf0ec 100644 --- a/src/ips_options/CMakeLists.txt +++ b/src/ips_options/CMakeLists.txt @@ -56,7 +56,6 @@ set (IPS_SOURCES ips_dsize.cc ips_file_data.cc ips_flow.cc - ips_flow.h ips_flowbits.cc ips_flowbits.h ips_hash.cc diff --git a/src/ips_options/ips_content.cc b/src/ips_options/ips_content.cc index 0830bbdec..30183a6e7 100644 --- a/src/ips_options/ips_content.cc +++ b/src/ips_options/ips_content.cc @@ -23,12 +23,14 @@ #endif #include "detection/pattern_match_data.h" +#include "detection/treenodes.h" #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" #include "hash/hash_key_operations.h" #include "helpers/literal_search.h" #include "log/messages.h" +#include "main/thread_config.h" #include "parser/parse_utils.h" #include "profiler/profiler.h" #include "utils/util.h" @@ -179,12 +181,11 @@ bool ContentOption::retry(Cursor& c) uint32_t ContentOption::hash() const { - uint32_t a,b,c; const ContentData* cd = config; - a = cd->pmd.flags; - b = cd->pmd.offset; - c = cd->pmd.depth; + uint32_t a = cd->pmd.flags; + uint32_t b = cd->pmd.offset; + uint32_t c = cd->pmd.depth; mix(a,b,c); @@ -194,38 +195,36 @@ uint32_t ContentOption::hash() const mix(a,b,c); + a += cd->pmd.pm_type; + if ( cd->pmd.pattern_size ) mix_str(a, b, c, cd->pmd.pattern_buf, cd->pmd.pattern_size); - mix_str(a,b,c,get_name()); + mix_str(a, b, c, get_name()); a += cd->depth_var; b += cd->offset_var; c += cd->match_delta; - mix(a,b,c); - finalize(a,b,c); + mix(a, b, c); + + finalize(a, b, c); return c; } -#if 0 -// see below for why this is disabled static bool same_buffers( unsigned len1, const char* buf1, bool no_case1, unsigned len2, const char* buf2, bool no_case2) { - /* Sizes will be most different, check that first */ if ( len1 != len2 or no_case1 != no_case2 ) return false; if ( !len1 ) return true; - /* Next compare the patterns for uniqueness */ if ( no_case1 ) { - /* If no_case is set, do case insensitive compare on pattern */ for ( unsigned i = 0; i < len1; ++i ) { if ( toupper(buf1[i]) != toupper(buf2[i]) ) @@ -234,21 +233,14 @@ static bool same_buffers( } else { - /* If no_case is not set, do case sensitive compare on pattern */ if ( memcmp(buf1, buf2, len1) ) return false; } return true; } -#endif - -// FIXIT-P FAST_PAT and fp_only are set after hash table comparisons so this must -// return this == &ips to avoid unnecessary reevaluation and false positives. -// when this is fixed, add PatternMatchData::operator==(). bool ContentOption::operator==(const IpsOption& ips) const { -#if 0 if ( !IpsOption::operator==(ips) ) return false; @@ -260,22 +252,20 @@ bool ContentOption::operator==(const IpsOption& ips) const right.pmd.pattern_size, right.pmd.pattern_buf, right.pmd.is_no_case()) ) return false; - /* Now check the rest of the options */ - if ((left.pmd.flags == right.pmd.flags) && - (left.pmd.fp_offset == right.pmd.fp_offset) && - (left.pmd.fp_length == right.pmd.fp_length) && - (left.pmd.offset == right.pmd.offset) && - (left.pmd.depth == right.pmd.depth) && - // pattern_size and pattern_buf already checked - // pm_type set later (but determined by CAT) - (left.match_delta == right.match_delta) && - (left.offset_var == right.offset_var) && + if ( + (left.pmd.flags == right.pmd.flags) and + (left.pmd.offset == right.pmd.offset) and + (left.pmd.depth == right.pmd.depth) and + (left.pmd.fp_offset == right.pmd.fp_offset) and + (left.pmd.fp_length == right.pmd.fp_length) and + (left.pmd.pm_type == right.pmd.pm_type) and + (left.match_delta == right.match_delta) and + (left.offset_var == right.offset_var) and (left.depth_var == right.depth_var) ) { return true; } -#endif - return this == &ips; + return false; } //------------------------------------------------------------------------- @@ -690,6 +680,13 @@ bool ContentModule::end(const char*, int, SnortConfig*) s[i] = toupper(cd->pmd.pattern_buf[i]); } cd->setup_bm(); + + if ( cd->pmd.is_negated() ) + { + cd->pmd.last_check = (PmdLastCheck*)snort_calloc( + ThreadConfig::get_instance_max(), sizeof(*cd->pmd.last_check)); + } + return true; } @@ -746,10 +743,11 @@ static void mod_dtor(Module* m) delete m; } -static IpsOption* content_ctor(Module* p, OptTreeNode*) +static IpsOption* content_ctor(Module* p, OptTreeNode* otn) { ContentModule* m = (ContentModule*)p; ContentData* cd = m->get_data(); + cd->pmd.pm_type = otn->sticky_buf; return new ContentOption(cd); } diff --git a/src/ips_options/ips_flow.cc b/src/ips_options/ips_flow.cc index bedf084aa..107c869db 100644 --- a/src/ips_options/ips_flow.cc +++ b/src/ips_options/ips_flow.cc @@ -22,8 +22,6 @@ #include "config.h" #endif -#include "ips_flow.h" - #include "detection/treenodes.h" #include "framework/ips_option.h" #include "framework/module.h" @@ -198,86 +196,57 @@ IpsOption::EvalStatus FlowCheckOption::eval(Cursor&, Packet* p) return MATCH; } -//------------------------------------------------------------------------- -// public methods -//------------------------------------------------------------------------- - -int OtnFlowFromServer(OptTreeNode* otn) -{ - FlowCheckOption* fco = - (FlowCheckOption*)get_rule_type_data(otn, s_name); - - if (fco ) - { - if ( fco->config.from_server ) - return 1; - } - return 0; -} - -int OtnFlowFromClient(OptTreeNode* otn) -{ - FlowCheckOption* fco = - (FlowCheckOption*)get_rule_type_data(otn, s_name); - - if (fco ) - { - if ( fco->config.from_client ) - return 1; - } - return 0; -} - //------------------------------------------------------------------------- // support methods //------------------------------------------------------------------------- -static void flow_verify(FlowCheckData* fcd) +static bool flow_verify(FlowCheckData* fcd) { if (fcd->from_client && fcd->from_server) { ParseError("can't use both from_client and flow_from server"); - return; + return false; } if ((fcd->ignore_reassembled & IGNORE_STREAM) && (fcd->only_reassembled & ONLY_STREAM)) { ParseError("can't use no_stream and only_stream"); - return; + return false; } if ((fcd->ignore_reassembled & IGNORE_FRAG) && (fcd->only_reassembled & ONLY_FRAG)) { ParseError("can't use no_frag and only_frag"); - return; + return false; } if (fcd->stateless && (fcd->from_client || fcd->from_server)) { ParseError("can't use flow: stateless option with other options"); - return; + return false; } if (fcd->stateless && fcd->established) { ParseError("can't specify established and stateless " "options in same rule"); - return; + return false; } if (fcd->stateless && fcd->unestablished) { ParseError("can't specify unestablished and stateless " "options in same rule"); - return; + return false; } if (fcd->established && fcd->unestablished) { ParseError("can't specify unestablished and established " "options in same rule"); - return; + return false; } + return true; } //------------------------------------------------------------------------- @@ -331,6 +300,7 @@ public: FlowModule() : Module(s_name, s_help, s_params) { } bool begin(const char*, int, SnortConfig*) override; + bool end(const char*, int, SnortConfig*) override; bool set(const char*, Value&, SnortConfig*) override; ProfileStats* get_profile() const override @@ -349,6 +319,11 @@ bool FlowModule::begin(const char*, int, SnortConfig*) return true; } +bool FlowModule::end(const char*, int, SnortConfig*) +{ + return flow_verify(&data); +} + bool FlowModule::set(const char*, Value& v, SnortConfig*) { if ( v.is("to_server") ) @@ -387,7 +362,6 @@ bool FlowModule::set(const char*, Value& v, SnortConfig*) else return false; - flow_verify(&data); return true; } @@ -412,6 +386,12 @@ static IpsOption* flow_ctor(Module* p, OptTreeNode* otn) if ( m->data.stateless ) otn->set_stateless(); + if ( m->data.from_server ) + otn->set_to_client(); + + else if ( m->data.from_client ) + otn->set_to_server(); + if (otn->snort_protocol_id == SNORT_PROTO_ICMP) { if ( (m->data.only_reassembled != ONLY_FRAG) && diff --git a/src/ips_options/ips_flow.h b/src/ips_options/ips_flow.h deleted file mode 100644 index c2042ba34..000000000 --- a/src/ips_options/ips_flow.h +++ /dev/null @@ -1,30 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2020 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2002-2013 Sourcefire, Inc. -// Copyright (C) 1998-2002 Martin Roesch -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -#ifndef IPS_FLOW_H -#define IPS_FLOW_H - -struct OptTreeNode; - -int OtnFlowFromServer(OptTreeNode*); -int OtnFlowFromClient(OptTreeNode*); - -#endif - diff --git a/src/ips_options/ips_regex.cc b/src/ips_options/ips_regex.cc index c99ddcc24..d4b7dbd41 100644 --- a/src/ips_options/ips_regex.cc +++ b/src/ips_options/ips_regex.cc @@ -29,6 +29,7 @@ #include #include "detection/pattern_match_data.h" +#include "detection/treenodes.h" #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" @@ -43,14 +44,14 @@ using namespace snort; #define s_name "regex" #define s_help \ - "rule option for matching payload data with hyperscan regex" + "rule option for matching payload data with hyperscan regex; uses pcre syntax" struct RegexConfig { hs_database_t* db; std::string re; PatternMatchData pmd; - bool pcre_conversion; + bool pcre_upgrade; RegexConfig() { reset(); } @@ -60,7 +61,7 @@ struct RegexConfig memset(&pmd, 0, sizeof(pmd)); re.clear(); db = nullptr; - pcre_conversion = false; + pcre_upgrade = false; } }; @@ -120,14 +121,17 @@ RegexOption::~RegexOption() uint32_t RegexOption::hash() const { - uint32_t a = config.pmd.flags, b = config.pmd.mpse_flags, c = 0; + uint32_t a = config.pmd.flags; + uint32_t b = config.pmd.mpse_flags; + uint32_t c = config.pmd.pm_type; + mix_str(a, b, c, config.re.c_str()); mix_str(a, b, c, get_name()); finalize(a, b, c); + return c; } -// see ContentOption::operator==() bool RegexOption::operator==(const IpsOption& ips) const { if ( !IpsOption::operator==(ips) ) @@ -135,11 +139,11 @@ bool RegexOption::operator==(const IpsOption& ips) const const RegexOption& rhs = (const RegexOption&)ips; - if ( config.pcre_conversion && rhs.config.pcre_conversion ) - if ( config.re == rhs.config.re and - config.pmd.flags == rhs.config.pmd.flags and - config.pmd.mpse_flags == rhs.config.pmd.mpse_flags ) - return true; + if ( config.re == rhs.config.re and + config.pmd.pm_type == rhs.config.pmd.pm_type and + config.pmd.flags == rhs.config.pmd.flags and + config.pmd.mpse_flags == rhs.config.pmd.mpse_flags ) + return true; return false; } @@ -258,111 +262,118 @@ RegexModule::~RegexModule() bool RegexModule::begin(const char* name, int, SnortConfig*) { config.reset(); - config.pmd.flags |= PatternMatchData::NO_FP; config.pmd.mpse_flags |= HS_FLAG_SINGLEMATCH; - // if regex is in pcre syntax set conversion mode if ( strcmp(name, "pcre") == 0 ) - config.pcre_conversion = true; + config.pcre_upgrade = true; return true; } -// The regex string received from the ips_pcre plugin must be scrubbed to remove +// The regex string is in pcre syntax so it must be scrubbed to remove // two characters from the front; an extra '"' and the '/' and also the same // two characters from the end of the string as well as any pcre modifier flags // included in the expression. The modifier flags are checked to set the // corresponding hyperscan regex engine flags. -// Hyperscan regex also does not support negated pcre expression so negated expression -// are not converted and will be compiled by the pcre engine. bool RegexModule::convert_pcre_to_regex_form() { - size_t pos = config.re.find_first_of("\"!"); - if (pos != std::string::npos and config.re[pos] == '!') + // we get string with quotes so length is at least 3 + // start with a bang: ! "/regex/smi" + if ( config.re[0] == '!' ) + { + if ( !config.pcre_upgrade ) + ParseError("regex does not (yet) support negation"); return false; + } - config.re.erase(0,2); - std::size_t re_end = config.re.rfind("/"); - if ( re_end != std::string::npos ) + // remove quotes: "/regex/smi" -> /regex/smi + config.re.erase(0, 1); + config.re.erase(config.re.length() - 1, 1); + + // remove leading slash: /regex/smi -> regex/smi + size_t len = config.re.length(); + + if ( len < 3 or config.re[0] != '/' ) { - std::size_t mod_len = (config.re.length() - 2) - re_end; - std::string modifiers = config.re.substr(re_end + 1, mod_len); - std::size_t erase_len = config.re.length() - re_end; - config.re.erase(re_end, erase_len); + ParseError("regex uses pcre syntax"); + return false; + } + config.re.erase(0, 1); - for( char& c : modifiers ) - { - switch (c) - { - case 'i': - config.pmd.mpse_flags |= HS_FLAG_CASELESS; - config.pmd.set_no_case(); - break; - - case 'm': - config.pmd.mpse_flags |= HS_FLAG_MULTILINE; - break; - - case 's': - config.pmd.mpse_flags |= HS_FLAG_DOTALL; - break; - - case 'O': - break; - - case 'R': - config.pmd.set_relative(); - break; - - default: - return false; - break; - } - } + // remove trailing slash: regex/smi -> regexsmi + size_t re_end = config.re.rfind("/"); - return true; + if ( re_end == std::string::npos ) + { + ParseError("regex uses pcre syntax"); + return false; } + config.re.erase(re_end, 1); - return false; + // capture and remove optional modifiers: regex/smi -> regex, smi + std::string modifiers; + len = config.re.length() - re_end; + + if ( len > 0 ) + { + modifiers = config.re.substr(re_end, len); + config.re.erase(re_end, len); + } + + // finally, process the modifiers + for ( char& c : modifiers ) + { + switch ( c ) + { + case 'i': + config.pmd.mpse_flags |= HS_FLAG_CASELESS; + config.pmd.set_no_case(); + break; + + case 'm': + config.pmd.mpse_flags |= HS_FLAG_MULTILINE; + break; + + case 's': + config.pmd.mpse_flags |= HS_FLAG_DOTALL; + break; + + case 'O': + if ( !config.pcre_upgrade ) + ParseWarning(WARN_RULES, "regex does not support override, ignored"); + break; + + case 'R': + config.pmd.set_relative(); + break; + + default: + return false; + } + } + return true; } bool RegexModule::set(const char*, Value& v, SnortConfig*) { - bool valid_opt = true; - if ( v.is("~re") ) { config.re = v.get_string(); - - if ( config.pcre_conversion ) - valid_opt = convert_pcre_to_regex_form(); - else - { - // remove quotes - config.re.erase(0, 1); - config.re.erase(config.re.length() - 1, 1); - } + return convert_pcre_to_regex_form(); } - else if ( v.is("dotall") ) - config.pmd.mpse_flags |= HS_FLAG_DOTALL; else if ( v.is("fast_pattern") ) - { - config.pmd.flags &= ~PatternMatchData::NO_FP; - config.pmd.flags |= PatternMatchData::FAST_PAT; - } - else if ( v.is("multiline") ) - config.pmd.mpse_flags |= HS_FLAG_MULTILINE; + config.pmd.set_fast_pattern(); + + else if ( v.is("nocase") ) { config.pmd.mpse_flags |= HS_FLAG_CASELESS; config.pmd.set_no_case(); } - else if ( v.is("relative") ) - config.pmd.set_relative(); else - valid_opt = false; + return false; - return valid_opt; + return true; } bool RegexModule::end(const char*, int, SnortConfig*) @@ -373,12 +384,16 @@ bool RegexModule::end(const char*, int, SnortConfig*) return false; } + if ( !config.pmd.is_fast_pattern() ) + config.pmd.flags |= PatternMatchData::NO_FP; + hs_compile_error_t* err = nullptr; if ( hs_compile(config.re.c_str(), config.pmd.mpse_flags, HS_MODE_BLOCK, nullptr, &config.db, &err) or !config.db ) { - if ( !config.pcre_conversion ) + // gracefully fall back to pcre upon upgrade failure + if ( !config.pcre_upgrade ) ParseError("can't compile regex '%s'", config.re.c_str()); hs_free_compile_error(err); return false; @@ -396,11 +411,12 @@ static Module* mod_ctor() static void mod_dtor(Module* p) { delete p; } -static IpsOption* regex_ctor(Module* m, OptTreeNode*) +static IpsOption* regex_ctor(Module* m, OptTreeNode* otn) { RegexModule* mod = (RegexModule*)m; RegexConfig c; mod->get_data(c); + c.pmd.pm_type = otn->sticky_buf; return new RegexOption(c); } diff --git a/src/ips_options/ips_sd_pattern.cc b/src/ips_options/ips_sd_pattern.cc index 3a09e2ef9..38e7a7f1a 100644 --- a/src/ips_options/ips_sd_pattern.cc +++ b/src/ips_options/ips_sd_pattern.cc @@ -28,6 +28,7 @@ #include #include "detection/pattern_match_data.h" +#include "detection/treenodes.h" #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" @@ -146,10 +147,14 @@ SdPatternOption::~SdPatternOption() uint32_t SdPatternOption::hash() const { - uint32_t a = 0, b = 0, c = config.threshold; + uint32_t a = config.pmd.pattern_size; + uint32_t b = config.pmd.pm_type; + uint32_t c = config.threshold; + mix_str(a, b, c, config.pii.c_str()); mix_str(a, b, c, get_name()); finalize(a, b, c); + return c; } @@ -160,7 +165,7 @@ bool SdPatternOption::operator==(const IpsOption& ips) const const SdPatternOption& rhs = static_cast(ips); - if ( config == rhs.config ) + if ( config == rhs.config ) // FIXIT-H seems incorrect return true; return false; @@ -397,11 +402,12 @@ static void mod_dtor(Module* p) delete p; } -static IpsOption* sd_pattern_ctor(Module* m, OptTreeNode*) +static IpsOption* sd_pattern_ctor(Module* m, OptTreeNode* otn) { SdPatternModule* mod = (SdPatternModule*)m; SdPatternConfig c; mod->get_data(c); + c.pmd.pm_type = otn->sticky_buf; return new SdPatternOption(c); } diff --git a/src/ips_options/test/ips_regex_test.cc b/src/ips_options/test/ips_regex_test.cc index 16bb91889..e408e2fcb 100644 --- a/src/ips_options/test/ips_regex_test.cc +++ b/src/ips_options/test/ips_regex_test.cc @@ -22,12 +22,15 @@ #include "config.h" #endif +#include "detection/treenodes.h" #include "framework/base_api.h" #include "framework/counts.h" #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" +#include "log/messages.h" #include "main/snort_config.h" +#include "ports/port_group.h" #include "profiler/profiler_defs.h" #include "protocols/packet.h" @@ -81,6 +84,8 @@ static unsigned s_parse_errors = 0; void ParseError(const char*, ...) { s_parse_errors++; } +void ParseWarning(WarningGroup, const char*, ...) { } + unsigned get_instance_id() { return 0; } @@ -101,6 +106,8 @@ Cursor::Cursor(Packet* p) void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { } void show_stats(PegCount*, const PegInfo*, const IndexVec&, const char*, FILE*) { } +OptTreeNode::~OptTreeNode() { } + //------------------------------------------------------------------------- // helpers //------------------------------------------------------------------------- @@ -118,24 +125,21 @@ static const Parameter* get_param(Module* m, const char* s) return nullptr; } -static IpsOption* get_option(Module* mod, const char* pat, bool relative = false) +static IpsOption* get_option(Module* mod, const char* pat) { mod->begin(ips_regex->name, 0, nullptr); Value vs(pat); vs.set(get_param(mod, "~re")); - mod->set(ips_regex->name, vs, nullptr); - if ( relative ) - { - Value vb(relative); - vb.set(get_param(mod, "relative")); - mod->set(ips_regex->name, vb, nullptr); - } + mod->set(ips_regex->name, vs, nullptr); mod->end(ips_regex->name, 0, nullptr); + OptTreeNode otn; + otn.sticky_buf = 0; + const IpsApi* api = (const IpsApi*) ips_regex; - IpsOption* opt = api->ctor(mod, nullptr); + IpsOption* opt = api->ctor(mod, &otn); return opt; } @@ -198,7 +202,7 @@ TEST_GROUP(ips_regex_module) TEST(ips_regex_module, basic) { // always need a re - Value vs("foo"); + Value vs("\"/highway star/\""); const Parameter* p = get_param(mod, "~re"); CHECK(p); vs.set(p); @@ -209,42 +213,22 @@ TEST(ips_regex_module, basic) TEST(ips_regex_module, config_pass) { - Value vs("foo"); + Value vs("\"/jon lord/\""); const Parameter* p = get_param(mod, "~re"); CHECK(p); vs.set(p); CHECK(mod->set(ips_regex->name, vs, nullptr)); Value vb(true); - p = get_param(mod, "dotall"); - CHECK(p); - vb.set(p); - CHECK(mod->set(ips_regex->name, vb, nullptr)); - p = get_param(mod, "fast_pattern"); CHECK(p); vb.set(p); CHECK(mod->set(ips_regex->name, vb, nullptr)); - - p = get_param(mod, "multiline"); - CHECK(p); - vb.set(p); - CHECK(mod->set(ips_regex->name, vb, nullptr)); - - p = get_param(mod, "nocase"); - CHECK(p); - vb.set(p); - CHECK(mod->set(ips_regex->name, vb, nullptr)); - - p = get_param(mod, "relative"); - CHECK(p); - vb.set(p); - CHECK(mod->set(ips_regex->name, vb, nullptr)); } TEST(ips_regex_module, config_fail_name) { - Value vs("unknown"); + Value vs("lazy"); Parameter bad { "bad", Parameter::PT_STRING, nullptr, nullptr, "bad" }; vs.set(&bad); CHECK(!mod->set(ips_regex->name, vs, nullptr)); @@ -254,7 +238,7 @@ TEST(ips_regex_module, config_fail_name) TEST(ips_regex_module, config_fail_regex) { - Value vs("[[:fubar:]]"); + Value vs("\"/[[:fubar:]]/\""); const Parameter* p = get_param(mod, "~re"); CHECK(p); vs.set(p); @@ -276,7 +260,7 @@ TEST_GROUP(ips_regex_option) void setup() override { mod = ips_regex->mod_ctor(); - opt = get_option(mod, " foo "); + opt = get_option(mod, "\"/foo/\""); } void teardown() override { @@ -290,7 +274,7 @@ TEST_GROUP(ips_regex_option) TEST(ips_regex_option, hash) { - IpsOption* opt2 = get_option(mod, "bar"); + IpsOption* opt2 = get_option(mod, "\"/machine head/\""); CHECK(opt2); uint32_t h1 = opt->hash(); @@ -305,7 +289,7 @@ TEST(ips_regex_option, hash) TEST(ips_regex_option, opeq) { - IpsOption* opt2 = get_option(mod, " foo "); + IpsOption* opt2 = get_option(mod, "\"/foo/\""); CHECK(opt2); do_cleanup = scratcher->setup(snort_conf); @@ -355,7 +339,7 @@ TEST_GROUP(ips_regex_option_relative) void setup() override { mod = ips_regex->mod_ctor(); - opt = get_option(mod, "\\bfoo", true); + opt = get_option(mod, "\"/\\bfoo/R\""); } void teardown() override { diff --git a/src/main/modules.cc b/src/main/modules.cc index 2241531b7..1230faf71 100644 --- a/src/main/modules.cc +++ b/src/main/modules.cc @@ -1788,7 +1788,10 @@ bool RuleStateModule::set(const char* fqn, Value& v, SnortConfig*) if ( key.gid and key.sid ) { if ( v.is("action") ) + { + state.rule_action = v.get_string(); state.action = snort::Actions::Type(v.get_uint8() + 1); + } else if ( v.is("enable") ) state.enable = IpsPolicy::Enable(v.get_uint8()); diff --git a/src/managers/ips_manager.cc b/src/managers/ips_manager.cc index 494132d81..35545be44 100644 --- a/src/managers/ips_manager.cc +++ b/src/managers/ips_manager.cc @@ -264,12 +264,12 @@ bool IpsManager::option_set( return true; } -bool IpsManager::option_end( +IpsOption* IpsManager::option_end( SnortConfig* sc, OptTreeNode* otn, SnortProtocolId snort_protocol_id, const char* key, RuleOptType& type) { if ( current_keyword.empty() ) - return false; + return nullptr; assert(!strcmp(current_keyword.c_str(), key)); @@ -290,14 +290,14 @@ bool IpsManager::option_end( { ParseError("unknown option %s", key); current_keyword.clear(); - return false; + return nullptr; } if ( mod and !mod->end(key, 0, sc) ) { ParseError("can't finalize %s", key); current_keyword.clear(); - return false; + return nullptr; } IpsOption* ips = opt->api->ctor(mod, otn); @@ -305,7 +305,7 @@ bool IpsManager::option_end( current_keyword.clear(); if ( !ips ) - return ( type == OPT_TYPE_META ); + return nullptr; if ( void* prev = add_detection_option(sc, ips->get_type(), ips) ) { @@ -326,7 +326,7 @@ bool IpsManager::option_end( ParseWarning(WARN_RULES, "at most one action per rule is allowed; other actions disabled"); } - return true; + return ips; } //------------------------------------------------------------------------- diff --git a/src/managers/ips_manager.h b/src/managers/ips_manager.h index f00ab2501..7196537a8 100644 --- a/src/managers/ips_manager.h +++ b/src/managers/ips_manager.h @@ -64,7 +64,7 @@ public: static bool option_begin(snort::SnortConfig*, const char* key, SnortProtocolId); static bool option_set( snort::SnortConfig*, const char* key, const char* opt, const char* val); - static bool option_end( + static snort::IpsOption* option_end( snort::SnortConfig*, OptTreeNode*, SnortProtocolId, const char* key, snort::RuleOptType&); static void delete_option(snort::IpsOption*); diff --git a/src/managers/mpse_manager.cc b/src/managers/mpse_manager.cc index 9cb739a64..ea2c074ce 100644 --- a/src/managers/mpse_manager.cc +++ b/src/managers/mpse_manager.cc @@ -141,12 +141,6 @@ void MpseManager::stop_search_engine(const MpseApi* api) api->stop(); } -bool MpseManager::search_engine_trim(const MpseApi* api) -{ - assert(api); - return (api->flags & MPSE_TRIM) != 0; -} - bool MpseManager::is_async_capable(const MpseApi* api) { assert(api); diff --git a/src/managers/mpse_manager.h b/src/managers/mpse_manager.h index 0e721ffbb..3417c06bc 100644 --- a/src/managers/mpse_manager.h +++ b/src/managers/mpse_manager.h @@ -74,7 +74,6 @@ public: static void setup_search_engine(const snort::MpseApi*, snort::SnortConfig*); static void start_search_engine(const snort::MpseApi*); static void stop_search_engine(const snort::MpseApi*); - static bool search_engine_trim(const snort::MpseApi*); static bool is_async_capable(const snort::MpseApi*); static bool is_regex_capable(const snort::MpseApi*); static bool parallel_compiles(const snort::MpseApi*); diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc index fa9e994b8..8d62fda16 100644 --- a/src/network_inspectors/appid/appid_discovery.cc +++ b/src/network_inspectors/appid/appid_discovery.cc @@ -51,8 +51,8 @@ using namespace snort; AppIdDiscovery::AppIdDiscovery() { - tcp_patterns = new SearchTool("ac_full", true); - udp_patterns = new SearchTool("ac_full", true); + tcp_patterns = new SearchTool; + udp_patterns = new SearchTool; } AppIdDiscovery::~AppIdDiscovery() diff --git a/src/network_inspectors/appid/appid_utils/sf_mlmp.cc b/src/network_inspectors/appid/appid_utils/sf_mlmp.cc index 20c4dedd7..6eb3668dd 100644 --- a/src/network_inspectors/appid/appid_utils/sf_mlmp.cc +++ b/src/network_inspectors/appid/appid_utils/sf_mlmp.cc @@ -205,7 +205,7 @@ static int createTreesRecusively(tMlmpTree* rootNode) tPatternNode* ddPatternNode; /* set up the MPSE for url patterns */ - patternMatcher = rootNode->patternTree = new SearchTool("ac_full", true); + patternMatcher = rootNode->patternTree = new SearchTool; for (primaryPatternNode = rootNode->patternList; primaryPatternNode; diff --git a/src/network_inspectors/appid/detector_plugins/detector_imap.cc b/src/network_inspectors/appid/detector_plugins/detector_imap.cc index fd009b75b..e2b0151ba 100644 --- a/src/network_inspectors/appid/detector_plugins/detector_imap.cc +++ b/src/network_inspectors/appid/detector_plugins/detector_imap.cc @@ -520,7 +520,7 @@ void ImapClientDetector::do_custom_init() { if (cmd_matcher) delete cmd_matcher; - cmd_matcher = new SearchTool("ac_full", true); + cmd_matcher = new SearchTool; if ( !tcp_patterns.empty() ) { diff --git a/src/network_inspectors/appid/detector_plugins/detector_pattern.cc b/src/network_inspectors/appid/detector_plugins/detector_pattern.cc index 739961ce8..e39ad6239 100644 --- a/src/network_inspectors/appid/detector_plugins/detector_pattern.cc +++ b/src/network_inspectors/appid/detector_plugins/detector_pattern.cc @@ -130,7 +130,7 @@ static void register_pattern(SearchTool** patterns, Pattern* pattern) { if (!*patterns) { - *patterns = new SearchTool("ac_full", true); + *patterns = new SearchTool; if (!*patterns) { ErrorMessage("Error initializing the pattern table\n"); diff --git a/src/network_inspectors/appid/detector_plugins/detector_pop3.cc b/src/network_inspectors/appid/detector_plugins/detector_pop3.cc index b10748c01..10628baa0 100644 --- a/src/network_inspectors/appid/detector_plugins/detector_pop3.cc +++ b/src/network_inspectors/appid/detector_plugins/detector_pop3.cc @@ -219,7 +219,7 @@ void Pop3ClientDetector::do_custom_init() { if (cmd_matcher) delete cmd_matcher; - cmd_matcher = new SearchTool("ac_full", true); + cmd_matcher = new SearchTool; if ( !tcp_patterns.empty() ) { diff --git a/src/network_inspectors/appid/detector_plugins/dns_patterns.h b/src/network_inspectors/appid/detector_plugins/dns_patterns.h index afa33cf56..887d822c6 100644 --- a/src/network_inspectors/appid/detector_plugins/dns_patterns.h +++ b/src/network_inspectors/appid/detector_plugins/dns_patterns.h @@ -54,7 +54,7 @@ public: private: DnsHostPatternList* dns_host_pattern_list = nullptr; - snort::SearchTool dns_host_matcher = snort::SearchTool("ac_full", true); + snort::SearchTool dns_host_matcher = snort::SearchTool(); }; #endif diff --git a/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc b/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc index f94dd7afd..0e9dbb769 100644 --- a/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc +++ b/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc @@ -667,7 +667,7 @@ int HttpPatternMatchers::process_host_patterns(DetectorHTTPPatterns& patterns) int HttpPatternMatchers::process_chp_list(CHPListElement* chplist) { for (size_t i = 0; i < NUM_HTTP_FIELDS; i++) - chp_matchers[i] = new SearchTool("ac_full", true); + chp_matchers[i] = new SearchTool; for (CHPListElement* chpe = chplist; chpe; chpe = chpe->next) chp_matchers[chpe->chp_action.ptype]->add(chpe->chp_action.pattern, @@ -710,7 +710,7 @@ static FieldPattern http_field_patterns[] = static SearchTool* process_http_field_patterns(FieldPattern* patternList, size_t patternListCount) { - SearchTool* patternMatcher = new SearchTool("ac_full", true); + SearchTool* patternMatcher = new SearchTool; for (size_t i=0; i < patternListCount; i++) patternMatcher->add( (const char*)patternList[i].data, patternList[i].length, diff --git a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h index e541445d9..6a26de92b 100644 --- a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h +++ b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h @@ -270,8 +270,7 @@ class HttpPatternMatchers { public: HttpPatternMatchers() - : url_matcher("ac_full", true), client_agent_matcher("ac_full", true), via_matcher("ac_full", true), - content_type_matcher("ac_full", true) + : url_matcher(), client_agent_matcher(), via_matcher(), content_type_matcher() { } ~HttpPatternMatchers(); diff --git a/src/network_inspectors/appid/detector_plugins/ssl_patterns.h b/src/network_inspectors/appid/detector_plugins/ssl_patterns.h index ce33cf3ec..da9380eea 100644 --- a/src/network_inspectors/appid/detector_plugins/ssl_patterns.h +++ b/src/network_inspectors/appid/detector_plugins/ssl_patterns.h @@ -58,8 +58,8 @@ public: private: SslPatternList* cert_pattern_list = nullptr; SslPatternList* cname_pattern_list = nullptr; - snort::SearchTool ssl_host_matcher = snort::SearchTool("ac_full", true); - snort::SearchTool ssl_cname_matcher= snort::SearchTool("ac_full", true); + snort::SearchTool ssl_host_matcher = snort::SearchTool(); + snort::SearchTool ssl_cname_matcher= snort::SearchTool(); }; #endif diff --git a/src/network_inspectors/appid/service_plugins/service_mdns.cc b/src/network_inspectors/appid/service_plugins/service_mdns.cc index c05520087..0e9a22d0f 100644 --- a/src/network_inspectors/appid/service_plugins/service_mdns.cc +++ b/src/network_inspectors/appid/service_plugins/service_mdns.cc @@ -104,7 +104,7 @@ MdnsServiceDetector::MdnsServiceDetector(ServiceDiscovery* sd) { 5353, IpProtocol::UDP, false }, }; - matcher = new SearchTool("ac_full", true); + matcher = new SearchTool; for (unsigned i = 0; i < sizeof(patterns) / sizeof(*patterns); i++) matcher->add((const char*)patterns[i].pattern, patterns[i].length, &patterns[i]); matcher->prep(); diff --git a/src/parser/parse_rule.cc b/src/parser/parse_rule.cc index 68b815b44..d87acbdfb 100644 --- a/src/parser/parse_rule.cc +++ b/src/parser/parse_rule.cc @@ -88,6 +88,7 @@ static rule_count_t svcCnt; // dummy for now static bool s_ignore = false; // for skipping drop rules when not inline, etc. static bool s_capture = false; +static std::string s_type; static std::string s_body; struct SoRule @@ -168,28 +169,20 @@ static int FinishPortListRule( if ( !rtn->any_src_port() and !rtn->any_dst_port() ) prc->both++; + int src_cnt = rtn->any_src_port() ? 65535 : PortObjectPortCount(rtn->src_portobject); + int dst_cnt = rtn->any_dst_port() ? 65535 : PortObjectPortCount(rtn->dst_portobject); + /* If not an any-any rule test for port bleedover, if we are using a * single rule group, don't bother */ if ( !fp->get_single_rule_group() and !rtn->any_any_port() ) { - int dst_cnt = 0; - int src_cnt = 0; - - if ( !rtn->any_src_port() ) - { - src_cnt = PortObjectPortCount(rtn->src_portobject); - if (src_cnt >= fp->get_bleed_over_port_limit()) - large_port_group = 1; - } + if (src_cnt >= fp->get_bleed_over_port_limit()) + ++large_port_group; - if ( !rtn->any_dst_port() ) - { - dst_cnt = PortObjectPortCount(rtn->dst_portobject); - if (dst_cnt >= fp->get_bleed_over_port_limit()) - large_port_group = 1; - } + if (dst_cnt >= fp->get_bleed_over_port_limit()) + ++large_port_group; - if (large_port_group && fp->get_bleed_over_warnings()) + if (large_port_group == 2 && fp->get_bleed_over_warnings()) { LogMessage("***Bleedover Port Limit(%d) Exceeded for rule %u:%u " "(%d)ports: ", fp->get_bleed_over_port_limit(), @@ -211,7 +204,7 @@ static int FinishPortListRule( * any-any port rules... * If we have an any-any rule or a large port group or * were using a single rule group we make it an any-any rule. */ - if ( rtn->any_any_port() or large_port_group or fp->get_single_rule_group() ) + if ( rtn->any_any_port() or large_port_group == 2 or fp->get_single_rule_group() ) { if (otn->snort_protocol_id == SNORT_PROTO_IP) { @@ -231,8 +224,10 @@ static int FinishPortListRule( return 0; /* done */ } + bool both_dirs = false; + /* add rule index to dst table if we have a specific dst port or port list */ - if ( !rtn->any_dst_port() ) + if ( dst_cnt < fp->get_bleed_over_port_limit() and dst_cnt <= src_cnt ) { prc->dst++; @@ -248,7 +243,7 @@ static int FinishPortListRule( PortObjectAddRule(pox, otn->ruleIndex); /* if bidir, add this rule and port group to the src table */ - if (rtn->flags & RuleTreeNode::BIDIRECTIONAL) + if ( rtn->flags & RuleTreeNode::BIDIRECTIONAL ) { pox = PortTableFindInputPortObjectPorts(srcTable, rtn->dst_portobject); if ( !pox ) @@ -258,11 +253,12 @@ static int FinishPortListRule( } PortObjectAddRule(pox, otn->ruleIndex); + both_dirs = true; } } /* add rule index to src table if we have a specific src port or port list */ - if ( !rtn->any_src_port() ) + if ( src_cnt < fp->get_bleed_over_port_limit() and src_cnt < dst_cnt ) { prc->src++; PortObject* pox = PortTableFindInputPortObjectPorts(srcTable, rtn->src_portobject); @@ -275,7 +271,7 @@ static int FinishPortListRule( PortObjectAddRule(pox, otn->ruleIndex); /* if bidir, add this rule and port group to the dst table */ - if (rtn->flags & RuleTreeNode::BIDIRECTIONAL) + if ( !both_dirs and rtn->flags & RuleTreeNode::BIDIRECTIONAL ) { pox = PortTableFindInputPortObjectPorts(dstTable, rtn->src_portobject); if ( !pox ) @@ -953,6 +949,7 @@ void parse_rule_print() void parse_rule_type(SnortConfig* sc, const char* s, RuleTreeNode& rtn) { + s_type = s; rtn = RuleTreeNode(); if ( s_so_rule ) @@ -1122,7 +1119,11 @@ void parse_rule_opt_end(SnortConfig* sc, const char* key, OptTreeNode* otn) s_body += ";"; } RuleOptType type = OPT_TYPE_MAX; - IpsManager::option_end(sc, otn, otn->snort_protocol_id, key, type); + IpsOption* ips = IpsManager::option_end(sc, otn, otn->snort_protocol_id, key, type); + CursorActionType cat = ips ? ips->get_cursor_type() : CAT_NONE; + + if ( cat > CAT_ADJUST ) + otn->sticky_buf = get_pm_type(cat); if ( type != OPT_TYPE_META ) otn->num_detection_opts++; @@ -1161,7 +1162,6 @@ OptTreeNode* parse_rule_open(SnortConfig* sc, RuleTreeNode& rtn, bool stub) return otn; } -// FIXIT-H parse_rule_state needs parsing policy for reload static void parse_rule_state(SnortConfig* sc, const RuleTreeNode& rtn, OptTreeNode* otn) { if ( !otn->sigInfo.gid ) @@ -1180,6 +1180,7 @@ static void parse_rule_state(SnortConfig* sc, const RuleTreeNode& rtn, OptTreeNo }; RuleState state = { + s_type, rtn.action, otn->enable }; @@ -1328,9 +1329,6 @@ void parse_rule_close(SnortConfig* sc, RuleTreeNode& rtn, OptTreeNode* otn) OptFpList* fpl = AddOptFuncToList(OptListEnd, otn); fpl->type = RULE_OPTION_TYPE_LEAF_NODE; - validate_fast_pattern(otn); - OtnLookupAdd(sc->otn_map, otn); - if ( is_service_protocol(otn->snort_protocol_id) ) { // copy required because the call to add_service_to_otn can @@ -1339,6 +1337,9 @@ void parse_rule_close(SnortConfig* sc, RuleTreeNode& rtn, OptTreeNode* otn) add_service_to_otn(sc, otn, service.c_str()); } + validate_services(sc, otn); + OtnLookupAdd(sc->otn_map, otn); + if ( FinishPortListRule(sc->port_tables, new_rtn, otn, sc->fast_pattern_config) ) ParseError("Failed to finish a port list rule."); diff --git a/src/piglet_plugins/pp_ips_option_iface.cc b/src/piglet_plugins/pp_ips_option_iface.cc index 9cc5109d9..22592f393 100644 --- a/src/piglet_plugins/pp_ips_option_iface.cc +++ b/src/piglet_plugins/pp_ips_option_iface.cc @@ -51,15 +51,6 @@ static const luaL_Reg methods[] = return 1; } }, - { - "fp_research", - [](lua_State* L) - { - bool result = IpsOptionIface.get(L).fp_research(); - lua_pushboolean(L, result); - return 1; - } - }, { "get_cursor_type", [](lua_State* L) diff --git a/src/ports/port_group.h b/src/ports/port_group.h index a96993ddf..33eeac672 100644 --- a/src/ports/port_group.h +++ b/src/ports/port_group.h @@ -44,12 +44,19 @@ enum PmType PM_TYPE_HEADER, PM_TYPE_BODY, PM_TYPE_FILE, + PM_TYPE_RAW_KEY, + PM_TYPE_RAW_HEADER, + PM_TYPE_METHOD, + PM_TYPE_STAT_CODE, + PM_TYPE_STAT_MSG, + PM_TYPE_COOKIE, PM_TYPE_MAX }; const char* const pm_type_strings[PM_TYPE_MAX] = { - "packet", "alt", "key", "header", "body", "file" + "packet", "alt", "key", "header", "body", "file", "raw_key", "raw_header", + "method", "stat_code", "stat_msg", "cookie" }; struct RULE_NODE diff --git a/src/search_engines/search_tool.cc b/src/search_engines/search_tool.cc index e385a125a..e872c4330 100644 --- a/src/search_engines/search_tool.cc +++ b/src/search_engines/search_tool.cc @@ -22,6 +22,7 @@ #endif #include +#include #include "detection/fp_config.h" #include "framework/mpse.h" @@ -37,15 +38,33 @@ SearchTool::SearchTool(const char* method, bool dfa) { mpsegrp = new MpseGroup; - // When no method is passed in, a normal search engine mpse will be created - // with the search method the same as that configured for the fast pattern - // normal search method, and also an offload search engine mpse will be - // created with the search method the same as that configured for the fast - // pattern offload search method. If a method is passed in then an offload - // search engine will not be created + // When no method is passed in, a normal search engine mpse will be created with the search + // method the same as that configured for the fast pattern normal search method, and also an + // offload search engine mpse will be created with the search method the same as that + // configured for the fast pattern offload search method. If a method is passed in then an + // offload search engine will not be created + + // FIXIT-L the above is flawed. Offload should not depend on default. Also, offload only + // works in an inline, synchronous fashion for SearchTool so the overhead must be offset by + // the speed gain. Offload support should be removed from here for now. + + // FIXIT-L we force non-hyperscan to be ac_full since the other algorithms like ac_bnfa don't + // return all matches. This could be fixed by adding the match iteration loops like ac_full to + // ac_bnfa or removing that and updating SearchTool to do a trivial vector of matches. assert(conf); + if ( !method ) + { + method = conf->fast_pattern_config->get_search_method(); + + if ( strcmp(method, "hyperscan") ) + { + method = "ac_full"; + dfa = true; + } + } + if (mpsegrp->create_normal_mpse(conf, method)) { if( dfa ) diff --git a/src/search_engines/test/search_tool_test.cc b/src/search_engines/test/search_tool_test.cc index ecdb9fce0..6b7b13eea 100644 --- a/src/search_engines/test/search_tool_test.cc +++ b/src/search_engines/test/search_tool_test.cc @@ -29,6 +29,7 @@ #include +#include "detection/fp_config.h" #include "framework/base_api.h" #include "framework/mpse.h" #include "framework/mpse_batch.h" @@ -150,6 +151,9 @@ void Mpse::_search(MpseBatch& batch, MpseType mpse_type) } +const char* FastPatternConfig::get_search_method() +{ return "ac_bnfa"; } + extern const BaseApi* se_ac_bnfa; extern const BaseApi* se_ac_full; Mpse* mpse = nullptr; diff --git a/src/service_inspectors/http_inspect/http_inspect.cc b/src/service_inspectors/http_inspect/http_inspect.cc index 4ed650aae..de3cb008f 100644 --- a/src/service_inspectors/http_inspect/http_inspect.cc +++ b/src/service_inspectors/http_inspect/http_inspect.cc @@ -180,13 +180,37 @@ bool HttpInspect::get_buf(InspectionBuffer::Type ibt, Packet* p, InspectionBuffe { case InspectionBuffer::IBT_KEY: return get_buf(HTTP_BUFFER_URI, p, b); + case InspectionBuffer::IBT_HEADER: if (get_latest_is(p) == IS_TRAILER) return get_buf(HTTP_BUFFER_TRAILER, p, b); else return get_buf(HTTP_BUFFER_HEADER, p , b); + case InspectionBuffer::IBT_BODY: return get_buf(HTTP_BUFFER_CLIENT_BODY, p, b); + + case InspectionBuffer::IBT_RAW_KEY: + return get_buf(HTTP_BUFFER_RAW_URI, p , b); + + case InspectionBuffer::IBT_RAW_HEADER: + if (get_latest_is(p) == IS_TRAILER) + return get_buf(HTTP_BUFFER_RAW_TRAILER, p, b); + else + return get_buf(HTTP_BUFFER_RAW_HEADER, p , b); + + case InspectionBuffer::IBT_METHOD: + return get_buf(HTTP_BUFFER_METHOD, p , b); + + case InspectionBuffer::IBT_STAT_CODE: + return get_buf(HTTP_BUFFER_STAT_CODE, p , b); + + case InspectionBuffer::IBT_STAT_MSG: + return get_buf(HTTP_BUFFER_STAT_MSG, p , b); + + case InspectionBuffer::IBT_COOKIE: + return get_buf(HTTP_BUFFER_COOKIE, p , b); + default: return false; } @@ -227,6 +251,7 @@ bool HttpInspect::get_fp_buf(InspectionBuffer::Type ibt, Packet* p, InspectionBu switch (ibt) { case InspectionBuffer::IBT_KEY: + case InspectionBuffer::IBT_RAW_KEY: // Many rules targeting POST feature http_uri fast pattern with http_client_body. We // accept the performance hit of rerunning http_uri fast pattern with request body message // sections @@ -234,6 +259,7 @@ bool HttpInspect::get_fp_buf(InspectionBuffer::Type ibt, Packet* p, InspectionBu return false; break; case InspectionBuffer::IBT_HEADER: + case InspectionBuffer::IBT_RAW_HEADER: // http_header fast patterns for response bodies limited to first section if ((get_latest_src(p) == SRC_SERVER) && (get_latest_is(p) == IS_BODY)) return false; @@ -242,8 +268,21 @@ bool HttpInspect::get_fp_buf(InspectionBuffer::Type ibt, Packet* p, InspectionBu if ((get_latest_is(p) != IS_FIRST_BODY) && (get_latest_is(p) != IS_BODY)) return false; break; + case InspectionBuffer::IBT_METHOD: + if ((get_latest_src(p) != SRC_CLIENT) || (get_latest_is(p) != IS_HEADER)) + return false; + break; + case InspectionBuffer::IBT_STAT_CODE: + case InspectionBuffer::IBT_STAT_MSG: + if ((get_latest_src(p) != SRC_SERVER) || (get_latest_is(p) != IS_HEADER)) + return false; + break; + case InspectionBuffer::IBT_COOKIE: + if (get_latest_is(p) != IS_HEADER) + return false; + break; default: - return false; + break; } return get_buf(ibt, p, b); } diff --git a/src/service_inspectors/http_inspect/ips_http.cc b/src/service_inspectors/http_inspect/ips_http.cc index 67b602206..8815684c7 100644 --- a/src/service_inspectors/http_inspect/ips_http.cc +++ b/src/service_inspectors/http_inspect/ips_http.cc @@ -328,7 +328,7 @@ static const Parameter http_cookie_params[] = static Module* cookie_mod_ctor() { - return new HttpCursorModule(IPS_OPT, IPS_HELP, HTTP_BUFFER_COOKIE, CAT_SET_OTHER, PSI_COOKIE, + return new HttpCursorModule(IPS_OPT, IPS_HELP, HTTP_BUFFER_COOKIE, CAT_SET_COOKIE, PSI_COOKIE, http_cookie_params); } @@ -440,7 +440,7 @@ static const Parameter http_method_params[] = static Module* method_mod_ctor() { - return new HttpCursorModule(IPS_OPT, IPS_HELP, HTTP_BUFFER_METHOD, CAT_SET_OTHER, PSI_METHOD, + return new HttpCursorModule(IPS_OPT, IPS_HELP, HTTP_BUFFER_METHOD, CAT_SET_METHOD, PSI_METHOD, http_method_params); } @@ -635,7 +635,7 @@ static const Parameter http_raw_header_params[] = static Module* raw_header_mod_ctor() { - return new HttpCursorModule(IPS_OPT, IPS_HELP, HTTP_BUFFER_RAW_HEADER, CAT_SET_OTHER, + return new HttpCursorModule(IPS_OPT, IPS_HELP, HTTP_BUFFER_RAW_HEADER, CAT_SET_RAW_HEADER, PSI_RAW_HEADER, http_raw_header_params); } @@ -850,7 +850,7 @@ static const Parameter http_raw_uri_params[] = static Module* raw_uri_mod_ctor() { - return new HttpCursorModule(IPS_OPT, IPS_HELP, HTTP_BUFFER_RAW_URI, CAT_SET_OTHER, + return new HttpCursorModule(IPS_OPT, IPS_HELP, HTTP_BUFFER_RAW_URI, CAT_SET_RAW_KEY, PSI_RAW_URI, http_raw_uri_params); } @@ -899,7 +899,7 @@ static const Parameter http_stat_code_params[] = static Module* stat_code_mod_ctor() { - return new HttpCursorModule(IPS_OPT, IPS_HELP, HTTP_BUFFER_STAT_CODE, CAT_SET_OTHER, + return new HttpCursorModule(IPS_OPT, IPS_HELP, HTTP_BUFFER_STAT_CODE, CAT_SET_STAT_CODE, PSI_STAT_CODE, http_stat_code_params); } @@ -948,7 +948,7 @@ static const Parameter http_stat_msg_params[] = static Module* stat_msg_mod_ctor() { - return new HttpCursorModule(IPS_OPT, IPS_HELP, HTTP_BUFFER_STAT_MSG, CAT_SET_OTHER, + return new HttpCursorModule(IPS_OPT, IPS_HELP, HTTP_BUFFER_STAT_MSG, CAT_SET_STAT_MSG, PSI_STAT_MSG, http_stat_msg_params); } diff --git a/src/service_inspectors/sip/ips_sip.cc b/src/service_inspectors/sip/ips_sip.cc index 2454c9f8a..ca0050800 100644 --- a/src/service_inspectors/sip/ips_sip.cc +++ b/src/service_inspectors/sip/ips_sip.cc @@ -83,7 +83,7 @@ class SipIpsOption : public IpsOption { public: SipIpsOption( - const char* s, SipIdx psi, CursorActionType c = CAT_SET_OTHER) : + const char* s, SipIdx psi, CursorActionType c) : IpsOption(s, RULE_OPTION_TYPE_BUFFER_SET) { key = s; cat = c; idx = psi; } diff --git a/src/utils/stats.cc b/src/utils/stats.cc index b83d5a72b..f5bd5a767 100644 --- a/src/utils/stats.cc +++ b/src/utils/stats.cc @@ -182,6 +182,12 @@ const PegInfo pc_names[] = { CountType::SUM, "header_searches", "fast pattern searches in header buffer" }, { CountType::SUM, "body_searches", "fast pattern searches in body buffer" }, { CountType::SUM, "file_searches", "fast pattern searches in file buffer" }, + { CountType::SUM, "raw_key_searches", "fast pattern searches in raw key buffer" }, + { CountType::SUM, "raw_header_searches", "fast pattern searches in raw header buffer" }, + { CountType::SUM, "method_searches", "fast pattern searches in method buffer" }, + { CountType::SUM, "stat_code_searches", "fast pattern searches in status code buffer" }, + { CountType::SUM, "stat_msg_searches", "fast pattern searches in status message buffer" }, + { CountType::SUM, "cookie_searches", "fast pattern searches in cookie buffer" }, { CountType::SUM, "offloads", "fast pattern searches that were offloaded" }, { CountType::SUM, "alerts", "alerts not including IP reputation" }, { CountType::SUM, "total_alerts", "alerts including IP reputation" }, diff --git a/src/utils/stats.h b/src/utils/stats.h index 38a5b9eef..a541fd70a 100644 --- a/src/utils/stats.h +++ b/src/utils/stats.h @@ -44,6 +44,12 @@ struct PacketCount PegCount header_searches; PegCount body_searches; PegCount file_searches; + PegCount raw_key_searches; + PegCount raw_header_searches; + PegCount method_searches; + PegCount stat_code_searches; + PegCount stat_msg_searches; + PegCount cookie_searches; PegCount offloads; PegCount alert_pkts; PegCount total_alert_pkts;