From: Russ Combs (rucombs) Date: Fri, 1 Apr 2022 20:39:13 +0000 (+0000) Subject: Pull request #3324: Ips bag X-Git-Tag: 3.1.27.0~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=00aaedfc36c8449827cb15e077e93353f2e5ccd6;p=thirdparty%2Fsnort3.git Pull request #3324: Ips bag Merge in SNORT/snort3 from ~RUCOMBS/snort3:ips_bag to master Squashed commit of the following: commit 7f28f5c4cbda2834d6f50ba43eb45a0d34b57abd Author: russ Date: Sun Mar 27 14:03:38 2022 -0400 hyperscan: ensure adequate scratch when deserializing commit 0d4f03134ec1d17101774a9080a3e86dc7cf7a3c Author: russ Date: Sat Mar 26 22:26:31 2022 -0400 detection: skip match deduplication for hyperscan commit 2b5fb8dce61bb23cf190200d6b99419b24bea1f1 Author: russ Date: Sat Mar 26 15:55:18 2022 -0400 search_engines: ensure SearchTool with hyperscan gets multi-match mode commit f50810182e6f1c2900afa9bf7c9a5c1a11f0ec84 Author: russ Date: Sat Mar 26 14:39:10 2022 -0400 search_engines: add and refactor unit tests commit a7af03c532dce85a2d9eae6d3ec89e36f75e439a Author: russ Date: Sat Mar 26 10:08:56 2022 -0400 ac_full: refactor api access commit 8c29afb0e0cac16aa360b659281b7dcaa012b090 Author: russ Date: Fri Mar 25 23:29:11 2022 -0400 search_engine: always build ac_full since it is a hard default case SearchTool will use hyperscan if configured else ac_full since that is the only builtin MPSE that returns all matches. commit 96f2c0943fc35638f2ee1e611c4e76ba994d0ceb Author: russ Date: Fri Mar 25 21:08:59 2022 -0400 search_engine: remove search_optimize parameter (always true) Also remove broken support for offload from SearchTool. commit 01271621d4af3bc5dd97ce7fab38887774b7675e Author: russ Date: Thu Mar 24 20:33:25 2022 -0400 detection: do not check ips policy when builtin events are queued Builtin events are for now only checked for the current policy when dequeued. This allows the policy to be changed after inspection, which is how Snort 2 does it. This is flawed however and can be fixed by pairing an ips policy with each nap or just including the builtin rules and state stubs directly in the nap. commit 95e6beb3ff36ac35d481265b690bb19e88ea9f64 Author: russ Date: Thu Mar 24 12:55:54 2022 -0400 detection: minor refactoring of rule header access commit 676606491ee0f74675deb8df59a0986ffef1e25f Author: russ Date: Thu Mar 24 10:21:36 2022 -0400 rate_filter: move to inspection policy commit 76716c997dadb485e3e2bf4d3011196c61db0821 Author: russ Date: Sat Mar 19 09:40:51 2022 -0400 alerts: remove obsolete stateful parameter commit 4bcc7ca6fa19963d21768deee31692453a844322 Author: russ Date: Fri Mar 18 10:17:31 2022 -0400 ac_full: remove cruft commit 4cb95706bd2e13085ee7fe4a158f33f1e35804e3 Author: russ Date: Fri Mar 18 10:00:48 2022 -0400 search_engines: remove the legacy ac_sparse_bands algorithm commit 57b19a41e7125701e75ea017630a5eeef9f6ecc5 Author: russ Date: Fri Mar 18 09:53:03 2022 -0400 search_engines: remove the legacy ac_sparse algorithm commit 36b258d99f0b32f7d46f782bce76ca740f320cfe Author: russ Date: Fri Mar 18 09:44:09 2022 -0400 search_engines: remove the legacy ac_banded algorithm commit 29720b96a3b54702119dfa98bcc1d8b0b82b7c8f Author: russ Date: Fri Mar 18 09:33:18 2022 -0400 search_engines: remove the legacy ac_std algorithm commit 5af3cd8074287bc865563f2e26be17df64fa4046 Author: russ Date: Sun Mar 13 00:12:12 2022 -0500 detection: override match queue limit for offload commit 00183d5cc1cb7802e3f2f9a5a9becc3319f76c0f Author: russ Date: Sat Mar 12 12:47:59 2022 -0500 ac_std: fix case translation buffer size commit 20ceb4956bd6eaa2b6165723df7dd833a044f957 Author: russ Date: Fri Mar 11 19:49:22 2022 -0500 search_engine: remove obsolete warning on max_pattern_len change commit be971a82799a9da367f0867970b9a20615f327ee Author: russ Date: Fri Mar 11 15:03:54 2022 -0500 search_engine: fix .debug = true output ... and 7 more commits --- diff --git a/daqs/daq_hext.c b/daqs/daq_hext.c index 67655d1e2..44644d37a 100644 --- a/daqs/daq_hext.c +++ b/daqs/daq_hext.c @@ -523,7 +523,7 @@ static DAQ_RecvStatus hext_read_message(HextContext* hc, HextMsgDesc* desc) static int hext_setup(HextContext* hc) { - if (!strcmp(hc->filename, "tty")) + if (!strcmp(hc->filename, "-")) { hc->fp = stdin; } diff --git a/src/actions/actions.cc b/src/actions/actions.cc index ec3df45f7..87b74e583 100644 --- a/src/actions/actions.cc +++ b/src/actions/actions.cc @@ -22,6 +22,8 @@ #include "actions.h" +#include + #include "detection/detect.h" #include "managers/action_manager.h" #include "parser/parser.h" @@ -36,16 +38,15 @@ void Actions::pass() void Actions::log(Packet* p, const OptTreeNode* otn) { - RuleTreeNode* rtn = getRuntimeRtnFromOtn(otn); + RuleTreeNode* rtn = getRtnFromOtn(otn); + assert(rtn); CallLogFuncs(p, otn, rtn->listhead); } void Actions::alert(Packet* p, const OptTreeNode* otn) { - RuleTreeNode* rtn = getRuntimeRtnFromOtn(otn); - - if (rtn == nullptr) - return; + RuleTreeNode* rtn = getRtnFromOtn(otn); + assert(rtn); /* Call OptTreeNode specific output functions */ if (otn->outputFuncs) diff --git a/src/detection/detection_engine.cc b/src/detection/detection_engine.cc index c0e0e6ddf..dd1a80b56 100644 --- a/src/detection/detection_engine.cc +++ b/src/detection/detection_engine.cc @@ -665,7 +665,7 @@ int DetectionEngine::queue_event(const OptTreeNode* otn) int DetectionEngine::queue_event(unsigned gid, unsigned sid) { - OptTreeNode* otn = GetOTN(gid, sid); + OptTreeNode* otn = OtnLookup(SnortConfig::get_conf()->otn_map, gid, sid); if ( !otn ) return 0; diff --git a/src/detection/detection_options.cc b/src/detection/detection_options.cc index 887f678c2..81a515d60 100644 --- a/src/detection/detection_options.cc +++ b/src/detection/detection_options.cc @@ -315,8 +315,10 @@ void print_option_tree(detection_option_tree_node_t* node, int level) opt = buf; } - debug_logf(detection_trace, TRACE_OPTION_TREE, nullptr, "%3d %3d %p %*s\n", - level, node->num_children, node->option_data, (int)(level + strlen(opt)), opt); + const char* srtn = node->otn ? " (rtn)" : ""; + + debug_logf(detection_trace, TRACE_OPTION_TREE, nullptr, "%3d %3d %p %*s%s\n", + level+1, node->num_children, node->option_data, (int)(level + strlen(opt)), opt, srtn); for ( int i=0; inum_children; i++ ) print_option_tree(node->children[i], level+1); @@ -439,7 +441,7 @@ int detection_option_node_evaluate( } } - if ( !fp_eval_rtn(getRuntimeRtnFromOtn(node->otn), p, check_ports) ) + if ( !fp_eval_rtn(getRtnFromOtn(node->otn), p, check_ports) ) break; } diff --git a/src/detection/fp_config.cc b/src/detection/fp_config.cc index 32c56a0b4..6972d0c53 100644 --- a/src/detection/fp_config.cc +++ b/src/detection/fp_config.cc @@ -53,6 +53,12 @@ bool FastPatternConfig::set_search_method(const char* method) return false; search_api = api; + + // FIXIT-L query search_api capabilities when API is redone + // same for offload + if ( !strcmp(method, "hyperscan") ) + dedup = false; + return true; } @@ -72,15 +78,15 @@ bool FastPatternConfig::set_offload_search_method(const char* method) return false; offload_search_api = api; + + if ( !dedup and !strcmp(method, "hyperscan") ) + dedup = false; + return true; } void FastPatternConfig::set_max_pattern_len(unsigned int max_len) { - if (max_pattern_len != 0) - ParseWarning(WARN_CONF, "maximum pattern length redefined from %d to %u.\n", - max_pattern_len, max_len); - max_pattern_len = max_len; } diff --git a/src/detection/fp_config.h b/src/detection/fp_config.h index 143736063..36029b61a 100644 --- a/src/detection/fp_config.h +++ b/src/detection/fp_config.h @@ -124,12 +124,6 @@ public: const std::string& get_rule_db_dir() const { return rule_db_dir; } - void set_search_opt(bool flag) - { search_opt = flag; } - - bool get_search_opt() const - { return search_opt; } - bool set_search_method(const char*); const char* get_search_method(); @@ -151,6 +145,9 @@ public: unsigned set_max(unsigned bytes); + bool deduplicate() const + { return dedup; } + private: const snort::MpseApi* search_api = nullptr; const snort::MpseApi* offload_search_api = nullptr; @@ -159,7 +156,7 @@ private: bool split_any_any = false; bool debug_print_fast_pattern = false; bool debug = false; - bool search_opt = false; + bool dedup = true; unsigned max_queue_events = 5; unsigned bleedover_port_limit = 1024; @@ -168,7 +165,7 @@ private: unsigned queue_limit = 0; int portlists_flags = 0; - int num_patterns_truncated = 0; // due to max_pattern_len + unsigned num_patterns_truncated = 0; // due to max_pattern_len std::string rule_db_dir; }; diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc index 75c7434fc..e0bb5d7ed 100644 --- a/src/detection/fp_create.cc +++ b/src/detection/fp_create.cc @@ -65,6 +65,7 @@ using namespace std; static unsigned mpse_count = 0; static unsigned offload_mpse_count = 0; +static unsigned fp_only = 0; static const char* s_group = ""; static void fpDeletePMX(void* data); @@ -76,27 +77,6 @@ static void print_nfp_info(const char*, OptTreeNode*); static void print_fp_info(const char*, const OptTreeNode*, const PatternMatchData*, const char* pattern, unsigned pattern_length); -static int finalize_detection_option_tree(SnortConfig* sc, detection_option_tree_root_t* root) -{ - if ( !root ) - return -1; - - for ( int i=0; inum_children; i++ ) - { - detection_option_tree_node_t* node = root->children[i]; - - if ( void* dup_node = add_detection_option_tree(sc, node) ) - { - // FIXIT-L delete dup_node and keep original? - free_detection_option_tree(node); - root->children[i] = (detection_option_tree_node_t*)dup_node; - } - print_option_tree(root->children[i], 0); - } - - return 0; -} - static OptTreeNode* fixup_tree( detection_option_tree_node_t* dot, bool branched, unsigned contents) { @@ -127,19 +107,30 @@ static OptTreeNode* fixup_tree( return nullptr; } -static void fixup_trees(SnortConfig* sc) +static int finalize_detection_option_tree(SnortConfig* sc, detection_option_tree_root_t* root) { - if ( !sc->detection_option_tree_hash_table ) - return; - - HashNode* hn = sc->detection_option_tree_hash_table->find_first_node(); + if ( !root ) + return -1; - while ( hn ) + for ( int i=0; inum_children; i++ ) { - detection_option_tree_node_t* node = (detection_option_tree_node_t*)hn->data; - fixup_tree(node, true, 0); - hn = sc->detection_option_tree_hash_table->find_next_node(); + detection_option_tree_node_t* node = root->children[i]; + + if ( void* dup_node = add_detection_option_tree(sc, node) ) + { + // FIXIT-L delete dup_node and keep original? + free_detection_option_tree(node); + root->children[i] = (detection_option_tree_node_t*)dup_node; + } + fixup_tree(root->children[i], true, 0); + + debug_logf(detection_trace, TRACE_OPTION_TREE, nullptr, "%3d %3d %p %4s\n", + 0, root->num_children, (void*)root, "root" ); + + print_option_tree(root->children[i], 0); } + + return 0; } static bool new_sig(int num_children, detection_option_tree_node_t** nodes, OptTreeNode* otn) @@ -452,7 +443,7 @@ static int fpFinishRuleGroupRule( pmx->pmd = pmd; Mpse::PatternDescriptor desc( - pmd->is_no_case(), pmd->is_negated(), pmd->is_literal(), pmd->mpse_flags); + pmd->is_no_case(), pmd->is_negated(), pmd->is_literal(), false, pmd->mpse_flags); mpse->add_pattern((const uint8_t*)pattern, pattern_length, desc, pmx); @@ -481,10 +472,6 @@ static int fpFinishRuleGroup(SnortConfig* sc, RuleGroup* pg, FastPatternConfig* if (pg->mpsegrp[i]->normal_mpse->get_pattern_count() != 0) { queue_mpse(pg->mpsegrp[i]->normal_mpse); - - if (fp->get_debug_mode()) - pg->mpsegrp[i]->normal_mpse->print_info(); - rules = 1; } else @@ -498,10 +485,6 @@ static int fpFinishRuleGroup(SnortConfig* sc, RuleGroup* pg, FastPatternConfig* if (pg->mpsegrp[i]->offload_mpse->get_pattern_count() != 0) { queue_mpse(pg->mpsegrp[i]->offload_mpse); - - if (fp->get_debug_mode()) - pg->mpsegrp[i]->offload_mpse->print_info(); - rules = 1; } else @@ -623,8 +606,6 @@ static int fpAddRuleGroupRule( } mpse_count++; - if ( fp->get_search_opt() ) - pg->mpsegrp[main_pmd->pm_type]->normal_mpse->set_opt(1); } if (add_to_offload) @@ -649,8 +630,6 @@ static int fpAddRuleGroupRule( } offload_mpse_count++; - if ( fp->get_search_opt() ) - pg->mpsegrp[main_pmd->pm_type]->offload_mpse->set_opt(1); } } @@ -671,7 +650,10 @@ static int fpAddRuleGroupRule( otn->longestPatternLen = main_pmd->pattern_size; if ( make_fast_pattern_only(ofp, main_pmd) ) + { otn->normal_fp_only = ofp; + fp_only++; + } // Add Alternative patterns for (auto p : pmv) @@ -694,7 +676,10 @@ static int fpAddRuleGroupRule( otn->longestPatternLen = ol_pmd->pattern_size; if ( make_fast_pattern_only(ofp_ol, ol_pmd) ) + { otn->offload_fp_only = ofp_ol; + fp_only++; + } // Add Alternative patterns for (auto p : pmv_ol) @@ -951,12 +936,15 @@ static int fpGetFinalPattern( * is taken care of during parsing */ assert(pmd->fp_offset + pmd->fp_length <= pmd->pattern_size); pattern = pmd->pattern_buf + pmd->fp_offset; - bytes = pmd->fp_length ? pmd->fp_length : pmd->pattern_size - pmd->fp_length; + bytes = pmd->fp_length ? pmd->fp_length : pmd->pattern_size - pmd->fp_offset; } ret_pattern = pattern; ret_bytes = fp->set_max(bytes); + if ( ret_bytes < pmd->pattern_size ) + pmd->fp_length = ret_bytes; + return 0; } @@ -1315,7 +1303,7 @@ static void fpCreateServiceMapRuleGroups(SnortConfig* sc) } /* - * Print the rule gid:sid based onm the otn list + * Print the rule gid:sid based on the otn list */ static void fpPrintRuleList(SF_LIST* list) { @@ -1576,6 +1564,7 @@ int fpCreateFastPacketDetection(SnortConfig* sc) mpse_count = 0; offload_mpse_count = 0; + fp_only = 0; MpseManager::start_search_engine(fp->get_search_api()); @@ -1616,8 +1605,6 @@ int fpCreateFastPacketDetection(SnortConfig* sc) if ( c != expected ) ParseError("Failed to compile %u search engines", expected - c); - - fixup_trees(sc); } fp_print_port_groups(port_tables); @@ -1639,6 +1626,7 @@ int fpCreateFastPacketDetection(SnortConfig* sc) } LogCount("truncated patterns", fp->get_num_patterns_truncated()); + LogCount("fast pattern only", fp_only); LogCount("mpse_loaded", mpse_loaded); LogCount("mpse_dumped", mpse_dumped); diff --git a/src/detection/fp_detect.cc b/src/detection/fp_detect.cc index c53f84bc8..0cf5fb35d 100644 --- a/src/detection/fp_detect.cc +++ b/src/detection/fp_detect.cc @@ -182,19 +182,6 @@ int fpLogEvent(const RuleTreeNode* rtn, const OptTreeNode* otn, Packet* p) p->packet_flags &= ~PKT_STATELESS; } - if ((p->packet_flags & PKT_STREAM_UNEST_UNI) && - p->context->conf->assure_established() && - (!(p->packet_flags & PKT_REBUILT_STREAM)) && - !otn->stateless() ) - { - // We still want to drop packets that are drop rules. - // We just don't want to see the alert. - IpsAction * act = get_ips_policy()->action[rtn->action]; - act->exec(p); - fpLogOther(p, rtn, otn, rtn->action); - return 1; - } - // perform rate filtering tests - impacts action taken rateAction = RateFilter_Test(otn, p); override = ( rateAction >= Actions::get_max_types() ); @@ -290,7 +277,7 @@ int fpLogEvent(const RuleTreeNode* rtn, const OptTreeNode* otn, Packet* p) */ int fpAddMatch(OtnxMatchData* omd, const OptTreeNode* otn) { - RuleTreeNode* rtn = getRuntimeRtnFromOtn(otn); + RuleTreeNode* rtn = getRtnFromOtn(otn); unsigned evalIndex = rtn->listhead->ruleListNode->evalIndex; const SnortConfig* sc = SnortConfig::get_conf(); @@ -667,9 +654,14 @@ static inline int fpFinalSelectEvent(OtnxMatchData* omd, Packet* p) for (unsigned j = 0; j < omd->matchInfo[i].iMatchCount; j++) { const OptTreeNode* otn = omd->matchInfo[i].MatchArray[j]; + assert(otn); + RuleTreeNode* rtn = getRtnFromOtn(otn); - if ( otn && rtn && ( p->packet_flags & PKT_PASS_RULE ) ) + if ( !rtn ) + continue; + + if ( p->packet_flags & PKT_PASS_RULE ) { /* Already acted on rules, so just don't act on anymore */ if ( tcnt > 0 ) @@ -686,7 +678,10 @@ static inline int fpFinalSelectEvent(OtnxMatchData* omd, Packet* p) } } - if ( otn && !fpSessionAlerted(p, otn) ) + if ( !otn ) + continue; + + if ( !fpSessionAlerted(p, otn) ) { if ( DetectionEngine::queue_event(otn) ) pc.queue_limit++; @@ -710,7 +705,7 @@ static inline int fpFinalSelectEvent(OtnxMatchData* omd, Packet* p) } /* only log/count one pass */ - if ( otn && rtn && ( p->packet_flags & PKT_PASS_RULE ) ) + if ( p->packet_flags & PKT_PASS_RULE ) return 1; } } @@ -733,7 +728,11 @@ public: using MatchStore = std::vector; public: - MpseStash(unsigned limit) : max(limit) { } + MpseStash(const FastPatternConfig& fp) + { + max = fp.get_queue_limit(); + dedup = fp.deduplicate(); + } // this is done in the offload thread bool push(void* user, void* tree, int index, void* context, void* list); @@ -750,6 +749,7 @@ private: unsigned inserts = 0; unsigned max; + bool dedup; MatchStore queue; MatchStore defer; @@ -761,12 +761,15 @@ bool MpseStash::push(void* user, void* tree, int index, void* context, void* lis bool checker = !root or root->otn->checks_flowbits(); MatchStore& store = checker ? defer : queue; - for ( auto it = store.rbegin(); it != store.rend(); it++ ) + if ( dedup ) { - if ( tree == (*it).tree ) + for ( auto it = store.rbegin(); it != store.rend(); it++ ) { - inserts++; - return true; + if ( tree == (*it).tree ) + { + inserts++; + return true; + } } } @@ -776,7 +779,7 @@ bool MpseStash::push(void* user, void* tree, int index, void* context, void* lis return false; } - if ( !checker and qmax == queue.size() ) + if ( !checker and qmax == queue.size() and is_packet_thread() ) { Profile rule_profile(rulePerfStats); process((IpsContext*)context, queue); @@ -830,7 +833,7 @@ void MpseStash::process(IpsContext* context, MatchStore& store) void fp_set_context(IpsContext& c) { FastPatternConfig* fp = c.conf->fast_pattern_config; - c.stash = new MpseStash(fp->get_queue_limit()); + c.stash = new MpseStash(*fp); c.otnx = (OtnxMatchData*)snort_calloc(sizeof(OtnxMatchData)); c.otnx->matchInfo = (MatchInfo*)snort_calloc(MAX_NUM_RULE_TYPES, sizeof(MatchInfo)); c.context_num = 0; diff --git a/src/detection/fp_utils.cc b/src/detection/fp_utils.cc index 702e38e7b..e262db522 100644 --- a/src/detection/fp_utils.cc +++ b/src/detection/fp_utils.cc @@ -45,6 +45,7 @@ #include "treenodes.h" #include "utils/util.h" +#include "fp_config.h" #include "service_map.h" #ifdef UNIT_TEST @@ -143,7 +144,7 @@ static const char* get_service(const char* opt) { if ( !strncmp(opt, "http_", 5) ) return "http"; - + if ( !strncmp(opt, "js_data", 7) ) return "http"; @@ -538,6 +539,9 @@ PatternMatchVector get_fp_content( bool make_fast_pattern_only(const OptFpList* ofp, const PatternMatchData* pmd) { + if ( pmd->fp_offset or (pmd->fp_length and pmd->pattern_size != pmd->fp_length) ) + return false; + // FIXIT-L no_case consideration is mpse specific, delegate if ( !pmd->is_relative() and !pmd->is_negated() and !pmd->offset and !pmd->depth and pmd->is_no_case() ) @@ -588,7 +592,12 @@ static void compile_mpse(SnortConfig* sc, unsigned id, unsigned* count) while ( Mpse* m = get_mpse() ) { if ( !m->prep_patterns(sc) ) + { + if ( sc->fast_pattern_config->get_debug_mode() ) + m->print_info(); + c++; + } } std::lock_guard lock(s_mutex); *count += c; diff --git a/src/detection/rtn_checks.cc b/src/detection/rtn_checks.cc index 01931458f..3984352d0 100644 --- a/src/detection/rtn_checks.cc +++ b/src/detection/rtn_checks.cc @@ -258,8 +258,3 @@ int RuleListEnd(Packet*, RuleTreeNode*, RuleFpList*, int) return 1; } -int OptListEnd(void*, Cursor&, Packet*) -{ - return (int)IpsOption::MATCH; -} - diff --git a/src/detection/rtn_checks.h b/src/detection/rtn_checks.h index f64b4f89b..da5168e1e 100644 --- a/src/detection/rtn_checks.h +++ b/src/detection/rtn_checks.h @@ -30,7 +30,6 @@ struct RuleTreeNode; // parsing int RuleListEnd(snort::Packet*, RuleTreeNode*, RuleFpList*, int); -int OptListEnd(void* option_data, class Cursor&, snort::Packet*); // detection int CheckBidirectional(snort::Packet*, RuleTreeNode*, RuleFpList*, int); diff --git a/src/detection/signature.cc b/src/detection/signature.cc index 2ba2f3d33..d798ecb16 100644 --- a/src/detection/signature.cc +++ b/src/detection/signature.cc @@ -208,23 +208,6 @@ OptTreeNode* OtnLookup(GHash* otn_map, uint32_t gid, uint32_t sid) return otn; } -OptTreeNode* GetOTN(uint32_t gid, uint32_t sid) -{ - OptTreeNode* otn = OtnLookup(SnortConfig::get_conf()->otn_map, gid, sid); - - if ( !otn ) - return nullptr; - - if ( !getRtnFromOtn(otn) ) - { - // If not configured to autogenerate and there isn't an RTN, meaning - // this rule isn't in the current policy, return nullptr. - return nullptr; - } - - return otn; -} - void OtnLookupFree(GHash* otn_map) { if ( otn_map ) diff --git a/src/detection/signature.h b/src/detection/signature.h index 43821cf06..98d8eb739 100644 --- a/src/detection/signature.h +++ b/src/detection/signature.h @@ -115,8 +115,6 @@ OptTreeNode* OtnLookup(snort::GHash*, uint32_t gid, uint32_t sid); void OtnLookupFree(snort::GHash*); void OtnRemove(snort::GHash*, OptTreeNode*); -OptTreeNode* GetOTN(uint32_t gid, uint32_t sid); - void dump_msg_map(const snort::SnortConfig*); void dump_rule_deps(const snort::SnortConfig*); void dump_rule_meta(const snort::SnortConfig*); diff --git a/src/detection/tag.cc b/src/detection/tag.cc index 81730b8cc..7592d210f 100644 --- a/src/detection/tag.cc +++ b/src/detection/tag.cc @@ -629,7 +629,7 @@ void SetTags(const Packet* p, const OptTreeNode* otn, uint16_t event_id) { if (otn->tag->tag_type != 0) { - RuleTreeNode* rtn = getRuntimeRtnFromOtn(otn); + RuleTreeNode* rtn = getRtnFromOtn(otn); void* log_list = rtn ? rtn->listhead : nullptr; switch (otn->tag->tag_type) diff --git a/src/detection/treenodes.cc b/src/detection/treenodes.cc index 63cdabad3..43c0a081f 100644 --- a/src/detection/treenodes.cc +++ b/src/detection/treenodes.cc @@ -70,22 +70,6 @@ void otn_trigger_actions(const OptTreeNode* otn, Packet* p) // rule FOO //------------------------------------------------------------------------- -void* get_rule_type_data(OptTreeNode* otn, const char* name) -{ - OptFpList* fpl = otn->opt_func; - - while ( fpl ) - { - if ( fpl->ips_opt ) - { - if ( !strcmp(fpl->ips_opt->get_name(), name) ) - return fpl->ips_opt; - } - fpl = fpl->next; - } - return nullptr; -} - namespace snort { bool otn_has_plugin(OptTreeNode* otn, const char* name) diff --git a/src/detection/treenodes.h b/src/detection/treenodes.h index 6304cbd28..b98c5fcd1 100644 --- a/src/detection/treenodes.h +++ b/src/detection/treenodes.h @@ -261,8 +261,6 @@ struct OptTreeNode typedef int (* RuleOptEvalFunc)(void*, Cursor&, snort::Packet*); OptFpList* AddOptFuncToList(RuleOptEvalFunc, OptTreeNode*); -void* get_rule_type_data(OptTreeNode*, const char* name); - namespace snort { SO_PUBLIC bool otn_has_plugin(OptTreeNode* otn, const char* name); diff --git a/src/file_api/file_service.cc b/src/file_api/file_service.cc index 1a269b986..47480e341 100644 --- a/src/file_api/file_service.cc +++ b/src/file_api/file_service.cc @@ -59,11 +59,9 @@ void FileService::init() FileFlows::init(); } -void FileService::post_init(const SnortConfig* sc) +void FileService::post_init() { - SearchTool::set_conf(sc); MimeSession::init(); - SearchTool::set_conf(nullptr); const FileConfig* const conf = get_file_config(); diff --git a/src/file_api/file_service.h b/src/file_api/file_service.h index c3287dfe2..d369cf132 100644 --- a/src/file_api/file_service.h +++ b/src/file_api/file_service.h @@ -40,7 +40,7 @@ public: static void init(); // Called after permission is dropped - static void post_init(const SnortConfig*); + static void post_init(); // Called during reload static void verify_reload(const SnortConfig*); diff --git a/src/filters/rate_filter.cc b/src/filters/rate_filter.cc index e9d7f46f5..ef62fe5b8 100644 --- a/src/filters/rate_filter.cc +++ b/src/filters/rate_filter.cc @@ -133,12 +133,12 @@ int RateFilter_Test(const OptTreeNode* otn, Packet* p) // events and these require: src -> client, dst -> server. if ( p->is_from_server() ) { - return SFRF_TestThreshold(rfc, gid, sid, get_network_policy()->policy_id, + return SFRF_TestThreshold(rfc, gid, sid, get_inspection_policy()->policy_id, dip, sip, p->pkth->ts.tv_sec, SFRF_COUNT_INCREMENT); } } - return SFRF_TestThreshold(rfc, gid, sid, get_network_policy()->policy_id, + return SFRF_TestThreshold(rfc, gid, sid, get_inspection_policy()->policy_id, sip, dip, p->pkth->ts.tv_sec, SFRF_COUNT_INCREMENT); } diff --git a/src/filters/sfrf.cc b/src/filters/sfrf.cc index 892e310ec..11b556fa2 100644 --- a/src/filters/sfrf.cc +++ b/src/filters/sfrf.cc @@ -232,7 +232,7 @@ int SFRF_ConfigAdd(SnortConfig*, RateFilterConfig* rf_config, tSFRFConfigNode* c tSFRFConfigNode* pNewConfigNode; tSFRFGenHashKey key = { 0,0 }; - PolicyId policy_id = get_ips_policy()->policy_id; + PolicyId policy_id = get_inspection_policy()->policy_id; if ((rf_config == nullptr) || (cfgNode == nullptr)) return -1; @@ -782,7 +782,7 @@ static tSFRFTrackingNode* _getSFRFTrackingNode(const SfIp* ip, unsigned tid, tim /* Setup key */ key.ip = *(ip); key.tid = tid; - key.policyId = get_ips_policy()->policy_id; + key.policyId = get_inspection_policy()->policy_id; key.padding = 0; // Check for any Permanent sid objects for this gid or add this one ... diff --git a/src/framework/cursor.cc b/src/framework/cursor.cc index c83070043..8fc6e4ca6 100644 --- a/src/framework/cursor.cc +++ b/src/framework/cursor.cc @@ -96,7 +96,6 @@ void Cursor::reset(Packet* p) { InspectionBuffer buf; - // FIXIT-M should this be converted to get_fp_buf()? if ( p->flow and p->flow->gadget and p->flow->gadget->get_buf(buf.IBT_ALT, p, buf) ) { diff --git a/src/framework/mpse.h b/src/framework/mpse.h index ca12f5c5f..fd2d3b3ee 100644 --- a/src/framework/mpse.h +++ b/src/framework/mpse.h @@ -67,11 +67,13 @@ public: bool no_case; bool negated; bool literal; + bool multi_match; + unsigned flags; PatternDescriptor( - bool noc = false, bool neg = false, bool lit = false, unsigned f = 0) - { no_case = noc; negated = neg; literal = lit; flags = f; } + bool noc = false, bool neg = false, bool lit = false, bool multi = false, unsigned f = 0) + { no_case = noc; negated = neg; literal = lit; multi_match = multi; flags = f; } }; virtual int add_pattern( diff --git a/src/framework/mpse_batch.h b/src/framework/mpse_batch.h index c0a88042a..7bf5660fd 100644 --- a/src/framework/mpse_batch.h +++ b/src/framework/mpse_batch.h @@ -60,8 +60,8 @@ public: { return get_offload_mpse() != normal_mpse; } public: // FIXIT-L privatize - Mpse* normal_mpse; - Mpse* offload_mpse; + Mpse* normal_mpse; + Mpse* offload_mpse; }; template diff --git a/src/main/modules.cc b/src/main/modules.cc index a2d134786..eae929911 100644 --- a/src/main/modules.cc +++ b/src/main/modules.cc @@ -193,9 +193,6 @@ static const Parameter search_engine_params[] = { "rule_db_dir", Parameter::PT_STRING, nullptr, nullptr, "deserialize rule databases from given directory" }, - { "search_optimize", Parameter::PT_BOOL, nullptr, "true", - "tweak state machine construction for better performance" }, - { "show_fast_patterns", Parameter::PT_BOOL, nullptr, "false", "print fast pattern info for each rule" }, @@ -309,9 +306,6 @@ bool SearchEngineModule::set(const char*, Value& v, SnortConfig* sc) if ( !fp->set_offload_search_method(v.get_string()) ) return false; } - else if ( v.is("search_optimize") ) - fp->set_search_opt(v.get_bool()); - else if ( v.is("show_fast_patterns") ) fp->set_debug_print_fast_patterns(v.get_bool()); @@ -641,9 +635,6 @@ static const Parameter alerts_params[] = "set the CIDR for homenet " "(for use with -l or -B, does NOT change $HOME_NET in IDS mode)" }, - { "stateful", Parameter::PT_BOOL, nullptr, "false", - "don't alert w/o established session (note: rule action still taken)" }, - { "tunnel_verdicts", Parameter::PT_STRING, nullptr, nullptr, "let DAQ handle non-allow verdicts for gtp|teredo|6in4|4in6|4in4|6in6|gre|mpls|vxlan traffic" }, @@ -686,9 +677,6 @@ bool AlertsModule::set(const char*, Value& v, SnortConfig* sc) else if ( v.is("reference_net") ) return ( sc->homenet.set(v.get_string()) == SFIP_SUCCESS ); - else if ( v.is("stateful") ) - v.update_mask(sc->run_flags, RUN_FLAG__ASSURE_EST); - else if ( v.is("tunnel_verdicts") ) sc->set_tunnel_verdicts(v.get_string()); @@ -1745,7 +1733,7 @@ public: } Usage get_usage() const override - { return CONTEXT; } + { return INSPECT; } private: tSFRFConfigNode thdx; diff --git a/src/main/snort.cc b/src/main/snort.cc index 3d4b2a51c..e6ec48121 100644 --- a/src/main/snort.cc +++ b/src/main/snort.cc @@ -214,7 +214,7 @@ void Snort::init(int argc, char** argv) InspectorManager::prepare_controls(sc); // Must be after InspectorManager::configure() - FileService::post_init(sc); + FileService::post_init(); if (sc->file_mask != 0) umask(sc->file_mask); diff --git a/src/main/snort_config.h b/src/main/snort_config.h index f2da7cab8..441165707 100644 --- a/src/main/snort_config.h +++ b/src/main/snort_config.h @@ -61,18 +61,14 @@ enum RunFlag RUN_FLAG__PAUSE = 0x00004000, RUN_FLAG__NO_PCRE = 0x00008000, - /* If stream is configured, the STATEFUL flag is set. This is - * somewhat misnamed and is used to assure a session is established */ - RUN_FLAG__ASSURE_EST = 0x00010000, + RUN_FLAG__DUMP_RULE_STATE = 0x00010000, RUN_FLAG__DUMP_RULE_DEPS = 0x00020000, RUN_FLAG__TEST = 0x00040000, RUN_FLAG__MEM_CHECK = 0x00080000, RUN_FLAG__TRACK_ON_SYN = 0x00100000, RUN_FLAG__IP_FRAGS_ONLY = 0x00200000, - RUN_FLAG__DUMP_RULE_STATE = 0x00400000, - - RUN_FLAG__TEST_FEATURES = 0x00800000, + RUN_FLAG__TEST_FEATURES = 0x00400000, #ifdef SHELL RUN_FLAG__SHELL = 0x01000000, @@ -591,9 +587,6 @@ public: bool conf_error_out() const { return run_flags & RUN_FLAG__CONF_ERROR_OUT; } - bool assure_established() const - { return run_flags & RUN_FLAG__ASSURE_EST; } - bool test_features() const { return run_flags & RUN_FLAG__TEST_FEATURES; } diff --git a/src/managers/inspector_manager.cc b/src/managers/inspector_manager.cc index 6db5bff7b..4c2ae796f 100644 --- a/src/managers/inspector_manager.cc +++ b/src/managers/inspector_manager.cc @@ -1791,8 +1791,6 @@ bool InspectorManager::configure(SnortConfig* sc, bool cloned) } bool ok = true; - SearchTool::set_conf(sc); - SingleInstanceInspectorPolicy* fid = sc->policy_map->get_file_id(); fid->configure(sc); @@ -1829,7 +1827,6 @@ bool InspectorManager::configure(SnortConfig* sc, bool cloned) assert(np); set_network_policy(np); set_inspection_policy(np->get_inspection_policy()); - SearchTool::set_conf(nullptr); return ok; } diff --git a/src/managers/test/get_inspector_stubs.h b/src/managers/test/get_inspector_stubs.h index 1c7f44a34..832c91a1d 100644 --- a/src/managers/test/get_inspector_stubs.h +++ b/src/managers/test/get_inspector_stubs.h @@ -42,7 +42,6 @@ void set_default_policy(const snort::SnortConfig*) { } namespace snort { unsigned THREAD_LOCAL Inspector::slot = 0; -const SnortConfig* SearchTool::conf = nullptr; [[noreturn]] void FatalError(const char*,...) { exit(-1); } void LogMessage(const char*, ...) { } void LogLabel(const char*, FILE*) { } diff --git a/src/network_inspectors/appid/appid_config.cc b/src/network_inspectors/appid/appid_config.cc index 7fef03787..4f040b53c 100644 --- a/src/network_inspectors/appid/appid_config.cc +++ b/src/network_inspectors/appid/appid_config.cc @@ -139,7 +139,6 @@ bool AppIdContext::init_appid(SnortConfig* sc, AppIdInspector& inspector) void AppIdContext::create_odp_ctxt() { SnortConfig* sc = SnortConfig::get_main_conf(); - SearchTool::set_conf(sc); odp_ctxt = new OdpContext(config, sc); } diff --git a/src/network_inspectors/appid/client_plugins/test/client_plugins_mock.h b/src/network_inspectors/appid/client_plugins/test/client_plugins_mock.h index 95148a451..aea3efd23 100644 --- a/src/network_inspectors/appid/client_plugins/test/client_plugins_mock.h +++ b/src/network_inspectors/appid/client_plugins/test/client_plugins_mock.h @@ -26,7 +26,7 @@ void LogMessage(const char*,...) { } void WarningMessage(const char*,...) { } // Stubs for search_tool.cc -SearchTool::SearchTool(char const*, bool) { } +SearchTool::SearchTool(bool) { } SearchTool::~SearchTool() = default; void SearchTool::add(const char*, unsigned, int, bool) { } void SearchTool::add(const char*, unsigned, void*, bool) { } diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_sip_test.cc b/src/network_inspectors/appid/detector_plugins/test/detector_sip_test.cc index 42a3caab6..9db13097d 100644 --- a/src/network_inspectors/appid/detector_plugins/test/detector_sip_test.cc +++ b/src/network_inspectors/appid/detector_plugins/test/detector_sip_test.cc @@ -66,7 +66,7 @@ Flow::~Flow() = default; AppIdSession* AppIdApi::get_appid_session(snort::Flow const&) { return nullptr; } MpseGroup::~MpseGroup() = default; -SearchTool::SearchTool(const char*, bool) +SearchTool::SearchTool(bool) { mpsegrp = &mpse_group; } diff --git a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc index f1302eef3..2fe6aebf9 100644 --- a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc +++ b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc @@ -60,7 +60,7 @@ namespace snort { AppIdSessionApi::AppIdSessionApi(const AppIdSession*, const SfIp&) : StashGenericObject(STASH_GENERIC_OBJECT_APPID) {} -SearchTool::SearchTool(const char*, bool) { } +SearchTool::SearchTool(bool) { } void SearchTool::reload() { } static bool test_find_all_done = false; static bool test_find_all_enabled = false; diff --git a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h index 0fc7c38bf..40d83fae4 100644 --- a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h +++ b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h @@ -44,7 +44,7 @@ bool Inspector::get_buf(const char*, Packet*, InspectionBuffer&) { return true; class StreamSplitter* Inspector::get_splitter(bool) { return nullptr; } // Stubs for search_tool.cc -SearchTool::SearchTool(const char*, bool) { } +SearchTool::SearchTool(bool) { } SearchTool::~SearchTool() = default; // Stubs for util.cc diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc index 47e4b3acd..0bd3a3b2e 100644 --- a/src/network_inspectors/appid/test/appid_discovery_test.cc +++ b/src/network_inspectors/appid/test/appid_discovery_test.cc @@ -100,7 +100,7 @@ char* snort_strndup(const char* src, size_t) time_t packet_time() { return std::time(nullptr); } // Stubs for search_tool -SearchTool::SearchTool(const char*, bool) {} +SearchTool::SearchTool(bool) {} SearchTool::~SearchTool() = default; void SearchTool::add(const char*, unsigned, int, bool) {} void SearchTool::add(const char*, unsigned, void*, bool) {} diff --git a/src/network_inspectors/appid/test/appid_mock_definitions.h b/src/network_inspectors/appid/test/appid_mock_definitions.h index 603737084..40df9c3a8 100644 --- a/src/network_inspectors/appid/test/appid_mock_definitions.h +++ b/src/network_inspectors/appid/test/appid_mock_definitions.h @@ -57,7 +57,7 @@ void LogText(const char*, FILE*) {} void ParseWarning(WarningGroup, const char*, ...) { } void LogLabel(const char*, FILE*) {} -SearchTool::SearchTool(char const*, bool) { } +SearchTool::SearchTool(bool) { } SearchTool::~SearchTool() = default; } DiscoveryFilter::~DiscoveryFilter(){} diff --git a/src/network_inspectors/appid/test/service_state_test.cc b/src/network_inspectors/appid/test/service_state_test.cc index 7396e9698..8114b1868 100644 --- a/src/network_inspectors/appid/test/service_state_test.cc +++ b/src/network_inspectors/appid/test/service_state_test.cc @@ -125,7 +125,7 @@ EveCaPatternMatchers::~EveCaPatternMatchers() = default; HttpPatternMatchers::~HttpPatternMatchers() = default; SipPatternMatchers::~SipPatternMatchers() = default; SslPatternMatchers::~SslPatternMatchers() = default; -snort::SearchTool::SearchTool(char const*, bool) { } +snort::SearchTool::SearchTool(bool) { } snort::SearchTool::~SearchTool() = default; TEST_GROUP(service_state_tests) diff --git a/src/network_inspectors/appid/test/tp_lib_handler_test.cc b/src/network_inspectors/appid/test/tp_lib_handler_test.cc index 931ff8544..0f3848a5a 100644 --- a/src/network_inspectors/appid/test/tp_lib_handler_test.cc +++ b/src/network_inspectors/appid/test/tp_lib_handler_test.cc @@ -43,7 +43,7 @@ static OdpContext stub_odp_ctxt(config, nullptr); OdpContext* AppIdContext::odp_ctxt = &stub_odp_ctxt; ThirdPartyAppIdContext* AppIdContext::tp_appid_ctxt = nullptr; -snort::SearchTool::SearchTool(char const*, bool) { } +snort::SearchTool::SearchTool(bool) { } snort::SearchTool::~SearchTool() = default; AppIdDiscovery::~AppIdDiscovery() = default; diff --git a/src/network_inspectors/rna/rna_fingerprint_ua.cc b/src/network_inspectors/rna/rna_fingerprint_ua.cc index c9e936fb8..6abb849e8 100644 --- a/src/network_inspectors/rna/rna_fingerprint_ua.cc +++ b/src/network_inspectors/rna/rna_fingerprint_ua.cc @@ -102,13 +102,19 @@ void UaFpProcessor::push(const RawFingerprint& rfp) } } -void UaFpProcessor::make_mpse(SnortConfig* sc) +void UaFpProcessor::make_mpse(bool priority) { - if ( !sc ) - sc = SnortConfig::get_main_conf(); - SearchTool::set_conf(sc); + if ( priority ) + { + delete os_mpse; + delete device_mpse; + delete jb_mpse; + delete jb_host_mpse; + + os_mpse = device_mpse = jb_mpse = jb_host_mpse = nullptr; + } - if ( !os_fps.empty() ) + if ( !os_mpse and !os_fps.empty() ) { os_mpse = new SearchTool; for (auto& fp : os_fps) @@ -116,7 +122,7 @@ void UaFpProcessor::make_mpse(SnortConfig* sc) os_mpse->prep(); } - if ( !device_fps.empty() ) + if ( !device_mpse and !device_fps.empty() ) { device_mpse = new SearchTool; for (auto& fp : device_fps) @@ -124,7 +130,7 @@ void UaFpProcessor::make_mpse(SnortConfig* sc) device_mpse->prep(); } - if ( !jb_fps.empty() ) + if ( !jb_mpse and !jb_fps.empty() ) { jb_mpse = new SearchTool; for (auto& fp : jb_fps) @@ -132,7 +138,7 @@ void UaFpProcessor::make_mpse(SnortConfig* sc) jb_mpse->prep(); } - if ( !jb_host_fps.empty() ) + if ( !jb_host_mpse and !jb_host_fps.empty() ) { jb_host_mpse = new SearchTool; for (auto& fp : jb_host_fps) diff --git a/src/network_inspectors/rna/rna_fingerprint_ua.h b/src/network_inspectors/rna/rna_fingerprint_ua.h index b81bbb4b8..0a9e79f1d 100644 --- a/src/network_inspectors/rna/rna_fingerprint_ua.h +++ b/src/network_inspectors/rna/rna_fingerprint_ua.h @@ -21,7 +21,6 @@ #ifndef RNA_FINGERPRINT_UA_H #define RNA_FINGERPRINT_UA_H -#include "main/snort_config.h" #include "main/snort_types.h" #include "search_engines/search_tool.h" @@ -52,7 +51,7 @@ public: bool has_pattern() { return os_mpse != nullptr; } - void make_mpse(SnortConfig* sc = nullptr); + void make_mpse(bool priority = false); void match_mpse(const char*, const char*, const UaFingerprint*&, const char*&, bool&); diff --git a/src/network_inspectors/rna/rna_inspector.cc b/src/network_inspectors/rna/rna_inspector.cc index 08ea9977e..15fb65cd8 100644 --- a/src/network_inspectors/rna/rna_inspector.cc +++ b/src/network_inspectors/rna/rna_inspector.cc @@ -110,6 +110,9 @@ bool RnaInspector::configure(SnortConfig*) if (rna_conf && rna_conf->log_when_idle) DataBus::subscribe_network( THREAD_IDLE_EVENT, new RnaIdleEventHandler(*pnd) ); + if ( mod_conf->ua_processor ) + mod_conf->ua_processor->make_mpse(); + return true; } diff --git a/src/network_inspectors/rna/rna_module.cc b/src/network_inspectors/rna/rna_module.cc index 547cb37b1..9561d0392 100644 --- a/src/network_inspectors/rna/rna_module.cc +++ b/src/network_inspectors/rna/rna_module.cc @@ -533,10 +533,6 @@ bool RnaModule::end(const char* fqn, int index, SnortConfig* sc) mod_conf->tcp_processor->make_tcp_fp_tables(TcpFpProcessor::TCP_FP_MODE::SERVER); mod_conf->tcp_processor->make_tcp_fp_tables(TcpFpProcessor::TCP_FP_MODE::CLIENT); } - - if ( mod_conf->ua_processor ) - mod_conf->ua_processor->make_mpse(sc); - } if ( index > 0 and mod_conf->tcp_processor and !strcmp(fqn, "rna.tcp_fingerprints") ) diff --git a/src/network_inspectors/rna/test/CMakeLists.txt b/src/network_inspectors/rna/test/CMakeLists.txt index dfc6e9102..b002a92d1 100644 --- a/src/network_inspectors/rna/test/CMakeLists.txt +++ b/src/network_inspectors/rna/test/CMakeLists.txt @@ -10,3 +10,10 @@ add_cpputest( rna_module_test ${DNET_LIBRARIES} ${LUAJIT_LIBRARIES} ) + +add_cpputest( rna_ua_fp_processor_test + SOURCES + ../rna_fingerprint_ua.cc + ua_fp_stubs.cc +) + diff --git a/src/network_inspectors/rna/test/rna_module_stubs.h b/src/network_inspectors/rna/test/rna_module_stubs.h index f8dd9f8fa..a78d3c83f 100644 --- a/src/network_inspectors/rna/test/rna_module_stubs.h +++ b/src/network_inspectors/rna/test/rna_module_stubs.h @@ -57,7 +57,7 @@ void set_tcp_fp_processor(TcpFpProcessor*) { } TcpFingerprint::TcpFingerprint(const RawFingerprint&) { } UaFpProcessor::~UaFpProcessor() = default; -void UaFpProcessor::make_mpse(SnortConfig*) { } +void UaFpProcessor::make_mpse(bool) { } void UaFpProcessor::push(RawFingerprint const&) { } void UdpFpProcessor::push(RawFingerprint const&) { } diff --git a/src/network_inspectors/rna/test/rna_ua_fp_processor_test.cc b/src/network_inspectors/rna/test/rna_ua_fp_processor_test.cc new file mode 100644 index 000000000..bc1c29496 --- /dev/null +++ b/src/network_inspectors/rna/test/rna_ua_fp_processor_test.cc @@ -0,0 +1,146 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// rna_ua_fp_processor_test.cc author Russ Combs + +// The goal of these tests is to validate make_mpse priority. Given that rna +// adds fingerprints via module and makes mpse in configure and that util +// does both steps in configure, there are two possible init sequences that +// can occur based on the ordering of configure calls: +// +// Sequential: +// +// 1. rna adds fingerprints +// 2. rna makes mpse +// 3. util adds fingerprints +// 4. util makes mpse with priority - replaces +// +// Interleaved: +// +// 1. rna adds fingerprints +// 2. util adds fingerprints +// 3. util makes mpse with priority +// 4. rna makes mpse - no change + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include "network_inspectors/rna/rna_fingerprint_ua.h" +#include "search_engines/search_tool.h" + +#include +#include +#include + +using namespace snort; + +//-------------------------------------------------------------------------- +// stubs and mocks +//-------------------------------------------------------------------------- + +static unsigned s_count, s_prep_count; +static std::string s_data, s_prep_data; + +namespace snort +{ + SearchTool::SearchTool(bool) + { s_prep_count = s_count = 0; } + + SearchTool::~SearchTool() + { + s_prep_count = s_count = 0; + s_data.clear(); + s_prep_data.clear(); + } + + void SearchTool::add(const char* s, unsigned n, void*, bool) + { + s_count++; + s_data.append(s, n); + } + + void SearchTool::prep() + { + s_prep_count = s_count; + s_prep_data = s_data; + } +} + +//-------------------------------------------------------------------------- +// tests +//-------------------------------------------------------------------------- + +TEST_GROUP(rna_ua_fp_processor_test) +{ + UaFpProcessor* ua_proc; + + void setup() override + { ua_proc = new UaFpProcessor; } + + void teardown() override + { delete ua_proc; } +}; + +TEST(rna_ua_fp_processor_test, sequential_setup) +{ + UaFingerprint fp; + + fp.user_agent = "Pink "; + ua_proc->push_agent(fp); + ua_proc->make_mpse(); + + CHECK(s_prep_count == 1); + + fp.user_agent = "Floyd"; + ua_proc->push_agent(fp); + ua_proc->make_mpse(true); + + CHECK(s_prep_count == 2); + CHECK(s_prep_data == "Pink Floyd"); +} + +TEST(rna_ua_fp_processor_test, interleaved_setup) +{ + UaFingerprint fp; + + fp.user_agent = "Pink "; + ua_proc->push_agent(fp); + + CHECK(s_prep_count == 0); + + fp.user_agent = "Floyd"; + ua_proc->push_agent(fp); + ua_proc->make_mpse(true); + + CHECK(s_prep_count == 2); + CHECK(s_prep_data == "Pink Floyd"); + + ua_proc->make_mpse(); + + CHECK(s_prep_count == 2); + CHECK(s_prep_data == "Pink Floyd"); +} + +int main(int argc, char** argv) +{ + return CommandLineTestRunner::RunAllTests(argc, argv); +} diff --git a/src/network_inspectors/rna/test/ua_fp_stubs.cc b/src/network_inspectors/rna/test/ua_fp_stubs.cc new file mode 100644 index 000000000..d8f5f0c85 --- /dev/null +++ b/src/network_inspectors/rna/test/ua_fp_stubs.cc @@ -0,0 +1,44 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// ua_fp_stubs.cc author Russ Combs + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "search_engines/search_tool.h" + +namespace snort +{ +void SearchTool::add(const char*, unsigned, int, bool) { } +void SearchTool::add(const uint8_t*, unsigned, int, bool) { } +void SearchTool::add(const uint8_t*, unsigned, void*, bool) { } + +void SearchTool::reload() { } + +int SearchTool::find(const char*, unsigned, MpseMatch, int&, bool, void*) +{ return 0; } + +int SearchTool::find(const char*, unsigned, MpseMatch, bool, void*) +{ return 0; } + +int SearchTool::find_all(const char*, unsigned, MpseMatch, bool, void*) +{ return 0; } +} + diff --git a/src/parser/parse_rule.cc b/src/parser/parse_rule.cc index 23f376cd6..5d5cfb389 100644 --- a/src/parser/parse_rule.cc +++ b/src/parser/parse_rule.cc @@ -1179,7 +1179,7 @@ void parse_rule_close(SnortConfig* sc, RuleTreeNode& rtn, OptTreeNode* otn) if ( otn->sigInfo.message.empty() ) otn->sigInfo.message = "\"no msg in rule\""; - OptFpList* fpl = AddOptFuncToList(OptListEnd, otn); + OptFpList* fpl = AddOptFuncToList(nullptr, otn); fpl->type = RULE_OPTION_TYPE_LEAF_NODE; if ( is_service_protocol(otn->snort_protocol_id) ) diff --git a/src/parser/parser.h b/src/parser/parser.h index a7407143c..fa1beb7fb 100644 --- a/src/parser/parser.h +++ b/src/parser/parser.h @@ -80,11 +80,6 @@ inline RuleTreeNode* getRtnFromOtn(const struct OptTreeNode* otn) return getRtnFromOtn(otn, snort::get_ips_policy()->policy_id); } -inline RuleTreeNode* getRuntimeRtnFromOtn(const struct OptTreeNode* otn) -{ - return getRtnFromOtn(otn); -} - RuleListNode* CreateRuleType(snort::SnortConfig* sc, const char* name, Actions::Type action_type); void FreeRuleTreeNode(RuleTreeNode*); diff --git a/src/search_engines/CMakeLists.txt b/src/search_engines/CMakeLists.txt index e3f3f8aa4..816bc0ea6 100644 --- a/src/search_engines/CMakeLists.txt +++ b/src/search_engines/CMakeLists.txt @@ -4,20 +4,10 @@ set (SEARCH_ENGINE_INCLUDES search_tool.h ) -set (ACSMX_SOURCES - ac_std.cc - acsmx.cc - acsmx.h -) - set (ACSMX2_SOURCES - ac_banded.cc ac_full.cc - ac_sparse.cc - ac_sparse_bands.cc acsmx2.cc acsmx2.h - acsmx2_api.cc ) set (BNFA_SOURCES @@ -38,14 +28,12 @@ set (SEARCH_ENGINE_SOURCES search_engines.h search_tool.cc ${BNFA_SOURCES} + ${ACSMX2_SOURCES} ) if ( STATIC_SEARCH_ENGINES ) add_library(search_engines OBJECT - ${ACSMX_SOURCES} - ${ACSMX2_SOURCES} ${HYPER_SOURCES} - ${INTEL_SOURCES} ${SEARCH_ENGINE_SOURCES} ${SEARCH_ENGINE_INCLUDES} ) @@ -56,8 +44,6 @@ else ( STATIC_SEARCH_ENGINES) ${SEARCH_ENGINE_INCLUDES} ) - add_dynamic_module(acsmx search_engines ${ACSMX_SOURCES}) - add_dynamic_module(acsmx2 search_engines ${ACSMX2_SOURCES}) if ( HAVE_HYPERSCAN ) add_dynamic_module(hyperscan search_engines ${HYPER_SOURCES}) endif () diff --git a/src/search_engines/ac_banded.cc b/src/search_engines/ac_banded.cc deleted file mode 100644 index 02199dadb..000000000 --- a/src/search_engines/ac_banded.cc +++ /dev/null @@ -1,133 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2022 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2013-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "framework/mpse.h" - -#include "acsmx2.h" - -using namespace snort; - -//------------------------------------------------------------------------- -// "ac_banded" -//------------------------------------------------------------------------- - -class AcbMpse : public Mpse -{ -private: - ACSM_STRUCT2* obj; - -public: - AcbMpse(const MpseAgent* agent) : Mpse("ac_banded") - { obj = acsmNew2(agent, ACF_BANDED); } - - ~AcbMpse() override - { acsmFree2(obj); } - - void set_opt(int) override - { obj->enable_dfa(); } - - int add_pattern( - const uint8_t* P, unsigned m, const PatternDescriptor& desc, void* user) override - { - return acsmAddPattern2(obj, P, m, desc.no_case, desc.negated, user); - } - - int prep_patterns(SnortConfig* sc) override - { return acsmCompile2(sc, obj); } - - int _search( - const uint8_t* T, int n, MpseMatch match, - void* context, int* current_state) override - { -#if 1 - return acsm_search_dfa_banded(obj, T, n, match, context, current_state); -#else - if ( obj->dfa_enabled() ) - return acsm_search_dfa_banded(obj, T, n, match, context, current_state); - - // FIXIT-L banded will crash in get_next_state_nfa() - return acsm_search_nfa(obj, T, n, match, context, current_state); -#endif - } - - int print_info() override - { return acsmPrintDetailInfo2(obj); } - - int get_pattern_count() const override - { return acsmPatternCount2(obj); } -}; - -//------------------------------------------------------------------------- -// api -//------------------------------------------------------------------------- - -static Mpse* acb_ctor( - const SnortConfig*, class Module*, const MpseAgent* agent) -{ - return new AcbMpse(agent); -} - -static void acb_dtor(Mpse* p) -{ - delete p; -} - -static void acb_init() -{ - acsmx2_init_xlatcase(); - acsm_init_summary(); -} - -static void acb_print() -{ - acsmPrintSummaryInfo2(); -} - -static const MpseApi acb_api = -{ - { - PT_SEARCH_ENGINE, - sizeof(MpseApi), - SEAPI_VERSION, - 0, - API_RESERVED, - API_OPTIONS, - "ac_banded", - "Aho-Corasick Banded (high memory, moderate performance)", - nullptr, - nullptr - }, - MPSE_BASE, - nullptr, - nullptr, - nullptr, - nullptr, - acb_ctor, - acb_dtor, - acb_init, - acb_print, - nullptr, -}; - -const BaseApi* se_ac_banded = &acb_api.base; - diff --git a/src/search_engines/ac_bnfa.cc b/src/search_engines/ac_bnfa.cc index 9fad94ac1..f577815fe 100644 --- a/src/search_engines/ac_bnfa.cc +++ b/src/search_engines/ac_bnfa.cc @@ -58,12 +58,6 @@ public: bnfaFree(obj); } - void set_opt(int flag) override - { - if (obj) - bnfaSetOpt(obj, flag); - } - int add_pattern( const uint8_t* P, unsigned m, const PatternDescriptor& desc, void* user) override { diff --git a/src/search_engines/ac_full.cc b/src/search_engines/ac_full.cc index fcbec1146..7d92a9a58 100644 --- a/src/search_engines/ac_full.cc +++ b/src/search_engines/ac_full.cc @@ -38,17 +38,11 @@ private: public: AcfMpse(const MpseAgent* agent) : Mpse("ac_full") - { obj = acsmNew2(agent, ACF_FULL); } + { obj = acsmNew2(agent); } ~AcfMpse() override { acsmFree2(obj); } - void set_opt(int flag) override - { - acsmCompressStates(obj, flag); - obj->enable_dfa(); - } - int add_pattern( const uint8_t* P, unsigned m, const PatternDescriptor& desc, void* user) override { @@ -62,20 +56,14 @@ public: const uint8_t* T, int n, MpseMatch match, void* context, int* current_state) override { - if ( obj->dfa_enabled() ) - return acsm_search_dfa_full(obj, T, n, match, context, current_state); - - return acsm_search_nfa(obj, T, n, match, context, current_state); + return acsm_search_dfa_full(obj, T, n, match, context, current_state); } int search_all( const uint8_t* T, int n, MpseMatch match, void* context, int* current_state) override { - if ( !obj->dfa_enabled() ) - return acsm_search_nfa(obj, T, n, match, context, current_state); - else - return acsm_search_dfa_full_all(obj, T, n, match, context, current_state); + return acsm_search_dfa_full_all(obj, T, n, match, context, current_state); } int print_info() override @@ -137,5 +125,9 @@ static const MpseApi acf_api = nullptr, }; -const BaseApi* se_ac_full = &acf_api.base; +const BaseApi* se_ac_full[] = +{ + &acf_api.base, + nullptr +}; diff --git a/src/search_engines/ac_sparse.cc b/src/search_engines/ac_sparse.cc deleted file mode 100644 index a4ed83e42..000000000 --- a/src/search_engines/ac_sparse.cc +++ /dev/null @@ -1,128 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2022 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2013-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "framework/mpse.h" - -#include "acsmx2.h" - -using namespace snort; - -//------------------------------------------------------------------------- -// "ac_sparse" -//------------------------------------------------------------------------- - -class AcsMpse : public Mpse -{ -private: - ACSM_STRUCT2* obj; - -public: - AcsMpse(const MpseAgent* agent) : Mpse("ac_sparse") - { obj = acsmNew2(agent, ACF_SPARSE); } - - ~AcsMpse() override - { if (obj) acsmFree2(obj); } - - void set_opt(int) override - { obj->enable_dfa(); } - - int add_pattern( - const uint8_t* P, unsigned m, const PatternDescriptor& desc, void* user) override - { - return acsmAddPattern2(obj, P, m, desc.no_case, desc.negated, user); - } - - int prep_patterns(SnortConfig* sc) override - { return acsmCompile2(sc, obj); } - - int _search( - const uint8_t* T, int n, MpseMatch match, - void* context, int* current_state) override - { - if ( obj->dfa_enabled() ) - return acsm_search_dfa_sparse(obj, T, n, match, context, current_state); - - return acsm_search_nfa(obj, T, n, match, context, current_state); - } - - int print_info() override - { return acsmPrintDetailInfo2(obj); } - - int get_pattern_count() const override - { return acsmPatternCount2(obj); } -}; - -//------------------------------------------------------------------------- -// api -//------------------------------------------------------------------------- - -static Mpse* acs_ctor( - const SnortConfig*, class Module*, const MpseAgent* agent) -{ - return new AcsMpse(agent); -} - -static void acs_dtor(Mpse* p) -{ - delete p; -} - -static void acs_init() -{ - acsmx2_init_xlatcase(); - acsm_init_summary(); -} - -static void acs_print() -{ - acsmPrintSummaryInfo2(); -} - -static const MpseApi acs_api = -{ - { - PT_SEARCH_ENGINE, - sizeof(MpseApi), - SEAPI_VERSION, - 0, - API_RESERVED, - API_OPTIONS, - "ac_sparse", - "Aho-Corasick Sparse (high memory, moderate performance) MPSE", - nullptr, - nullptr - }, - MPSE_BASE, - nullptr, - nullptr, - nullptr, - nullptr, - acs_ctor, - acs_dtor, - acs_init, - acs_print, - nullptr, -}; - -const BaseApi* se_ac_sparse = &acs_api.base; - diff --git a/src/search_engines/ac_sparse_bands.cc b/src/search_engines/ac_sparse_bands.cc deleted file mode 100644 index b084aad3a..000000000 --- a/src/search_engines/ac_sparse_bands.cc +++ /dev/null @@ -1,128 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2022 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2013-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "framework/mpse.h" - -#include "acsmx2.h" - -using namespace snort; - -//------------------------------------------------------------------------- -// "ac_sparse_bands" -//------------------------------------------------------------------------- - -class AcsbMpse : public Mpse -{ -private: - ACSM_STRUCT2* obj; - -public: - AcsbMpse(const MpseAgent* agent) : Mpse("ac_sparse_bands") - { obj = acsmNew2(agent, ACF_SPARSE_BANDS); } - - ~AcsbMpse() override - { acsmFree2(obj); } - - void set_opt(int) override - { obj->enable_dfa(); } - - int add_pattern( - const uint8_t* P, unsigned m, const PatternDescriptor& desc, void* user) override - { - return acsmAddPattern2(obj, P, m, desc.no_case, desc.negated, user); - } - - int prep_patterns(SnortConfig* sc) override - { return acsmCompile2(sc, obj); } - - int _search( - const uint8_t* T, int n, MpseMatch match, - void* context, int* current_state) override - { - if ( obj->dfa_enabled() ) - return acsm_search_dfa_sparse(obj, T, n, match, context, current_state); - - return acsm_search_nfa(obj, T, n, match, context, current_state); - } - - int print_info() override - { return acsmPrintDetailInfo2(obj); } - - int get_pattern_count() const override - { return acsmPatternCount2(obj); } -}; - -//------------------------------------------------------------------------- -// api -//------------------------------------------------------------------------- - -static Mpse* acsb_ctor( - const SnortConfig*, class Module*, const MpseAgent* agent) -{ - return new AcsbMpse(agent); -} - -static void acsb_dtor(Mpse* p) -{ - delete p; -} - -static void acsb_init() -{ - acsmx2_init_xlatcase(); - acsm_init_summary(); -} - -static void acsb_print() -{ - acsmPrintSummaryInfo2(); -} - -static const MpseApi acsb_api = -{ - { - PT_SEARCH_ENGINE, - sizeof(MpseApi), - SEAPI_VERSION, - 0, - API_RESERVED, - API_OPTIONS, - "ac_sparse_bands", - "Aho-Corasick Sparse-Banded (high memory, moderate performance) MPSE", - nullptr, - nullptr - }, - MPSE_BASE, - nullptr, - nullptr, - nullptr, - nullptr, - acsb_ctor, - acsb_dtor, - acsb_init, - acsb_print, - nullptr, -}; - -const BaseApi* se_ac_sparse_bands = &acsb_api.base; - diff --git a/src/search_engines/ac_std.cc b/src/search_engines/ac_std.cc deleted file mode 100644 index e21a06e9a..000000000 --- a/src/search_engines/ac_std.cc +++ /dev/null @@ -1,129 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2022 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2013-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "framework/mpse.h" - -#include "acsmx.h" - -using namespace snort; - -//------------------------------------------------------------------------- -// "ac_std" -//------------------------------------------------------------------------- - -class AcMpse : public Mpse -{ -private: - ACSM_STRUCT* obj; - -public: - AcMpse(const MpseAgent* agent) : Mpse("ac_std") - { obj = acsmNew(agent); } - - ~AcMpse() override - { acsmFree(obj); } - - int add_pattern( - const uint8_t* P, unsigned m, const PatternDescriptor& desc, void* user) override - { - return acsmAddPattern(obj, P, m, desc.no_case, desc.negated, user); - } - - int prep_patterns(SnortConfig* sc) override - { return acsmCompile(sc, obj); } - - int _search( - const uint8_t* T, int n, MpseMatch match, - void* context, int* current_state) override - { - return acsmSearch(obj, T, n, match, context, current_state); - } - - int print_info() override - { return acsmPrintDetailInfo(obj); } - - int get_pattern_count() const override - { return acsmPatternCount(obj); } -}; - -//------------------------------------------------------------------------- -// api -//------------------------------------------------------------------------- - -static Mpse* ac_ctor( - const SnortConfig*, class Module*, const MpseAgent* agent) -{ - return new AcMpse(agent); -} - -static void ac_dtor(Mpse* p) -{ - delete p; -} - -static void ac_init() -{ - acsmx_init_xlatcase(); -} - -static void ac_print() -{ - acsmPrintSummaryInfo(); -} - -static const MpseApi ac_api = -{ - { - PT_SEARCH_ENGINE, - sizeof(MpseApi), - SEAPI_VERSION, - 0, - API_RESERVED, - API_OPTIONS, - "ac_std", - "Aho-Corasick Full (high memory, best performance) MPSE", - nullptr, - nullptr - }, - MPSE_BASE, - nullptr, - nullptr, - nullptr, - nullptr, - ac_ctor, - ac_dtor, - ac_init, - ac_print, - nullptr, -}; - -#ifdef BUILDING_SO -SO_PUBLIC const BaseApi* snort_plugins[] = -#else -const BaseApi* se_ac_std[] = -#endif -{ - &ac_api.base, - nullptr -}; - diff --git a/src/search_engines/acsmx.cc b/src/search_engines/acsmx.cc deleted file mode 100644 index ea35dda43..000000000 --- a/src/search_engines/acsmx.cc +++ /dev/null @@ -1,556 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2022 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2002-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- -/* -** Multi-Pattern Search Engine -** -** Aho-Corasick State Machine - uses a Deterministic Finite Automata - DFA -** Marc Norton -** -** -** Reference - Efficient String matching: An Aid to Bibliographic Search -** Alfred V Aho and Margaret J Corasick -** Bell Laboratories -** Copyright (C) 1975 Association for Computing Machinery,Inc -** -** Implemented from the 4 algorithms in the paper by Aho & Corasick -** and some implementation ideas from 'Practical Algorithms in C' -** -** Notes: -** 1) This version uses about 1024 bytes per pattern character - heavy on the memory. -** 2) This algorithm finds all occurrences of all patterns within a -** body of text. -** 3) Support is included to handle upper and lower case matching. -** 4) Some compilers optimize the search routine well, others don't, this makes all the difference. -** 5) Aho inspects all bytes of the search text, but only once so it's very efficient, -** if the patterns are all large than the Modified Wu-Manbar method is often faster. -** 6) I don't subscribe to any one method is best for all searching needs, -** the data decides which method is best, -** and we don't know until after the search method has been tested on the specific data sets. -** -** May 2002 : Marc Norton 1st Version -** June 2002 : Modified interface for SNORT, added case support -** Aug 2002 : Cleaned up comments, and removed dead code. -** Nov 2,2002: Fixed queue_init() , added count=0 -*/ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "acsmx.h" - -#include - -#include "main/thread.h" -#include "utils/util.h" - -using namespace snort; - -static int max_memory = 0; - -static void* AC_MALLOC(int n) -{ - void* p = snort_calloc(n); - max_memory += n; - return p; -} - -static void AC_FREE(void* p) -{ - if (p) - snort_free(p); -} - -/* -** Case Translation Table -*/ -static uint8_t xlatcase[256]; - -/* -* -*/ -void acsmx_init_xlatcase() -{ - int i; - for (i = 0; i < 256; i++) - { - xlatcase[i] = (uint8_t)toupper (i); - } -} - -/* -* -*/ -static inline void ConvertCaseEx(uint8_t* d, const uint8_t* s, int m) -{ - int i; - for (i = 0; i < m; i++) - { - d[i] = xlatcase[s[i]]; - } -} - -/* -* -*/ -static ACSM_PATTERN* CopyMatchListEntry(ACSM_PATTERN* px) -{ - ACSM_PATTERN* p; - p = (ACSM_PATTERN*)AC_MALLOC(sizeof(ACSM_PATTERN)); - memcpy(p, px, sizeof (ACSM_PATTERN)); - px->udata->ref_count++; - p->next = nullptr; - return p; -} - -/* -* Add a pattern to the list of patterns terminated at this state. -* Insert at front of list. -*/ -static void AddMatchListEntry(ACSM_STRUCT* acsm, int state, ACSM_PATTERN* px) -{ - ACSM_PATTERN* p; - p = (ACSM_PATTERN*)AC_MALLOC(sizeof(ACSM_PATTERN)); - memcpy(p, px, sizeof (ACSM_PATTERN)); - p->next = acsm->acsmStateTable[state].MatchList; - acsm->acsmStateTable[state].MatchList = p; -} - -/* - Add Pattern States -*/ -static void AddPatternStates(ACSM_STRUCT* acsm, ACSM_PATTERN* p) -{ - int state = 0; - int n = p->n; - uint8_t* pattern = p->patrn; - - /* - * Match up pattern with existing states - */ - for (; n > 0; pattern++, n--) - { - int next = acsm->acsmStateTable[state].NextState[*pattern]; - if (next == ACSM_FAIL_STATE) - break; - state = next; - } - - /* - * Add new states for the rest of the pattern bytes, 1 state per byte - */ - for (; n > 0; pattern++, n--) - { - acsm->acsmNumStates++; - acsm->acsmStateTable[state].NextState[*pattern] = acsm->acsmNumStates; - state = acsm->acsmNumStates; - } - - AddMatchListEntry (acsm, state, p); -} - -/* -* Build Non-Deterministic Finite Automata -*/ -static void Build_NFA(ACSM_STRUCT* acsm) -{ - ACSM_PATTERN* mlist=nullptr; - ACSM_PATTERN* px=nullptr; - - std::list queue; - - /* Add the state 0 transitions 1st */ - for (int i = 0; i < ALPHABET_SIZE; i++) - { - int s = acsm->acsmStateTable[0].NextState[i]; - - if (s) - { - queue.emplace_back(s); - acsm->acsmStateTable[s].FailState = 0; - } - } - - /* Build the fail state transitions for each valid state */ - for ( auto r : queue ) - { - /* Find Final States for any Failure */ - for (int i = 0; i < ALPHABET_SIZE; i++) - { - int s = acsm->acsmStateTable[r].NextState[i]; - - if ( s != ACSM_FAIL_STATE ) - { - queue.emplace_back(s); - int fs = acsm->acsmStateTable[r].FailState; - int next; - - /* - * Locate the next valid state for 'i' starting at s - */ - while ((next=acsm->acsmStateTable[fs].NextState[i]) == - ACSM_FAIL_STATE) - { - fs = acsm->acsmStateTable[fs].FailState; - } - - /* - * Update 's' state failure state to point to the next valid state - */ - acsm->acsmStateTable[s].FailState = next; - - /* - * Copy 'next' states MatchList to 's' states MatchList, - * we copy them so each list can be AC_FREE'd later, - * else we could just manipulate pointers to fake the copy. - */ - for (mlist = acsm->acsmStateTable[next].MatchList; - mlist != nullptr; - mlist = mlist->next) - { - px = CopyMatchListEntry (mlist); - /* Insert at front of MatchList */ - px->next = acsm->acsmStateTable[s].MatchList; - acsm->acsmStateTable[s].MatchList = px; - } - } - } - } -} - -/* -* Build Deterministic Finite Automata from NFA -*/ -static void Convert_NFA_To_DFA(ACSM_STRUCT* acsm) -{ - std::list queue; - - /* Add the state 0 transitions 1st */ - for (int i = 0; i < ALPHABET_SIZE; i++) - { - if ( int s = acsm->acsmStateTable[0].NextState[i] ) - queue.emplace_back(s); - } - - /* Start building the next layer of transitions */ - for ( auto r : queue ) - { - /* State is a branch state */ - for (int i = 0; i < ALPHABET_SIZE; i++) - { - int s = acsm->acsmStateTable[r].NextState[i]; - - if ( s != ACSM_FAIL_STATE ) - { - queue.emplace_back(s); - } - else - { - acsm->acsmStateTable[r].NextState[i] = - acsm->acsmStateTable[acsm->acsmStateTable[r].FailState].NextState[i]; - } - } - } -} - -/* -* -*/ -ACSM_STRUCT* acsmNew(const MpseAgent* agent) -{ - ACSM_STRUCT* p = (ACSM_STRUCT*)AC_MALLOC (sizeof (ACSM_STRUCT)); - - if (p) - { - p->agent = agent; - } - return p; -} - -/* -* Add a pattern to the list of patterns for this state machine -*/ -int acsmAddPattern( - ACSM_STRUCT* p, const uint8_t* pat, unsigned n, bool nocase, - bool negative, void* user) -{ - ACSM_PATTERN* plist; - plist = (ACSM_PATTERN*)AC_MALLOC (sizeof (ACSM_PATTERN)); - plist->patrn = (uint8_t*)AC_MALLOC (n); - ConvertCaseEx (plist->patrn, pat, n); - plist->casepatrn = (uint8_t*)AC_MALLOC (n); - memcpy(plist->casepatrn, pat, n); - - plist->udata = (ACSM_USERDATA*)AC_MALLOC(sizeof(ACSM_USERDATA)); - plist->udata->ref_count = 1; - plist->udata->id = user; - - plist->n = n; - plist->nocase = nocase; - plist->negative = negative; - plist->next = p->acsmPatterns; - p->acsmPatterns = plist; - p->numPatterns++; - return 0; -} - -static void acsmBuildMatchStateTrees(SnortConfig* sc, ACSM_STRUCT* acsm) -{ - ACSM_PATTERN* mlist; - - /* Find the states that have a MatchList */ - for (int i = 0; i < acsm->acsmMaxStates; i++) - { - for ( mlist=acsm->acsmStateTable[i].MatchList; mlist!=nullptr; mlist=mlist->next ) - { - if (mlist->udata->id) - { - if (mlist->negative) - { - acsm->agent->negate_list( - mlist->udata->id, &acsm->acsmStateTable[i].MatchList->neg_list); - } - else - { - acsm->agent->build_tree(sc, mlist->udata->id, - &acsm->acsmStateTable[i].MatchList->rule_option_tree); - } - } - } - - if (acsm->acsmStateTable[i].MatchList) - { - /* Last call to finalize the tree */ - acsm->agent->build_tree(sc, nullptr, - &acsm->acsmStateTable[i].MatchList->rule_option_tree); - } - } -} - -/* -* Compile State Machine -*/ -static inline int _acsmCompile(ACSM_STRUCT* acsm) -{ - int i, k; - ACSM_PATTERN* plist; - - /* Count number of states */ - acsm->acsmMaxStates = 1; - for (plist = acsm->acsmPatterns; plist != nullptr; plist = plist->next) - { - acsm->acsmMaxStates += plist->n; - } - acsm->acsmStateTable = - (ACSM_STATETABLE*)AC_MALLOC (sizeof (ACSM_STATETABLE) * acsm->acsmMaxStates); - - /* Initialize state zero as a branch */ - acsm->acsmNumStates = 0; - - /* Initialize all States NextStates to FAILED */ - for (k = 0; k < acsm->acsmMaxStates; k++) - { - for (i = 0; i < ALPHABET_SIZE; i++) - { - acsm->acsmStateTable[k].NextState[i] = ACSM_FAIL_STATE; - } - } - - /* Add each Pattern to the State Table */ - for (plist = acsm->acsmPatterns; plist != nullptr; plist = plist->next) - { - AddPatternStates (acsm, plist); - } - - /* Set all failed state transitions to return to the 0'th state */ - for (i = 0; i < ALPHABET_SIZE; i++) - { - if (acsm->acsmStateTable[0].NextState[i] == ACSM_FAIL_STATE) - { - acsm->acsmStateTable[0].NextState[i] = 0; - } - } - - /* Build the NFA */ - Build_NFA (acsm); - - /* Convert the NFA to a DFA */ - Convert_NFA_To_DFA (acsm); - - return 0; -} - -int acsmCompile(SnortConfig* sc, ACSM_STRUCT* acsm) -{ - if ( int rval = _acsmCompile (acsm) ) - return rval; - - if ( acsm->agent ) - acsmBuildMatchStateTrees(sc, acsm); - - return 0; -} - -static THREAD_LOCAL uint8_t Tc[64*1024]; - -/* -* Search Text or Binary Data for Pattern matches -*/ -int acsmSearch( - ACSM_STRUCT* acsm, const uint8_t* Tx, int n, MpseMatch match, - void* context, int* current_state) -{ - int state = 0; - ACSM_PATTERN* mlist; - const uint8_t* Tend; - ACSM_STATETABLE* StateTable = acsm->acsmStateTable; - int nfound = 0; - const uint8_t* T; - int index; - - /* Case conversion */ - ConvertCaseEx (Tc, Tx, n); - T = Tc; - Tend = T + n; - - if ( !current_state ) - { - return 0; - } - - state = *current_state; - - for (; T < Tend; T++) - { - state = StateTable[state].NextState[*T]; - - if ( StateTable[state].MatchList != nullptr ) - { - mlist = StateTable[state].MatchList; - index = T + 1 - Tc; - nfound++; - if (match(mlist->udata->id, mlist->rule_option_tree, index, context, - mlist->neg_list) > 0) - { - *current_state = state; - return nfound; - } - } - } - *current_state = state; - return nfound; -} - -/* -* Free all memory -*/ -void acsmFree(ACSM_STRUCT* acsm) -{ - int i; - ACSM_PATTERN* mlist, * ilist; - for (i = 0; i < acsm->acsmMaxStates; i++) - { - mlist = acsm->acsmStateTable[i].MatchList; - while (mlist) - { - ilist = mlist; - mlist = mlist->next; - - ilist->udata->ref_count--; - if (ilist->udata->ref_count == 0) - { - if (acsm->agent && ilist->udata->id) - acsm->agent->user_free(ilist->udata->id); - - AC_FREE(ilist->udata); - } - - if (ilist->rule_option_tree && acsm->agent) - { - acsm->agent->tree_free(&(ilist->rule_option_tree)); - } - - if (ilist->neg_list && acsm->agent) - { - acsm->agent->list_free(&(ilist->neg_list)); - } - - AC_FREE (ilist); - } - } - AC_FREE (acsm->acsmStateTable); - mlist = acsm->acsmPatterns; - while (mlist) - { - ilist = mlist; - mlist = mlist->next; - AC_FREE(ilist->patrn); - AC_FREE(ilist->casepatrn); - AC_FREE(ilist); - } - AC_FREE (acsm); -} - -int acsmPatternCount(ACSM_STRUCT* acsm) -{ - return acsm->numPatterns; -} - -static void Print_DFA( ACSM_STRUCT * acsm ) -{ - int k; - int i; - int next; - - for (k = 0; k < acsm->acsmMaxStates; k++) - { - for (i = 0; i < ALPHABET_SIZE; i++) - { - next = acsm->acsmStateTable[k].NextState[i]; - - if( next == 0 || next == ACSM_FAIL_STATE ) - { - if( isprint(i) ) - printf("%3c->%-5d\t",i,next); - else - printf("%3d->%-5d\t",i,next); - } - } - printf("\n"); - } - -} - -int acsmPrintDetailInfo(ACSM_STRUCT* acsm) -{ - Print_DFA( acsm ); - return 0; -} - -int acsmPrintSummaryInfo() -{ -#if 0 - // FIXIT-L should output summary similar to acsmPrintSummaryInfo2() - printf ("ACSMX-Max Memory: %d bytes, %d states\n", max_memory, - acsm->acsmMaxStates); -#endif - - return 0; -} - diff --git a/src/search_engines/acsmx.h b/src/search_engines/acsmx.h deleted file mode 100644 index edb93fb27..000000000 --- a/src/search_engines/acsmx.h +++ /dev/null @@ -1,115 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2022 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2002-2013 Sourcefire, Inc. -// Copyright (C) 2002 Martin Roesch -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -// acsmx.h author Marc Norton - -#ifndef ACSMX_H -#define ACSMX_H - -// version 1 - -#include - -#include "search_common.h" - -namespace snort -{ -struct SnortConfig; -} - -#define ALPHABET_SIZE 256 -#define ACSM_FAIL_STATE (-1) - -struct ACSM_USERDATA -{ - void* id; - uint32_t ref_count; -}; - -struct ACSM_PATTERN -{ - ACSM_PATTERN* next; - ACSM_USERDATA* udata; - - uint8_t* patrn; - uint8_t* casepatrn; - - void* rule_option_tree; - void* neg_list; - - int n; - int nocase; - int negative; -}; - -struct ACSM_STATETABLE -{ - /* Next state - based on input character */ - int NextState[ ALPHABET_SIZE ]; - - /* Failure state - used while building NFA & DFA */ - int FailState; - - /* List of patterns that end here, if any */ - ACSM_PATTERN* MatchList; -}; - -/* -* State machine Struct -*/ -struct ACSM_STRUCT -{ - int acsmMaxStates; - int acsmNumStates; - - ACSM_PATTERN* acsmPatterns; - ACSM_STATETABLE* acsmStateTable; - - int bcSize; - short bcShift[256]; - - int numPatterns; - const MpseAgent* agent; -}; - -/* -* Prototypes -*/ -void acsmx_init_xlatcase(); - -ACSM_STRUCT* acsmNew(const MpseAgent*); - -int acsmAddPattern(ACSM_STRUCT* p, const uint8_t* pat, unsigned n, - bool nocase, bool negative, void* id); - -int acsmCompile(snort::SnortConfig*, ACSM_STRUCT*); - -int acsmSearch(ACSM_STRUCT * acsm, const uint8_t* T, - int n, MpseMatch, void* context, int* current_state); - -void acsmFree(ACSM_STRUCT* acsm); -int acsmPatternCount(ACSM_STRUCT* acsm); - -int acsmPrintDetailInfo(ACSM_STRUCT*); - -int acsmPrintSummaryInfo(); - -#endif - diff --git a/src/search_engines/acsmx2.cc b/src/search_engines/acsmx2.cc index a8b43ba40..f8bcd25a1 100644 --- a/src/search_engines/acsmx2.cc +++ b/src/search_engines/acsmx2.cc @@ -19,6 +19,12 @@ // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. //-------------------------------------------------------------------------- /* +** All but the full storage format have been deleted. The full format +** provides the best performance. Only the sparse bands format used +** less memory than full and it was way slower. The ac_bnfa algorithm +** is available for lower memory usage. The following notes are retained +** to help make sense of the code. +** ** acsmx2.c ** ** Multi-Pattern Search Engine @@ -174,13 +180,8 @@ void acsm_init_summary() acsm2_failstate_memory = 0; } -/* -** Case Translation Table -*/ static uint8_t xlatcase[256]; -/* -* -*/ + void acsmx2_init_xlatcase() { int i; @@ -190,43 +191,14 @@ void acsmx2_init_xlatcase() } } -/* -* Case Conversion -*/ static inline void ConvertCaseEx(uint8_t* d, const uint8_t* s, int m) { - int i; -#ifdef XXXX - int n; - n = m & 3; - m >>= 2; - - for (i = 0; i < m; i++ ) - { - d[0] = xlatcase[ s[0] ]; - d[2] = xlatcase[ s[2] ]; - d[1] = xlatcase[ s[1] ]; - d[3] = xlatcase[ s[3] ]; - d+=4; - s+=4; - } - - for (i=0; i < n; i++) - { - d[i] = xlatcase[ s[i] ]; - } -#else - for (i=0; i < m; i++) + for (int i=0; i < m; i++) { d[i] = xlatcase[ s[i] ]; } - -#endif } -/* -* -*/ enum Acsm2MemoryType { ACSM2_MEMORY_TYPE__NONE = 0, @@ -236,9 +208,6 @@ enum Acsm2MemoryType ACSM2_MEMORY_TYPE__FAILSTATE }; -/* -* -*/ static void* AC_MALLOC(int n, Acsm2MemoryType type) { void* p = snort_calloc(n); @@ -344,9 +313,8 @@ static void AC_FREE_DFA(void* p, int n, int sizeofstate) } -/* - * Get Next State-NFA, using direct index to speed up search - */ +// Get Next State-NFA, using direct index to speed up search + static int List_GetNextStateOpt( ACSM_STRUCT2 * acsm, trans_node_t **acsmTransTableOpt, int state, int input ) { @@ -362,10 +330,8 @@ static int List_GetNextStateOpt( ACSM_STRUCT2 * acsm, return ACSM_FAIL_STATE2; /* Fail state ??? */ } +// Get Next State-NFA -/* -* Get Next State-NFA -*/ static int List_GetNextState(ACSM_STRUCT2* acsm, int state, int input) { trans_node_t* t = acsm->acsmTransTable[state]; @@ -385,9 +351,8 @@ static int List_GetNextState(ACSM_STRUCT2* acsm, int state, int input) return ACSM_FAIL_STATE2; /* Fail state ??? */ } -/* -* Get Next State-DFA -*/ +// Get Next State-DFA + static int List_GetNextState2(ACSM_STRUCT2* acsm, int state, int input) { trans_node_t* t = acsm->acsmTransTable[state]; @@ -404,9 +369,8 @@ static int List_GetNextState2(ACSM_STRUCT2* acsm, int state, int input) return 0; /* default state */ } -/* - * Put Next State - Head insertion, and transition updates - */ +// Put Next State - Head insertion, and transition updates + static int List_PutNextStateOpt( ACSM_STRUCT2 * acsm, trans_node_t **acsmTransTableOpt, int state, int input, int next_state ) { @@ -438,9 +402,8 @@ static int List_PutNextStateOpt( ACSM_STRUCT2 * acsm, trans_node_t **acsmTransTa return 0; } -/* -* Put Next State - Head insertion, and transition updates -*/ +// Put Next State - Head insertion, and transition updates + static int List_PutNextState(ACSM_STRUCT2* acsm, int state, int input, int next_state) { trans_node_t* p; @@ -475,9 +438,8 @@ static int List_PutNextState(ACSM_STRUCT2* acsm, int state, int input, int next_ return 0; } -/* -* Free the entire transition table -*/ +// Free the entire transition table + static int List_FreeTransTable(ACSM_STRUCT2* acsm) { int i; @@ -507,9 +469,8 @@ static int List_FreeTransTable(ACSM_STRUCT2* acsm) } -/* -* Converts row of states from list to a full vector format -*/ +// Converts row of states from list to a full vector format + static inline int List_ConvToFull(ACSM_STRUCT2* acsm, acstate_t state, acstate_t* full) { int tcnt = 0; @@ -540,9 +501,9 @@ static inline int List_ConvToFull(ACSM_STRUCT2* acsm, acstate_t state, acstate_t return tcnt; } -/* -* Copy a Match List Entry - don't dup the pattern data -*/ + +// Copy a Match List Entry - don't dup the pattern data + static ACSM_PATTERN2* CopyMatchListEntry(ACSM_PATTERN2* px) { ACSM_PATTERN2* p; @@ -554,31 +515,9 @@ static ACSM_PATTERN2* CopyMatchListEntry(ACSM_PATTERN2* px) return p; } -/* -* Check if a pattern is in the list already, -* validate it using the 'id' field. This must be unique -* for every pattern. -*/ -/* -static int FindMatchListEntry (ACSM_STRUCT2 * acsm, int state, ACSM_PATTERN2 * px) -{ - ACSM_PATTERN2 * p; +// Add a pattern to the list of patterns terminated at this state. +// Insert at front of list. - p = acsm->acsmMatchList[state]; - while( p ) - { - if( p->id == px->id ) return 1; - p = p->next; - } - - return 0; -} -*/ - -/* -* Add a pattern to the list of patterns terminated at this state. -* Insert at front of list. -*/ static void AddMatchListEntry(ACSM_STRUCT2* acsm, int state, ACSM_PATTERN2* px) { ACSM_PATTERN2* p; @@ -597,9 +536,8 @@ static void AddPatternStates(ACSM_STRUCT2* acsm, ACSM_PATTERN2* p) int n = p->n; uint8_t* pattern = p->patrn; - /* - * Match up pattern with existing states - */ + // Match up pattern with existing states + for (; n > 0; pattern++, n--) { int next = List_GetNextState(acsm,state,*pattern); @@ -610,9 +548,8 @@ static void AddPatternStates(ACSM_STRUCT2* acsm, ACSM_PATTERN2* p) state = next; } - /* - * Add new states for the rest of the pattern bytes, 1 state per byte - */ + // Add new states for the rest of the pattern bytes, 1 state per byte + for (; n > 0; pattern++, n--) { acsm->acsmNumStates++; @@ -623,10 +560,9 @@ static void AddPatternStates(ACSM_STRUCT2* acsm, ACSM_PATTERN2* p) AddMatchListEntry (acsm, state, p); } -/* -* Build A Non-Deterministic Finite Automata -* The keyword state table must already be built, via AddPatternStates(). -*/ +// Build A Non-Deterministic Finite Automata +// The keyword state table must already be built, via AddPatternStates(). + static void Build_NFA(ACSM_STRUCT2* acsm) { acstate_t* FailState = acsm->acsmFailState; @@ -708,9 +644,8 @@ static void Build_NFA(ACSM_STRUCT2* acsm) snort_free(queue_array); } -/* -* Build Deterministic Finite Automata from the NFA -*/ +// Build Deterministic Finite Automata from the NFA + static void Convert_NFA_To_DFA(ACSM_STRUCT2* acsm) { int cFailState; @@ -779,11 +714,8 @@ static void Convert_NFA_To_DFA(ACSM_STRUCT2* acsm) snort_free(acsmTransTableOpt); } -/* -* -* Convert a row lists for the state table to a full vector format -* -*/ +// Convert a row lists for the state table to a full vector format + static int Conv_List_To_Full(ACSM_STRUCT2* acsm) { acstate_t k; @@ -801,17 +733,17 @@ static int Conv_List_To_Full(ACSM_STRUCT2* acsm) { case 1: List_ConvToFull(acsm, k, (acstate_t*)((uint8_t*)p + 2)); - *((uint8_t*)p) = ACF_FULL; + *((uint8_t*)p) = 0; *((uint8_t*)p + 1) = 0; break; case 2: List_ConvToFull(acsm, k, (acstate_t*)((uint16_t*)p + 2)); - *((uint16_t*)p) = ACF_FULL; + *((uint16_t*)p) = 0; *((uint16_t*)p + 1) = 0; break; default: List_ConvToFull(acsm, k, (p + 2)); - p[0] = ACF_FULL; + p[0] = 0; p[1] = 0; /* no matches yet */ break; } @@ -822,304 +754,20 @@ static int Conv_List_To_Full(ACSM_STRUCT2* acsm) return 0; } -/* -* Convert DFA memory usage from list based storage to a sparse-row storage. -* -* The Sparse format allows each row to be either full or sparse formatted. If the sparse row has -* too many transitions, performance or space may dictate that we use the standard full formatting -* for the row. More than 5 or 10 transitions per state ought to really whack performance. So the -* user can specify the max state transitions per state allowed in the sparse format. -* -* Standard Full Matrix Format -* --------------------------- -* acstate_t ** NextState ( 1st index is row/state, 2nd index is column=event/input) -* -* example: -* -* events -> a b c d e f g h i j k l m n o p -* states -* N 1 7 0 0 0 3 0 0 0 0 0 0 0 0 0 0 -* -* Sparse Format, each row : Words Value -* 1-1 fmt(0-full,1-sparse,2-banded,3-sparsebands) -* 2-2 bool match flag (indicates this state has pattern matches) -* 3-3 sparse state count ( # of input/next-state pairs ) -* 4-3+2*cnt 'input,next-state' pairs... each sizeof(acstate_t) -* -* above example case yields: -* Full Format: 0, 1 7 0 0 0 3 0 0 0 0 0 0 0 0 0 0 ... -* Sparse format: 1, 3, 'a',1,'b',7,'f',3 - uses 2+2*ntransitions (non-default transitions) -*/ -static int Conv_Full_DFA_To_Sparse(ACSM_STRUCT2* acsm) -{ - acstate_t* p; - acstate_t** NextState = acsm->acsmNextState; - - for (int k=0; kacsmNumStates; k++) - { - int cnt=0; - acstate_t full[MAX_ALPHABET_SIZE]; - - memset(full, 0, acsm->sizeofstate * acsm->acsmAlphabetSize); - List_ConvToFull(acsm, (acstate_t)k, full); - - for (int i = 0; i < acsm->acsmAlphabetSize; i++) - { - acstate_t state = full[i]; - if ( state != 0 && state != ACSM_FAIL_STATE2 ) - cnt++; - } - - if ( k== 0 || cnt > acsm->acsmSparseMaxRowNodes ) - { - p = (acstate_t*)AC_MALLOC_DFA(sizeof(acstate_t)*(acsm->acsmAlphabetSize+2), - sizeof(acstate_t)); - if (!p) - return -1; - - p[0] = ACF_FULL; - p[1] = 0; - memcpy(&p[2],full,acsm->acsmAlphabetSize*sizeof(acstate_t)); - } - else - { - p = (acstate_t*)AC_MALLOC_DFA(sizeof(acstate_t)*(3+2*cnt), - sizeof(acstate_t)); - if (!p) - return -1; - - int m = 0; - p[m++] = ACF_SPARSE; - p[m++] = 0; /* no matches */ - p[m++] = cnt; - - for (int i = 0; i < acsm->acsmAlphabetSize; i++) - { - acstate_t state = full[i]; - if ( state != 0 && state != ACSM_FAIL_STATE2 ) - { - p[m++] = i; - p[m++] = state; - } - } - } - - NextState[k] = p; /* now we are a sparse formatted state transition array */ - } - - return 0; -} - -/* - Convert Full matrix to Banded row format. - - Word values - 1 2 -> banded - 2 n number of values - 3 i index of 1st value (0-256) - 4 - 3+n next-state values at each index - -*/ -static int Conv_Full_DFA_To_Banded(ACSM_STRUCT2* acsm) -{ - acstate_t* p, full[MAX_ALPHABET_SIZE]; - acstate_t** NextState = acsm->acsmNextState; - - for (int k=0; kacsmNumStates; k++) - { - memset(full, 0, acsm->sizeofstate * acsm->acsmAlphabetSize); - List_ConvToFull(acsm, (acstate_t)k, full); - - int first=-1; - int last =-2; - - for (int i = 0; i < acsm->acsmAlphabetSize; i++) - { - acstate_t state = full[i]; - - if ( state !=0 && state != ACSM_FAIL_STATE2 ) - { - if ( first < 0 ) - first = i; - last = i; - } - } - - /* calc band width */ - int cnt= last - first + 1; - - p = (acstate_t*)AC_MALLOC_DFA(sizeof(acstate_t)*(4+cnt), sizeof(acstate_t)); - - if (!p) - return -1; - - int m = 0; - p[m++] = ACF_BANDED; - p[m++] = 0; /* no matches */ - p[m++] = cnt; - p[m++] = first; - - for (int i = first; i <= last; i++) - { - p[m++] = full[i]; - } - - NextState[k] = p; /* now we are a banded formatted state transition array */ - } - - return 0; -} - -/* -* Convert full matrix to Sparse Band row format. -* -* next - Full formatted row of next states -* asize - size of alphabet -* zcnt - max number of zeros in a run of zeros in any given band. -* -* Word Values -* 1 ACF_SPARSE_BANDS -* 2 number of bands -* repeat 3 - 5+ ....once for each band in this row. -* 3 number of items in this band* 4 start index of this band -* 5- next-state values in this band... -*/ -static int calcSparseBands(const acstate_t* next, int* begin, int* end, int asize, int zmax) -{ - int last=0; - int nbands = 0; - - for ( int i=0; i zmax ) - break; - } - else - { - zcnt=0; - last = i; - } - } - end[nbands++] = last; - } - } - return nbands; -} +// Create a new AC full state machine -/* -* Sparse Bands -* -* Row Format: -* Word -* 1 SPARSEBANDS format indicator -* 2 bool indicates a pattern match in this state -* 3 number of sparse bands -* 4 number of elements in this band -* 5 start index of this band -* 6- list of next states -* -* m number of elements in this band -* m+1 start index of this band -* m+2- list of next states -*/ -static int Conv_Full_DFA_To_SparseBands(ACSM_STRUCT2* acsm) -{ - acstate_t** NextState = acsm->acsmNextState; - int zcnt=acsm->acsmSparseMaxZcnt; - int band_begin[MAX_ALPHABET_SIZE]; - int band_end[MAX_ALPHABET_SIZE]; - - for (int k=0; kacsmNumStates; k++) - { - acstate_t full[MAX_ALPHABET_SIZE]; - memset(full, 0, acsm->sizeofstate * acsm->acsmAlphabetSize); - List_ConvToFull(acsm, (acstate_t)k, full); - - int nbands = calcSparseBands(full, band_begin, band_end, acsm->acsmAlphabetSize, zcnt); - - /* calc band width space*/ - int cnt = 3; - - for (int i=0; i= MAX_ALPHABET_SIZE) - { - AC_FREE_DFA(p, sizeof(acstate_t)*(cnt), sizeof(acstate_t)); - return -1; - } - - p[m++] = full[j]; /* some states may be state zero */ - } - } - - NextState[k] = p; /* now we are a sparse-banded formatted state transition array */ - } - - return 0; -} - -/* -* Create a new AC state machine -*/ -ACSM_STRUCT2* acsmNew2(const MpseAgent* agent, int format) +ACSM_STRUCT2* acsmNew2(const MpseAgent* agent) { ACSM_STRUCT2* p = (ACSM_STRUCT2*)AC_MALLOC(sizeof (ACSM_STRUCT2), ACSM2_MEMORY_TYPE__NONE); - if (p) - { - p->agent = agent; - p->acsmFormat = format; - - /* Some defaults */ - p->acsmAlphabetSize = 256; - p->acsmSparseMaxRowNodes = 256; - p->acsmSparseMaxZcnt = 10; - p->dfa = false; - } + p->agent = agent; + p->acsmAlphabetSize = 256; return p; } -/* -* Add a pattern to the list of patterns for this state machine -* -*/ +// Add a pattern to the list of patterns for this state machine + int acsmAddPattern2( ACSM_STRUCT2* p, const uint8_t* pat, unsigned n, bool nocase, bool negative, void* user) @@ -1151,9 +799,8 @@ int acsmAddPattern2( return 0; } -/* -* Copy a boolean match flag int NextState table, for caching purposes. -*/ +// Copy a boolean match flag int NextState table, for caching purposes. + static void acsmUpdateMatchStates(ACSM_STRUCT2* acsm) { acstate_t state; @@ -1215,19 +862,8 @@ static void acsmBuildMatchStateTrees2(SnortConfig* sc, ACSM_STRUCT2* acsm) } } -void acsmCompressStates( - ACSM_STRUCT2* acsm, - int flag - ) -{ - if (acsm == nullptr) - return; - acsm->compress_states = flag; -} +// Compile State Machine - NFA or DFA and Full -/* -* Compile State Machine - NFA or DFA and Full or Banded or Sparse or SparseBands -*/ static inline int _acsmCompile2(ACSM_STRUCT2* acsm) { ACSM_PATTERN2* plist; @@ -1262,27 +898,20 @@ static inline int _acsmCompile2(ACSM_STRUCT2* acsm) /* Add the 0'th state */ acsm->acsmNumStates++; - if (acsm->compress_states) + if (acsm->acsmNumStates < UINT8_MAX) { - if (acsm->acsmNumStates < UINT8_MAX) - { - acsm->sizeofstate = 1; - summary.num_1byte_instances++; - } - else if (acsm->acsmNumStates < UINT16_MAX) - { - acsm->sizeofstate = 2; - summary.num_2byte_instances++; - } - else - { - acsm->sizeofstate = 4; - summary.num_4byte_instances++; - } + acsm->sizeofstate = 1; + summary.num_1byte_instances++; + } + else if (acsm->acsmNumStates < UINT16_MAX) + { + acsm->sizeofstate = 2; + summary.num_2byte_instances++; } else { acsm->sizeofstate = 4; + summary.num_4byte_instances++; } /* Alloc a failure table - this has a failure state, and a match list for each state */ @@ -1295,45 +924,15 @@ static inline int _acsmCompile2(ACSM_STRUCT2* acsm) acsm->acsmNextState = (acstate_t**)AC_MALLOC_DFA(acsm->acsmNumStates * sizeof(acstate_t*), acsm->sizeofstate); - /* Build the NFA */ Build_NFA(acsm); + Convert_NFA_To_DFA(acsm); - if ( acsm->dfa ) - { - /* Convert the NFA to a DFA */ - Convert_NFA_To_DFA(acsm); - - /* Don't need the FailState table anymore */ - AC_FREE(acsm->acsmFailState, sizeof(acstate_t) * acsm->acsmNumStates, - ACSM2_MEMORY_TYPE__FAILSTATE); + /* Don't need the FailState table anymore */ + AC_FREE(acsm->acsmFailState, sizeof(acstate_t) * acsm->acsmNumStates, ACSM2_MEMORY_TYPE__FAILSTATE); + acsm->acsmFailState = nullptr; - acsm->acsmFailState = nullptr; - } - - /* Select Final Transition Table Storage Mode */ - if ( acsm->acsmFormat == ACF_SPARSE ) - { - /* Convert DFA Full matrix to a Sparse matrix */ - if (Conv_Full_DFA_To_Sparse(acsm)) - return -1; - } - else if ( acsm->acsmFormat == ACF_BANDED ) - { - /* Convert DFA Full matrix to a Sparse matrix */ - if ( Conv_Full_DFA_To_Banded(acsm) ) - return -1; - } - else if ( acsm->acsmFormat == ACF_SPARSE_BANDS ) - { - /* Convert DFA Full matrix to a Sparse matrix */ - if ( Conv_Full_DFA_To_SparseBands(acsm) ) - return -1; - } - else if ( acsm->acsmFormat == ACF_FULL ) - { - if ( Conv_List_To_Full(acsm) ) - return -1; - } + if ( Conv_List_To_Full(acsm) ) + return -1; /* load boolean match flags into state table */ acsmUpdateMatchStates(acsm); @@ -1362,255 +961,6 @@ int acsmCompile2(SnortConfig* sc, ACSM_STRUCT2* acsm) return 0; } -/* -* Get the NextState from the NFA, all NFA storage formats use this -*/ -static inline acstate_t get_next_state_nfa(acstate_t* ps, acstate_t state, unsigned input) -{ - acstate_t fmt = *ps++; - - ps++; /* skip bMatchState */ - - switch ( fmt ) - { - case ACF_BANDED: - { - acstate_t n = ps[0]; - unsigned index = ps[1]; - - if ( input < index ) - { - if (state==0) - { - return 0; - } - else - { - return (acstate_t)ACSM_FAIL_STATE2; - } - } - if ( input >= index + n ) - { - if (state==0) - { - return 0; - } - else - { - return (acstate_t)ACSM_FAIL_STATE2; - } - } - if ( ps[input-index] == 0 ) - { - if ( state != 0 ) - { - return ACSM_FAIL_STATE2; - } - } - - return (acstate_t)ps[input-index]; - } - - case ACF_SPARSE: - { - acstate_t n = *ps++; /* number of sparse index-value entries */ - - for (; n>0; n-- ) - { - if ( ps[0] > input ) /* cannot match the input, already a higher value than the input - */ - { - return (acstate_t)ACSM_FAIL_STATE2; /* default state */ - } - else if ( ps[0] == input ) - { - return ps[1]; /* next state */ - } - ps+=2; - } - if ( state == 0 ) - { - return 0; - } - return ACSM_FAIL_STATE2; - } - - case ACF_SPARSE_BANDS: - { - int nb = *ps++; /* number of bands */ - - while ( nb > 0 ) /* for each band */ - { - acstate_t n = *ps++; /* number of elements */ - unsigned index = *ps++; /* 1st element value */ - - if ( input < index ) - { - if ( state != 0 ) - { - return (acstate_t)ACSM_FAIL_STATE2; - } - return (acstate_t)0; - } - if ( (input >= index) && (input < (index + n)) ) - { - if ( ps[input-index] == 0 ) - { - if ( state != 0 ) - { - return ACSM_FAIL_STATE2; - } - } - return (acstate_t)ps[input-index]; - } - nb--; - ps += n; - } - if ( state != 0 ) - { - return (acstate_t)ACSM_FAIL_STATE2; - } - return (acstate_t)0; - } - - case ACF_FULL: - { - if ( ps[input] == 0 ) - { - if ( state != 0 ) - { - return ACSM_FAIL_STATE2; - } - } - return ps[input]; - } - } - - return 0; -} - -/* -* Get the NextState from the DFA Next State Transition table -* Full and banded are supported separately, this is for -* sparse and sparse-bands. But note that for optimization -* purposes, a given row may be full. -*/ -static inline acstate_t SparseGetNextStateDFA( - acstate_t* ps, acstate_t, unsigned input) -{ - switch ( ps[0] ) - { - case ACF_FULL: - { - return ps[2+input]; - } - - case ACF_SPARSE: - { - acstate_t n = ps[2]; /* number of entries/ key+next pairs */ - ps += 3; - - for (; n>0; n-- ) - { - if ( input < ps[0] ) /* cannot match the input, already a higher value than the - input */ - { - return (acstate_t)0; /* default state */ - } - else if ( ps[0] == input ) - { - return ps[1]; /* next state */ - } - ps += 2; - } - return (acstate_t)0; - } - - case ACF_SPARSE_BANDS: - { - acstate_t nb = ps[2]; /* number of bands */ - - ps += 3; - - while ( nb > 0 ) /* for each band */ - { - acstate_t n = ps[0]; /* number of elements in this band */ - unsigned index = ps[1]; /* start index/char of this band */ - - if ( input < index ) - { - return (acstate_t)0; - } - if ( (input < (index + n)) ) - { - return (acstate_t)ps[2+input-index]; - } - nb--; - ps += 2 + n; - } - return (acstate_t)0; - } - } - - return 0; -} - -/* -* Search Text or Binary Data for Pattern matches -* -* Sparse & Sparse-Banded Matrix search -*/ -int acsm_search_dfa_sparse( - ACSM_STRUCT2* acsm, const uint8_t* Tx, int n, MpseMatch match, - void* context, int* current_state) -{ - acstate_t state; - ACSM_PATTERN2* mlist; - const uint8_t* Tend; - int nfound = 0; - const uint8_t* T, * Tc; - int index; - acstate_t** NextState = acsm->acsmNextState; - ACSM_PATTERN2** MatchList = acsm->acsmMatchList; - - Tc = Tx; - T = Tx; - Tend = T + n; - - if ( !current_state ) - { - return 0; - } - - state = *current_state; - - for (; T < Tend; T++ ) - { - state = SparseGetNextStateDFA (NextState[state], state, xlatcase[*T]); - - /* test if this state has any matching patterns */ - if ( NextState[state][1] ) - { - mlist = MatchList[state]; - if (mlist) - { - index = T - Tc + 1; - nfound++; - if (match (mlist->udata, mlist->rule_option_tree, index, context, - mlist->neg_list) > 0) - { - *current_state = state; - return nfound; - } - } - } - } - - *current_state = state; - - return nfound; -} - void acsmx2_print_qinfo() { } @@ -1817,134 +1167,8 @@ int acsm_search_dfa_full_all( return nfound; } -/* -* Banded-Row format DFA search -* Do not change anything here, caching and prefetching -* performance is very sensitive to any changes. -* -* ps[0] = storage fmt -* ps[1] = bool match flag -* ps[2] = # elements in band -* ps[3] = index of 1st element -*/ -int acsm_search_dfa_banded( - ACSM_STRUCT2* acsm, const uint8_t* Tx, int n, MpseMatch match, - void* context, int* current_state) -{ - acstate_t** NextState = acsm->acsmNextState; - ACSM_PATTERN2** MatchList = acsm->acsmMatchList; - ACSM_PATTERN2* mlist; - int nfound = 0; - - if ( !current_state ) - { - return 0; - } - - acstate_t state = *current_state; - - const uint8_t* T = Tx; - const uint8_t* Tend = T + n; - - for (; T < Tend; T++ ) - { - acstate_t* ps = NextState[state]; - int sindex = xlatcase[ T[0] ]; - - /* test if this state has any matching patterns */ - if ( ps[1] ) - { - mlist = MatchList[state]; - if (mlist) - { - int index = T - Tx; - nfound++; - - if (match (mlist->udata, mlist->rule_option_tree, index, context, - mlist->neg_list) > 0) - { - *current_state = state; - return nfound; - } - } - } - - if ( (acstate_t)sindex < ps[3] ) - state = 0; - else if ( (acstate_t)sindex >= (ps[3] + ps[2]) ) - state = 0; - else - state = ps[ 4u + sindex - ps[3] ]; - } - - /* Check the last state for a pattern match */ - mlist = MatchList[state]; - if (mlist) - { - int index = T - Tx; - nfound++; - - if (match (mlist->udata, mlist->rule_option_tree, index, context, mlist->neg_list) > 0) - { - *current_state = state; - return nfound; - } - } - - return nfound; -} - -// Search Text or Binary Data for Pattern matches +// Free all memory -int acsm_search_nfa( - ACSM_STRUCT2* acsm, const uint8_t* Tx, int n, MpseMatch match, - void* context, int* current_state) -{ - int nfound = 0; - acstate_t** NextState= acsm->acsmNextState; - acstate_t* FailState = acsm->acsmFailState; - ACSM_PATTERN2** MatchList = acsm->acsmMatchList; - - if ( !current_state ) - { - return 0; - } - - acstate_t state = *current_state; - - const uint8_t* T = Tx; - const uint8_t* Tend = T + n; - - for (; T < Tend; T++ ) - { - uint8_t Tchar = xlatcase[ *T ]; - acstate_t nstate; - - while ( (nstate=get_next_state_nfa(NextState[state],state,Tchar))==ACSM_FAIL_STATE2 ) - state = FailState[state]; - - state = nstate; - - ACSM_PATTERN2* mlist = MatchList[state]; - - if (mlist) - { - int index = T - Tx + 1; - nfound++; - if (match (mlist->udata, mlist->rule_option_tree, index, context, mlist->neg_list) > 0) - { - *current_state = state; - return nfound; - } - } - } - - return nfound; -} - -/* -* Free all memory -*/ void acsmFree2(ACSM_STRUCT2* acsm) { int i; @@ -2013,7 +1237,7 @@ static void Print_DFA_MatchList(ACSM_STRUCT2* acsm, int state) static void Print_DFA(ACSM_STRUCT2* acsm) { int k,i; - acstate_t* p, state, n, fmt, index, nb; + acstate_t* p, state, fmt; acstate_t** NextState = acsm->acsmNextState; printf("Print DFA - %d active states\n",acsm->acsmNumStates); @@ -2029,59 +1253,16 @@ static void Print_DFA(ACSM_STRUCT2* acsm) printf("state %3d, fmt=%d: ",k,fmt); - if ( fmt ==ACF_SPARSE ) - { - n = *p++; - for (; n>0; n--, p+=2 ) - { - if ( isascii((int)p[0]) && isprint((int)p[0]) ) - printf("%3c->%-5d\t",p[0],p[1]); - else - printf("%3d->%-5d\t",p[0],p[1]); - } - } - else if ( fmt ==ACF_BANDED ) + for ( i=0; iacsmAlphabetSize; i++ ) { - n = *p++; - index = *p++; + state = p[i]; - for (; n>0; n--, p++ ) + if ( state != 0 && state != ACSM_FAIL_STATE2 ) { - if ( isascii((int)p[0]) && isprint((int)p[0]) ) - printf("%3c->%-5d\t",index++,p[0]); + if ( isascii(i) && isprint(i) ) + printf("%3c->%-5d\t",i,state); else - printf("%3d->%-5d\t",index++,p[0]); - } - } - else if ( fmt ==ACF_SPARSE_BANDS ) - { - nb = *p++; - for (i=0; (acstate_t)i0; n--, p++ ) - { - if ( isascii((int)index) && isprint((int)index) ) - printf("%3c->%-5d\t",index++,p[0]); - else - printf("%3d->%-5d\t",index++,p[0]); - } - } - } - else if ( fmt == ACF_FULL ) - { - for ( i=0; iacsmAlphabetSize; i++ ) - { - state = p[i]; - - if ( state != 0 && state != ACSM_FAIL_STATE2 ) - { - if ( isascii(i) && isprint(i) ) - printf("%3c->%-5d\t",i,state); - else - printf("%3d->%-5d\t",i,state); - } + printf("%3d->%-5d\t",i,state); } } @@ -2106,21 +1287,13 @@ int acsmPrintDetailInfo2(ACSM_STRUCT2* acsm) int acsmPrintSummaryInfo2() { - const char* sf[]= - { - "full", - "sparse", - "banded", - "sparse-bands", - }; - ACSM_STRUCT2* p = &summary.acsm; if ( !summary.num_states ) return 0; - LogValue("storage format", sf[p->acsmFormat]); - LogValue("finite automaton", p->dfa ? "DFA" : "NFA"); + LogValue("storage format", "full"); + LogValue("finite automaton", "DFA"); LogCount("alphabet size", p->acsmAlphabetSize); LogCount("instances", summary.num_instances); @@ -2131,21 +1304,16 @@ int acsmPrintSummaryInfo2() LogCount("transitions", summary.num_transitions); LogCount("match states", summary.num_match_states); - if ( !summary.acsm.compress_states ) - LogCount("sizeof state", (int)(sizeof(acstate_t))); - else - { - LogValue("sizeof state", "1, 2, or 4"); + LogValue("sizeof state", "1, 2, or 4"); - if ( summary.num_1byte_instances ) - LogCount("1 byte states", summary.num_1byte_instances); + if ( summary.num_1byte_instances ) + LogCount("1 byte states", summary.num_1byte_instances); - if ( summary.num_2byte_instances ) - LogCount("2 byte states", summary.num_2byte_instances); + if ( summary.num_2byte_instances ) + LogCount("2 byte states", summary.num_2byte_instances); - if ( summary.num_4byte_instances ) - LogCount("4 byte states", summary.num_4byte_instances); - } + if ( summary.num_4byte_instances ) + LogCount("4 byte states", summary.num_4byte_instances); double scale; @@ -2168,17 +1336,10 @@ int acsmPrintSummaryInfo2() #if 0 // FIXIT-L clean up format; not all this should be printed all the time if (acsm2_dfa_memory > 0) { - if (summary.acsm.compress_states) - { - LogMessage("| DFA\n"); - LogMessage("| 1 byte states : %.2f\n", acsm2_dfa1_memory/scale); - LogMessage("| 2 byte states : %.2f\n", acsm2_dfa2_memory/scale); - LogMessage("| 4 byte states : %.2f\n", acsm2_dfa4_memory/scale); - } - else - { - LogMessage("| DFA : %.2f\n", acsm2_dfa_memory/scale); - } + LogMessage("| DFA\n"); + LogMessage("| 1 byte states : %.2f\n", acsm2_dfa1_memory/scale); + LogMessage("| 2 byte states : %.2f\n", acsm2_dfa2_memory/scale); + LogMessage("| 4 byte states : %.2f\n", acsm2_dfa4_memory/scale); } #endif diff --git a/src/search_engines/acsmx2.h b/src/search_engines/acsmx2.h index c85ac2b64..77b14deb2 100644 --- a/src/search_engines/acsmx2.h +++ b/src/search_engines/acsmx2.h @@ -85,17 +85,6 @@ struct trans_node_t trans_node_t* next; /* next transition for this state */ }; -/* -* User specified final storage type for the state transitions -*/ -enum -{ - ACF_FULL, - ACF_SPARSE, - ACF_BANDED, - ACF_SPARSE_BANDS, -}; - /* * Aho-Corasick State Machine Struct - one per group of patterns */ @@ -115,24 +104,11 @@ struct ACSM_STRUCT2 int acsmMaxStates; int acsmNumStates; - int acsmFormat; - int acsmSparseMaxRowNodes; - int acsmSparseMaxZcnt; - int acsmNumTrans; int acsmAlphabetSize; int numPatterns; int sizeofstate; - int compress_states; - - bool dfa; - - void enable_dfa() - { dfa = true; } - - bool dfa_enabled() - { return dfa; } }; /* @@ -140,7 +116,7 @@ struct ACSM_STRUCT2 */ void acsmx2_init_xlatcase(); -ACSM_STRUCT2* acsmNew2(const MpseAgent*, int format); +ACSM_STRUCT2* acsmNew2(const MpseAgent*); int acsmAddPattern2( ACSM_STRUCT2* p, const uint8_t* pat, unsigned n, @@ -151,12 +127,6 @@ int acsmCompile2(snort::SnortConfig*, ACSM_STRUCT2*); int acsm_search_nfa( ACSM_STRUCT2*, const uint8_t* T, int n, MpseMatch, void* context, int* current_state); -int acsm_search_dfa_sparse( - ACSM_STRUCT2*, const uint8_t* T, int n, MpseMatch, void* context, int* current_state); - -int acsm_search_dfa_banded( - ACSM_STRUCT2*, const uint8_t* T, int n, MpseMatch, void* context, int* current_state); - int acsm_search_dfa_full( ACSM_STRUCT2*, const uint8_t* T, int n, MpseMatch, void* context, int* current_state); @@ -165,7 +135,6 @@ int acsm_search_dfa_full_all( void acsmFree2(ACSM_STRUCT2*); int acsmPatternCount2(ACSM_STRUCT2*); -void acsmCompressStates(ACSM_STRUCT2*, int); void acsmPrintInfo2(ACSM_STRUCT2* p); diff --git a/src/search_engines/acsmx2_api.cc b/src/search_engines/acsmx2_api.cc index c6bfefe1a..e0e1f50bd 100644 --- a/src/search_engines/acsmx2_api.cc +++ b/src/search_engines/acsmx2_api.cc @@ -27,10 +27,7 @@ using namespace snort; -extern const BaseApi* se_ac_banded; extern const BaseApi* se_ac_full; -extern const BaseApi* se_ac_sparse; -extern const BaseApi* se_ac_sparse_bands; #ifdef BUILDING_SO SO_PUBLIC const BaseApi* snort_plugins[] = @@ -38,10 +35,7 @@ SO_PUBLIC const BaseApi* snort_plugins[] = const BaseApi* se_acsmx2[] = #endif { - se_ac_banded, se_ac_full, - se_ac_sparse, - se_ac_sparse_bands, nullptr }; diff --git a/src/search_engines/bnfa_search.cc b/src/search_engines/bnfa_search.cc index cca01a90a..e81028af6 100644 --- a/src/search_engines/bnfa_search.cc +++ b/src/search_engines/bnfa_search.cc @@ -782,8 +782,7 @@ static int _bnfa_build_nfa(bnfa_struct_t* bnfa) #ifdef ENABLE_BNFA_FAIL_STATE_OPT // FIXIT-L low priority performance issue: bnfa fail state reduction // optimize the failure states - if ( bnfa->bnfaOpt ) - _bnfa_opt_nfa(bnfa); + _bnfa_opt_nfa(bnfa); #endif return 0; @@ -1190,7 +1189,6 @@ bnfa_struct_t* bnfaNew(const MpseAgent* agent) if ( p ) { - p->bnfaOpt = 0; p->bnfaCaseMode = BNFA_PER_PAT_CASE; p->bnfaFormat = BNFA_SPARSE; p->bnfaAlphabetSize = BNFA_MAX_ALPHABET_SIZE; @@ -1202,11 +1200,6 @@ bnfa_struct_t* bnfaNew(const MpseAgent* agent) return p; } -void bnfaSetOpt(bnfa_struct_t* p, int flag) -{ - p->bnfaOpt=flag; -} - void bnfaSetCase(bnfa_struct_t* p, int flag) { if ( flag == BNFA_PER_PAT_CASE ) diff --git a/src/search_engines/bnfa_search.h b/src/search_engines/bnfa_search.h index 832ce67d7..98f63d5d1 100644 --- a/src/search_engines/bnfa_search.h +++ b/src/search_engines/bnfa_search.h @@ -160,7 +160,6 @@ void bnfa_init_xlatcase(); bnfa_struct_t* bnfaNew(const MpseAgent*); -void bnfaSetOpt(bnfa_struct_t* p, int flag); void bnfaSetCase(bnfa_struct_t* p, int flag); void bnfaFree(bnfa_struct_t* pstruct); diff --git a/src/search_engines/hyperscan.cc b/src/search_engines/hyperscan.cc index fa59a27c5..0cc9f8d58 100644 --- a/src/search_engines/hyperscan.cc +++ b/src/search_engines/hyperscan.cc @@ -76,7 +76,8 @@ Pattern::Pattern( if ( no_case ) flags |= HS_FLAG_CASELESS; - flags |= HS_FLAG_SINGLEMATCH; + if ( !d.multi_match ) + flags |= HS_FLAG_SINGLEMATCH; } void Pattern::escape(const uint8_t* s, unsigned n, bool literal) @@ -177,11 +178,8 @@ public: unsigned id, unsigned long long from, unsigned long long to, unsigned flags, void*); - bool serialize(uint8_t*& buf, size_t& sz) const override - { return hs_db and (hs_serialize_database(hs_db, (char**)&buf, &sz) == HS_SUCCESS) and buf; } - - bool deserialize(const uint8_t* buf, size_t sz) override - { return (hs_deserialize_database((const char*)buf, sz, &hs_db) == HS_SUCCESS) and hs_db; } + bool serialize(uint8_t*&, size_t&) const override; + bool deserialize(const uint8_t*, size_t) override; void get_hash(std::string&) override; @@ -202,6 +200,22 @@ public: uint64_t HyperscanMpse::instances = 0; uint64_t HyperscanMpse::patterns = 0; +bool HyperscanMpse::serialize(uint8_t*& buf, size_t& sz) const +{ return hs_db and (hs_serialize_database(hs_db, (char**)&buf, &sz) == HS_SUCCESS) and buf; } + +bool HyperscanMpse::deserialize(const uint8_t* buf, size_t sz) +{ + if ( hs_deserialize_database((const char*)buf, sz, &hs_db) != HS_SUCCESS or !hs_db ) + return false; + + if ( hs_error_t err = hs_alloc_scratch(hs_db, &s_scratch[get_instance_id()]) ) + { + ParseError("can't allocate search scratch space (%d)", err); + return false; + } + return true; +} + // other mpse have direct access to their fsm match states and populate // user list and tree with each pattern that leads to the same match state. // with hyperscan we don't have internal fsm knowledge and each fast diff --git a/src/search_engines/search_engines.cc b/src/search_engines/search_engines.cc index 6ccda318b..1a3fc4aab 100644 --- a/src/search_engines/search_engines.cc +++ b/src/search_engines/search_engines.cc @@ -27,10 +27,9 @@ using namespace snort; extern const BaseApi* se_ac_bnfa[]; +extern const BaseApi* se_ac_full[]; #ifdef STATIC_SEARCH_ENGINES -extern const BaseApi* se_ac_std[]; -extern const BaseApi* se_acsmx2[]; #ifdef HAVE_HYPERSCAN extern const BaseApi* se_hyperscan[]; #endif @@ -39,10 +38,9 @@ extern const BaseApi* se_hyperscan[]; void load_search_engines() { PluginManager::load_plugins(se_ac_bnfa); + PluginManager::load_plugins(se_ac_full); #ifdef STATIC_SEARCH_ENGINES - PluginManager::load_plugins(se_ac_std); - PluginManager::load_plugins(se_acsmx2); #ifdef HAVE_HYPERSCAN PluginManager::load_plugins(se_hyperscan); #endif diff --git a/src/search_engines/search_tool.cc b/src/search_engines/search_tool.cc index ea56bd2c6..1e972cb3c 100644 --- a/src/search_engines/search_tool.cc +++ b/src/search_engines/search_tool.cc @@ -32,54 +32,20 @@ namespace snort { -const SnortConfig* SearchTool::conf = nullptr; - -SearchTool::SearchTool(const char* method, bool dfa) +SearchTool::SearchTool(bool multi) { - mpsegrp = new MpseGroup; - - // When no method is passed in, a normal search engine mpse will be created with the search - // method the same as that configured for the fast pattern normal search method, and also an - // offload search engine mpse will be created with the search method the same as that - // configured for the fast pattern offload search method. If a method is passed in then an - // offload search engine will not be created - - // FIXIT-L the above is flawed. Offload should not depend on default. Also, offload only - // works in an inline, synchronous fashion for SearchTool so the overhead must be offset by - // the speed gain. Offload support should be removed from here for now. - - // FIXIT-L we force non-hyperscan to be ac_full since the other algorithms like ac_bnfa don't - // return all matches. This could be fixed by adding the match iteration loops like ac_full to - // ac_bnfa or removing that and updating SearchTool to do a trivial vector of matches. - - assert(conf); - - if ( !method ) - { - method = conf->fast_pattern_config->get_search_method(); - - if ( strcmp(method, "hyperscan") ) - { - method = "ac_full"; - dfa = true; - } - } + const SnortConfig* sc = SnortConfig::get_conf(); + assert(sc and sc->fast_pattern_config); + const char* method = sc->fast_pattern_config->get_search_method(); - if (mpsegrp->create_normal_mpse(conf, method)) - { - if( dfa ) - mpsegrp->normal_mpse->set_opt(1); - } + if ( strcmp(method, "hyperscan") ) + method = "ac_full"; - if (method == nullptr) - { - if (mpsegrp->create_offload_mpse(conf)) - { - if ( dfa ) - mpsegrp->offload_mpse->set_opt(1); - } - } + mpsegrp = new MpseGroup; + mpsegrp->create_normal_mpse(sc, method); + assert(mpsegrp->get_normal_mpse()); + multi_match = multi; max_len = 0; } @@ -99,7 +65,7 @@ void SearchTool::add(const uint8_t* pat, unsigned len, int id, bool no_case) void SearchTool::add(const uint8_t* pat, unsigned len, void* id, bool no_case) { - Mpse::PatternDescriptor desc(no_case, false, true); + Mpse::PatternDescriptor desc(no_case, false, true, multi_match); if ( mpsegrp->normal_mpse ) mpsegrp->normal_mpse->add_pattern(pat, len, desc, id); @@ -143,7 +109,7 @@ int SearchTool::find( user_data = (void*)str; if ( fp and fp->get_offload_search_api() and (len >= sc->offload_limit) and - (mpsegrp->get_offload_mpse() != mpsegrp->get_normal_mpse()) ) + (mpsegrp->get_offload_mpse() != mpsegrp->get_normal_mpse()) ) { num = mpsegrp->get_offload_mpse()->search((const uint8_t*)str, len, mf, user_data, &state); @@ -185,18 +151,15 @@ int SearchTool::find_all( int state = 0; if ( fp and fp->get_offload_search_api() and (len >= sc->offload_limit) and - (mpsegrp->get_offload_mpse() != mpsegrp->get_normal_mpse()) ) + (mpsegrp->get_offload_mpse() != mpsegrp->get_normal_mpse()) ) { - num = mpsegrp->get_offload_mpse()->search_all((const uint8_t*)str, len, mf, user_data, - &state); + num = mpsegrp->get_offload_mpse()->search_all((const uint8_t*)str, len, mf, user_data, &state); if ( num < 0 ) - num = mpsegrp->get_normal_mpse()->search_all((const uint8_t*)str, len, mf, user_data, - &state); + num = mpsegrp->get_normal_mpse()->search_all((const uint8_t*)str, len, mf, user_data, &state); } else - num = mpsegrp->get_normal_mpse()->search_all((const uint8_t*)str, len, mf, user_data, - &state); + num = mpsegrp->get_normal_mpse()->search_all((const uint8_t*)str, len, mf, user_data, &state); // SeachTool::find expects the number found to be returned so if we have a failure return 0 if ( num < 0 ) @@ -205,3 +168,4 @@ int SearchTool::find_all( return num; } } // namespace snort + diff --git a/src/search_engines/search_tool.h b/src/search_engines/search_tool.h index bc9546c1e..2365214d8 100644 --- a/src/search_engines/search_tool.h +++ b/src/search_engines/search_tool.h @@ -22,17 +22,22 @@ #include "framework/mpse.h" +// FIXIT-L until APIs are updated, SearchTool depends on SnortConfig so it must +// not be instantiated until configure time (Inspector::configure) to ensure +// SnortConfig is fully initialized and the configured algorithm is used. + +// Use hyperscan if configured with search_engine.search_method else use ac_full. +// Offload is not supported for search tool. + +// We force non-hyperscan to be ac_full since the other algorithms like ac_bnfa +// don't implement search_all, which returns all patterns for a given match state. + namespace snort { class SO_PUBLIC SearchTool { public: - // FIXIT-L SnortConfig should be passed to ctor, a lot of appid plumbing - // for now set_conf must be called before instantiation - static void set_conf(const SnortConfig* sc) - { conf = sc; } - - SearchTool(const char* method = nullptr, bool dfa = false); + SearchTool(bool multi_match = true); ~SearchTool(); void add(const char* pattern, unsigned len, int s_id, bool no_case = true); @@ -55,9 +60,9 @@ public: bool confine = false, void* user_data = nullptr); private: - static const SnortConfig* conf; class MpseGroup* mpsegrp; unsigned max_len; + bool multi_match; }; } // namespace snort #endif diff --git a/src/search_engines/test/CMakeLists.txt b/src/search_engines/test/CMakeLists.txt index 6769b4cb5..a01686db7 100644 --- a/src/search_engines/test/CMakeLists.txt +++ b/src/search_engines/test/CMakeLists.txt @@ -1,20 +1,47 @@ -add_cpputest( search_tool_test +add_cpputest( ac_bnfa_test SOURCES + mpse_test_stubs.cc + mpse_test_stubs.h ../ac_bnfa.cc + ../bnfa_search.cc + ../search_tool.cc + ../../framework/mpse.cc +) + +add_cpputest( search_tool_test + SOURCES + mpse_test_stubs.cc + mpse_test_stubs.h ../ac_full.cc ../acsmx2.cc - ../bnfa_search.cc ../search_tool.cc + ../../framework/mpse.cc ) if ( HAVE_HYPERSCAN ) add_cpputest( hyperscan_test SOURCES + mpse_test_stubs.cc + mpse_test_stubs.h ../hyperscan.cc ../../framework/module.cc + ../../framework/mpse.cc + ../../helpers/scratch_allocator.cc + ../../helpers/hyper_scratch_allocator.cc + LIBS ${HS_LIBRARIES} + ) + add_cpputest( hyper_tool_test + SOURCES + mpse_test_stubs.cc + mpse_test_stubs.h + ../hyperscan.cc + ../search_tool.cc + ../../framework/module.cc + ../../framework/mpse.cc ../../helpers/scratch_allocator.cc ../../helpers/hyper_scratch_allocator.cc LIBS ${HS_LIBRARIES} ) endif() + diff --git a/src/search_engines/test/ac_bnfa_test.cc b/src/search_engines/test/ac_bnfa_test.cc new file mode 100644 index 000000000..8e83520c6 --- /dev/null +++ b/src/search_engines/test/ac_bnfa_test.cc @@ -0,0 +1,220 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// ac_bnfa_test.cc author Russ Combs + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "framework/base_api.h" +#include "framework/counts.h" +#include "framework/mpse.h" +#include "framework/mpse_batch.h" +#include "main/snort_config.h" + +#include "mpse_test_stubs.h" + +// must appear after snort_config.h to avoid broken c++ map include +#include +#include + +using namespace snort; + +//------------------------------------------------------------------------- +// stubs, spies, etc. +//------------------------------------------------------------------------- + +const MpseApi* get_test_api() +{ return nullptr; } + +static unsigned hits = 0; + +static int match( + void* /*user*/, void* /*tree*/, int /*index*/, void* /*context*/, void* /*list*/) +{ + ++hits; + return 0; +} + +//------------------------------------------------------------------------- +// fp tests +//------------------------------------------------------------------------- + +TEST_GROUP(mpse_bnfa_match) +{ + const MpseApi* mpse_api = (const MpseApi*)se_ac_bnfa; + Mpse* bnfa = nullptr; + + void setup() override + { + CHECK(se_ac_bnfa); + bnfa = mpse_api->ctor(snort_conf, nullptr, &s_agent); + CHECK(bnfa); + hits = 0; + parse_errors = 0; + } + void teardown() override + { + mpse_api->dtor(bnfa); + } +}; + +TEST(mpse_bnfa_match, empty) +{ + CHECK(bnfa->prep_patterns(snort_conf) == 0); + CHECK(parse_errors == 0); + CHECK(bnfa->get_pattern_count() == 0); + + int state = 0; + CHECK(bnfa->search((const uint8_t*)"foo", 3, match, nullptr, &state) == 0); + CHECK(hits == 0); +} + +TEST(mpse_bnfa_match, single) +{ + Mpse::PatternDescriptor desc; + + CHECK(bnfa->add_pattern((const uint8_t*)"foo", 3, desc, s_user) == 0); + CHECK(bnfa->prep_patterns(snort_conf) == 0); + CHECK(bnfa->get_pattern_count() == 1); + + int state = 0; + CHECK(bnfa->search((const uint8_t*)"foo", 3, match, nullptr, &state) == 1); + CHECK(hits == 1); +} + +TEST(mpse_bnfa_match, nocase) +{ + Mpse::PatternDescriptor desc(true, true, false); + + CHECK(bnfa->add_pattern((const uint8_t*)"foo", 3, desc, s_user) == 0); + CHECK(bnfa->prep_patterns(snort_conf) == 0); + CHECK(bnfa->get_pattern_count() == 1); + + int state = 0; + CHECK(bnfa->search((const uint8_t*)"foo", 3, match, nullptr, &state) == 1); + CHECK(bnfa->search((const uint8_t*)"fOo", 3, match, nullptr, &state) == 1); + CHECK(hits == 2); +} + +TEST(mpse_bnfa_match, other) +{ + Mpse::PatternDescriptor desc(false, true, false); + + CHECK(bnfa->add_pattern((const uint8_t*)"foo", 3, desc, s_user) == 0); + CHECK(bnfa->prep_patterns(snort_conf) == 0); + CHECK(bnfa->get_pattern_count() == 1); + + int state = 0; + CHECK(bnfa->search((const uint8_t*)"foo", 3, match, nullptr, &state) == 1); + CHECK(bnfa->search((const uint8_t*)"fOo", 3, match, nullptr, &state) == 1); + CHECK(hits == 2); +} + +TEST(mpse_bnfa_match, multi) +{ + Mpse::PatternDescriptor desc; + + CHECK(bnfa->add_pattern((const uint8_t*)"foo", 3, desc, s_user) == 0); + CHECK(bnfa->add_pattern((const uint8_t*)"bar", 3, desc, s_user) == 0); + CHECK(bnfa->add_pattern((const uint8_t*)"baz", 3, desc, s_user) == 0); + + CHECK(bnfa->prep_patterns(snort_conf) == 0); + CHECK(bnfa->get_pattern_count() == 3); + + // __STRDUMP_DISABLE__ + // ac_bnfa deficiency: does not match the final "baz" + // more generally, won't match repeated pattern w/o other pattern between + // such as "foo barfoo" + // this is ok for fp_detect but unacceptable for SearchTool + int state = 0; + CHECK(bnfa->search((const uint8_t*)"foo barfoo bazookibaz", 17, match, nullptr, &state) == 4); + // __STRDUMP_ENABLE__ + CHECK(hits == 4); +} + +//------------------------------------------------------------------------- +// multi fp tests +//------------------------------------------------------------------------- + +TEST_GROUP(mpse_bnfa_multi) +{ + const MpseApi* mpse_api = (const MpseApi*)se_ac_bnfa; + Mpse* bnfa1 = nullptr; + Mpse* bnfa2 = nullptr; + + void setup() override + { + CHECK(se_ac_bnfa); + + bnfa1 = mpse_api->ctor(snort_conf, nullptr, &s_agent); + CHECK(bnfa1); + + bnfa2 = mpse_api->ctor(snort_conf, nullptr, &s_agent); + CHECK(bnfa2); + + hits = 0; + parse_errors = 0; + } + void teardown() override + { + mpse_api->dtor(bnfa1); + mpse_api->dtor(bnfa2); + } +}; + +TEST(mpse_bnfa_multi, single) +{ + Mpse::PatternDescriptor desc; + + CHECK(bnfa1->add_pattern((const uint8_t*)"uba", 3, desc, s_user) == 0); + CHECK(bnfa2->add_pattern((const uint8_t*)"tuba", 4, desc, s_user) == 0); + + CHECK(bnfa1->prep_patterns(snort_conf) == 0); + CHECK(bnfa2->prep_patterns(snort_conf) == 0); + + CHECK(bnfa1->get_pattern_count() == 1); + CHECK(bnfa2->get_pattern_count() == 1); + + int state = 0; + CHECK(bnfa1->search((const uint8_t*)"fubar", 5, match, nullptr, &state) == 1 ); + CHECK(hits == 1); + + CHECK(bnfa2->search((const uint8_t*)"fubar", 5, match, nullptr, &state) == 0); + CHECK(hits == 1); + + CHECK(bnfa1->search((const uint8_t*)"snafu", 5, match, nullptr, &state) == 0); + CHECK(hits == 1); +} + +//------------------------------------------------------------------------- +// main +//------------------------------------------------------------------------- + +int main(int argc, char** argv) +{ + ((MpseApi*)se_ac_bnfa)->init(); + // FIXIT-L There is currently no external way to fully release the memory from the static + // s_scratch vector in hyperscan.cc + MemoryLeakWarningPlugin::turnOffNewDeleteOverloads(); + return CommandLineTestRunner::RunAllTests(argc, argv); +} + diff --git a/src/search_engines/test/hyper_tool_test.cc b/src/search_engines/test/hyper_tool_test.cc new file mode 100644 index 000000000..0b83e0605 --- /dev/null +++ b/src/search_engines/test/hyper_tool_test.cc @@ -0,0 +1,162 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2016-2022 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// search_tool_test.cc author Steve Chew + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +// Change private to public to give access to private members. +#define private public +#include "search_engines/search_tool.h" +#undef private + +#include + +#include "detection/fp_config.h" +#include "framework/base_api.h" +#include "framework/mpse_batch.h" +#include "main/snort_config.h" +#include "managers/mpse_manager.h" + +#include "mpse_test_stubs.h" + +// must appear after snort_config.h to avoid broken c++ map include +#include +#include + +using namespace snort; + +//------------------------------------------------------------------------- +// stubs, spies, etc. +//------------------------------------------------------------------------- + +const MpseApi* get_test_api() +{ return (const MpseApi*) se_hyperscan; } + +//------------------------------------------------------------------------- +// hyperscan tests +//------------------------------------------------------------------------- + +TEST_GROUP(search_tool_full) +{ + Module* mod = nullptr; + SearchTool* stool; + const MpseApi* mpse_api = (const MpseApi*)se_hyperscan; + + void setup() override + { + mod = mpse_api->base.mod_ctor(); + stool = new SearchTool; + CHECK(stool->mpsegrp->normal_mpse); + + int pattern_id = 1; + stool->add("the", 3, pattern_id); + CHECK(stool->max_len == 3); + + pattern_id = 77; + stool->add("tuba", 4, pattern_id); + CHECK(stool->max_len == 4); + + pattern_id = 78; + stool->add("uba", 3, pattern_id); + CHECK(stool->max_len == 4); + + pattern_id = 2112; + stool->add("away", 4, pattern_id); + CHECK(stool->max_len == 4); + + pattern_id = 1000; + stool->add("nothere", 7, pattern_id); + CHECK(stool->max_len == 7); + + stool->prep(); + + } + void teardown() override + { + delete stool; + scratcher->cleanup(snort_conf); + mpse_api->base.mod_dtor(mod); + } +}; + +TEST(search_tool_full, search) +{ + // 0 1 2 3 + // 0123456789012345678901234567890 + const char* datastr = "the tuba ran away with the tuna"; + const ExpectedMatch xm[] = + { + { 1, 3 }, + { 77, 8 }, + { 78, 8 }, + { 2112, 17 }, + { 1, 26 }, + { 0, 0 } + }; + + s_expect = xm; + s_found = 0; + scratcher->setup(snort_conf); + + int result = stool->find(datastr, strlen(datastr), check_mpse_match); + + CHECK(result == 5); + CHECK(s_found == 5); +} + +TEST(search_tool_full, search_all) +{ + // 0 1 2 3 + // 0123456789012345678901234567890 + const char* datastr = "the tuba ran away with the tuna"; + const ExpectedMatch xm[] = + { + { 1, 3 }, + { 77, 8 }, + { 78, 8 }, + { 2112, 17 }, + { 1, 26 }, + { 0, 0 } + }; + + s_expect = xm; + s_found = 0; + scratcher->setup(snort_conf); + + int result = stool->find_all(datastr, strlen(datastr), check_mpse_match); + + CHECK(result == 5); + CHECK(s_found == 5); +} + +//------------------------------------------------------------------------- +// main +//------------------------------------------------------------------------- + +int main(int argc, char** argv) +{ + ((MpseApi*)se_hyperscan)->init(); + // FIXIT-L There is currently no external way to fully release the memory from the static + // s_scratch vector in hyperscan.cc + MemoryLeakWarningPlugin::turnOffNewDeleteOverloads(); + return CommandLineTestRunner::RunAllTests(argc, argv); +} + diff --git a/src/search_engines/test/hyperscan_test.cc b/src/search_engines/test/hyperscan_test.cc index 6ef6332cf..b3d3df47b 100644 --- a/src/search_engines/test/hyperscan_test.cc +++ b/src/search_engines/test/hyperscan_test.cc @@ -29,7 +29,8 @@ #include "framework/mpse.h" #include "framework/mpse_batch.h" #include "main/snort_config.h" -#include "utils/stats.h" + +#include "mpse_test_stubs.h" // must appear after snort_config.h to avoid broken c++ map include #include @@ -37,120 +38,12 @@ using namespace snort; -//------------------------------------------------------------------------- -// base stuff -//------------------------------------------------------------------------- - -namespace snort -{ -Mpse::Mpse(const char*) { } - -int Mpse::search( - const unsigned char* T, int n, MpseMatch match, - void* context, int* current_state) -{ - return _search(T, n, match, context, current_state); -} - -int Mpse::search_all( - const unsigned char* T, int n, MpseMatch match, - void* context, int* current_state) -{ - return _search(T, n, match, context, current_state); -} - -void Mpse::search(MpseBatch& batch, MpseType mpse_type) -{ - _search(batch, mpse_type); -} - -void Mpse::_search(MpseBatch& batch, MpseType mpse_type) -{ - int start_state; - - for ( auto& item : batch.items ) - { - if (item.second.done) - continue; - - item.second.error = false; - item.second.matches = 0; - - for ( auto& so : item.second.so ) - { - start_state = 0; - switch (mpse_type) - { - case MPSE_TYPE_NORMAL: - item.second.matches += so->normal_mpse->search(item.first.buf, item.first.len, - batch.mf, batch.context, &start_state); - break; - case MPSE_TYPE_OFFLOAD: - item.second.matches += so->offload_mpse->search(item.first.buf, item.first.len, - batch.mf, batch.context, &start_state); - break; - } - } - item.second.done = true; - } -} - -SnortConfig s_conf; -THREAD_LOCAL SnortConfig* snort_conf = &s_conf; - -static std::vector s_state; -static ScratchAllocator* scratcher = nullptr; - -DataBus::DataBus() = default; -DataBus::~DataBus() = default; - -SnortConfig::SnortConfig(const SnortConfig* const, const char*) -{ - state = &s_state; - num_slots = 1; -} - -SnortConfig::~SnortConfig() = default; - -int SnortConfig::request_scratch(ScratchAllocator* s) -{ - scratcher = s; - s_state.resize(1); - return 0; -} - -void SnortConfig::release_scratch(int) -{ - scratcher = nullptr; - s_state.clear(); - s_state.shrink_to_fit(); -} - -const SnortConfig* SnortConfig::get_conf() -{ return snort_conf; } - -static unsigned parse_errors = 0; -void ParseError(const char*, ...) -{ parse_errors++; } -void ErrorMessage(const char*, ...) { } - -void LogCount(char const*, uint64_t, FILE*) -{ } - -unsigned get_instance_id() -{ return 0; } - -void md5(const unsigned char*, size_t, unsigned char*) { } -} - -void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { } -void show_stats(PegCount*, const PegInfo*, const IndexVec&, const char*, FILE*) { } - //------------------------------------------------------------------------- // stubs, spies, etc. //------------------------------------------------------------------------- -extern const BaseApi* se_hyperscan; +const MpseApi* get_test_api() +{ return nullptr; } static unsigned hits = 0; @@ -158,29 +51,6 @@ static int match( void* /*user*/, void* /*tree*/, int /*index*/, void* /*context*/, void* /*list*/) { ++hits; return 0; } -static void* s_user = (void*)"user"; -static void* s_tree = (void*)"tree"; -static void* s_list = (void*)"list"; - -static MpseAgent s_agent = -{ - [](struct SnortConfig* sc, void*, void** ppt) - { - CHECK(sc == snort_conf); - *ppt = s_tree; - return 0; - }, - [](void*, void** ppl) - { - *ppl = s_list; - return 0; - }, - - [](void* pu) { CHECK(pu == s_user); }, - [](void** ppt) { CHECK(*ppt == s_tree); }, - [](void** ppl) { CHECK(*ppl == s_list); } -}; - //------------------------------------------------------------------------- // base tests //------------------------------------------------------------------------- @@ -229,7 +99,6 @@ TEST_GROUP(mpse_hs_match) { Module* mod = nullptr; Mpse* hs = nullptr; - bool do_cleanup = false; const MpseApi* mpse_api = (const MpseApi*)se_hyperscan; void setup() override @@ -244,8 +113,7 @@ TEST_GROUP(mpse_hs_match) void teardown() override { mpse_api->dtor(hs); - if ( do_cleanup ) - scratcher->cleanup(snort_conf); + scratcher->cleanup(snort_conf); mpse_api->base.mod_dtor(mod); } }; @@ -256,7 +124,7 @@ TEST(mpse_hs_match, empty) CHECK(parse_errors == 0); CHECK(hs->get_pattern_count() == 0); - do_cleanup = scratcher->setup(snort_conf); + scratcher->setup(snort_conf); int state = 0; CHECK(hs->search((const uint8_t*)"foo", 3, match, nullptr, &state) == 0); @@ -271,7 +139,7 @@ TEST(mpse_hs_match, single) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - do_cleanup = scratcher->setup(snort_conf); + scratcher->setup(snort_conf); int state = 0; CHECK(hs->search((const uint8_t*)"foo", 3, match, nullptr, &state) == 1); @@ -286,7 +154,7 @@ TEST(mpse_hs_match, nocase) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - do_cleanup = scratcher->setup(snort_conf); + scratcher->setup(snort_conf); int state = 0; CHECK(hs->search((const uint8_t*)"foo", 3, match, nullptr, &state) == 1); @@ -302,7 +170,7 @@ TEST(mpse_hs_match, other) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - do_cleanup = scratcher->setup(snort_conf); + scratcher->setup(snort_conf); int state = 0; CHECK(hs->search((const uint8_t*)"foo", 3, match, nullptr, &state) == 1); @@ -321,10 +189,12 @@ TEST(mpse_hs_match, multi) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 3); - do_cleanup = scratcher->setup(snort_conf); + scratcher->setup(snort_conf); int state = 0; - CHECK(hs->search((const uint8_t*)"foo bar baz", 11, match, nullptr, &state) == 3); + // __STRDUMP_DISABLE__ + CHECK(hs->search((const uint8_t*)"foo barfoo bazbaz", 17, match, nullptr, &state) == 3); + // __STRDUMP_ENABLE__ CHECK(hits == 3); } @@ -338,7 +208,7 @@ TEST(mpse_hs_match, regex) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - do_cleanup = scratcher->setup(snort_conf); + scratcher->setup(snort_conf); int state = 0; CHECK(hs->search((const uint8_t*)"foo bar baz", 11, match, nullptr, &state) == 0); @@ -355,7 +225,7 @@ TEST(mpse_hs_match, pcre) CHECK(hs->prep_patterns(snort_conf) == 0); CHECK(hs->get_pattern_count() == 1); - do_cleanup = scratcher->setup(snort_conf); + scratcher->setup(snort_conf); int state = 0; CHECK(hs->search((const uint8_t*)":definition(", 12, match, nullptr, &state) == 0); @@ -375,7 +245,6 @@ TEST_GROUP(mpse_hs_multi) Module* mod = nullptr; Mpse* hs1 = nullptr; Mpse* hs2 = nullptr; - bool do_cleanup = false; const MpseApi* mpse_api = (const MpseApi*)se_hyperscan; void setup() override @@ -397,8 +266,7 @@ TEST_GROUP(mpse_hs_multi) { mpse_api->dtor(hs1); mpse_api->dtor(hs2); - if ( do_cleanup ) - scratcher->cleanup(snort_conf); + scratcher->cleanup(snort_conf); mpse_api->base.mod_dtor(mod); } }; @@ -416,7 +284,7 @@ TEST(mpse_hs_multi, single) CHECK(hs1->get_pattern_count() == 1); CHECK(hs2->get_pattern_count() == 1); - do_cleanup = scratcher->setup(snort_conf); + scratcher->setup(snort_conf); int state = 0; CHECK(hs1->search((const uint8_t*)"fubar", 5, match, nullptr, &state) == 1 ); diff --git a/src/search_engines/test/mpse_test_stubs.cc b/src/search_engines/test/mpse_test_stubs.cc new file mode 100644 index 000000000..eb24185eb --- /dev/null +++ b/src/search_engines/test/mpse_test_stubs.cc @@ -0,0 +1,195 @@ +////-------------------------------------------------------------------------- +// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// mpse_test_stubs.cc author Russ Combs + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "mpse_test_stubs.h" + +#include + +#include "detection/fp_config.h" +#include "framework/base_api.h" +#include "framework/mpse.h" +#include "framework/mpse_batch.h" +#include "main/snort_config.h" +#include "managers/mpse_manager.h" +#include "search_engines/pat_stats.h" +#include "utils/stats.h" + +//------------------------------------------------------------------------- +// base stuff +//------------------------------------------------------------------------- + +std::vector s_state; +snort::ScratchAllocator* scratcher = nullptr; + +namespace snort +{ +SnortConfig s_conf; + +THREAD_LOCAL SnortConfig* snort_conf = &s_conf; + +const SnortConfig* SnortConfig::get_conf() +{ return snort_conf; } + +SnortConfig::SnortConfig(const SnortConfig* const, const char*) +{ + state = &s_state; + num_slots = 1; + fast_pattern_config = new FastPatternConfig(); +} + +SnortConfig::~SnortConfig() = default; + +int SnortConfig::request_scratch(ScratchAllocator* s) +{ + scratcher = s; + s_state.resize(1); + return 0; +} + +void SnortConfig::release_scratch(int) +{ + scratcher = nullptr; + s_state.clear(); + s_state.shrink_to_fit(); +} + +DataBus::DataBus() = default; +DataBus::~DataBus() = default; + +unsigned get_instance_id() +{ return 0; } + +THREAD_LOCAL PatMatQStat pmqs; + +unsigned parse_errors = 0; +void ParseError(const char*, ...) +{ parse_errors++; } +void ErrorMessage(const char*, ...) { } + +void LogValue(const char*, const char*, FILE*) { } +void LogMessage(const char*, ...) { } +[[noreturn]] void FatalError(const char*,...) { exit(1); } +void LogCount(char const*, uint64_t, FILE*) { } +void LogStat(const char*, double, FILE*) { } + +void md5(const unsigned char*, size_t, unsigned char*) { } + +} // namespace snort + +FastPatternConfig::FastPatternConfig() +{ search_api = get_test_api(); } + +const char* FastPatternConfig::get_search_method() +{ return search_api ? search_api->base.name : nullptr; } + +using namespace snort; + +void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { } +void show_stats(PegCount*, const PegInfo*, const IndexVec&, const char*, FILE*) { } + +Mpse* mpse = nullptr; + +void* s_user = (void*)"user"; +void* s_tree = (void*)"tree"; +void* s_list = (void*)"list"; + +MpseAgent s_agent = +{ + [](struct SnortConfig*, void*, void** ppt) + { + *ppt = s_tree; + return 0; + }, + [](void*, void** ppl) + { + *ppl = s_list; + return 0; + }, + + [](void*) { }, + [](void** ppt) { assert(*ppt == s_tree); }, + [](void** ppl) { assert(*ppl == s_list); } +}; + +void MpseManager::delete_search_engine(Mpse* eng) +{ + const MpseApi* api = eng->get_api(); + api->dtor(eng); +} + +MpseGroup::~MpseGroup() +{ + if (normal_mpse) + { + MpseManager::delete_search_engine(normal_mpse); + normal_mpse = nullptr; + } + if (offload_mpse) + { + MpseManager::delete_search_engine(offload_mpse); + offload_mpse = nullptr; + } +} + +bool MpseGroup::create_normal_mpse(const SnortConfig* sc, const char* type) +{ + const MpseApi* api = sc->fast_pattern_config->get_search_api(); + assert(api and !strcmp(api->base.name, type)); + + api->init(); + mpse = api->ctor(sc, nullptr, &s_agent); + + assert(mpse); + + mpse->set_api(api); + normal_mpse = mpse; + + return true; +} + +bool MpseGroup::create_offload_mpse(const SnortConfig*) +{ + offload_mpse = nullptr; + return false; +} + +const ExpectedMatch* s_expect = nullptr; +int s_found = 0; + +int check_mpse_match( + void* pid, void* /*tree*/, int index, void* /*context*/, void* /*neg_list*/) +{ + auto id = reinterpret_cast(pid); + + if ( s_expect and s_found >= 0 and + s_expect[s_found].id == (int)id and + s_expect[s_found].offset == index ) + { + ++s_found; + } + else s_found = -1; + + return s_found == -1; +} + diff --git a/src/search_engines/test/mpse_test_stubs.h b/src/search_engines/test/mpse_test_stubs.h new file mode 100644 index 000000000..d2a9222b5 --- /dev/null +++ b/src/search_engines/test/mpse_test_stubs.h @@ -0,0 +1,75 @@ +////-------------------------------------------------------------------------- +// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// mpse_test_stubs.h author Russ Combs + +#ifndef TEST_STUBS_H +#define TEST_STUBS_H + +#include + +#include "detection/fp_config.h" +#include "framework/base_api.h" +#include "framework/mpse.h" +#include "framework/mpse_batch.h" +#include "main/snort_config.h" +#include "managers/mpse_manager.h" +#include "utils/stats.h" + +#include "search_engines/pat_stats.h" + +extern std::vector s_state; +extern snort::ScratchAllocator* scratcher; + +namespace snort +{ +extern SnortConfig s_conf; + +extern THREAD_LOCAL SnortConfig* snort_conf; +extern THREAD_LOCAL PatMatQStat pmqs; + +extern unsigned parse_errors; +} // namespace snort + +extern snort::Mpse* mpse; +const snort::MpseApi* get_test_api(); + +extern void* s_user; +extern void* s_tree; +extern void* s_list; + +extern MpseAgent s_agent; + +extern const snort::BaseApi* se_ac_bnfa; +extern const snort::BaseApi* se_ac_full; +extern const snort::BaseApi* se_hyperscan; + +struct ExpectedMatch +{ + int id; + int offset; +}; + +extern const ExpectedMatch* s_expect; +extern int s_found; + +int check_mpse_match( + void* pid, void* /*tree*/, int index, void* /*context*/, void* /*neg_list*/); + +#endif + diff --git a/src/search_engines/test/search_tool_test.cc b/src/search_engines/test/search_tool_test.cc index c19bbece0..001392c8c 100644 --- a/src/search_engines/test/search_tool_test.cc +++ b/src/search_engines/test/search_tool_test.cc @@ -31,11 +31,12 @@ #include "detection/fp_config.h" #include "framework/base_api.h" -#include "framework/mpse.h" #include "framework/mpse_batch.h" #include "main/snort_config.h" #include "managers/mpse_manager.h" +#include "mpse_test_stubs.h" + // must appear after snort_config.h to avoid broken c++ map include #include #include @@ -43,257 +44,11 @@ using namespace snort; //------------------------------------------------------------------------- -// base stuff +// stubs, spies, etc. //------------------------------------------------------------------------- -namespace snort -{ -SnortConfig s_conf; - -THREAD_LOCAL SnortConfig* snort_conf = &s_conf; - -static std::vector s_state; - -DataBus::DataBus() = default; -DataBus::~DataBus() = default; - -SnortConfig::SnortConfig(const SnortConfig* const, const char*) -{ - state = &s_state; - num_slots = 1; - fast_pattern_config = nullptr; -} - -SnortConfig::~SnortConfig() = default; - -const SnortConfig* SnortConfig::get_conf() -{ return snort_conf; } - -unsigned get_instance_id() -{ return 0; } - -void LogValue(const char*, const char*, FILE*) { } -void LogMessage(const char*, ...) { } -[[noreturn]] void FatalError(const char*,...) { exit(1); } -void LogCount(char const*, uint64_t, FILE*) { } -void LogStat(const char*, double, FILE*) { } - -static void* s_tree = (void*)"tree"; -static void* s_list = (void*)"list"; - -static MpseAgent s_agent = -{ - [](struct SnortConfig* sc, void*, void** ppt) - { - CHECK(sc == nullptr); - *ppt = s_tree; - return 0; - }, - [](void*, void** ppl) - { - *ppl = s_list; - return 0; - }, - - [](void*) { }, - [](void** ppt) { CHECK(*ppt == s_tree); }, - [](void** ppl) { CHECK(*ppl == s_list); } -}; - -Mpse::Mpse(const char*) { } - -int Mpse::search( - const unsigned char* T, int n, MpseMatch match, - void* context, int* current_state) -{ - return _search(T, n, match, context, current_state); -} - -int Mpse::search_all( - const unsigned char* T, int n, MpseMatch match, - void* context, int* current_state) -{ - return _search(T, n, match, context, current_state); -} - -void Mpse::search(MpseBatch&, MpseType) { } -void Mpse::_search(MpseBatch&, MpseType) { } - -} - -const char* FastPatternConfig::get_search_method() -{ return "ac_bnfa"; } - -extern const BaseApi* se_ac_bnfa; -extern const BaseApi* se_ac_full; -Mpse* mpse = nullptr; - -void MpseManager::delete_search_engine(Mpse* eng) -{ - const MpseApi* api = eng->get_api(); - api->dtor(eng); -} - -MpseGroup::~MpseGroup() -{ - if (normal_mpse) - { - MpseManager::delete_search_engine(normal_mpse); - normal_mpse = nullptr; - } - if (offload_mpse) - { - MpseManager::delete_search_engine(offload_mpse); - offload_mpse = nullptr; - } -} - -bool MpseGroup::create_normal_mpse(const SnortConfig*, const char* type) -{ - const MpseApi* api; - - if ( !strcmp(type, "ac_bnfa") ) - api = (const MpseApi*) se_ac_bnfa; - - else if ( !strcmp(type, "ac_full") ) - api = (const MpseApi*) se_ac_full; - - else - return false; - - api->init(); - mpse = api->ctor(snort_conf, nullptr, &s_agent); - - CHECK(mpse); - - mpse->set_api(api); - normal_mpse = mpse; - - return true; -} - -bool MpseGroup::create_offload_mpse(const SnortConfig*) -{ - offload_mpse = nullptr; - return false; -} - -struct ExpectedMatch -{ - int id; - int offset; -}; - -static const ExpectedMatch* s_expect = nullptr; -static int s_found = 0; - -static int Test_SearchStrFound( - void* pid, void* /*tree*/, int index, void* /*context*/, void* /*neg_list*/) -{ - auto id = reinterpret_cast(pid); - - if ( s_expect and s_found >= 0 and - s_expect[s_found].id == (int)id and - s_expect[s_found].offset == index ) - { - ++s_found; - } - else s_found = -1; - - return s_found == -1; -} - -//------------------------------------------------------------------------- -// ac_bnfa tests -//------------------------------------------------------------------------- - -TEST_GROUP(search_tool_bnfa) -{ - SearchTool* stool; - - void setup() override - { - CHECK(se_ac_bnfa); - SearchTool::set_conf(snort_conf); - stool = new SearchTool("ac_bnfa"); - SearchTool::set_conf(nullptr); - - CHECK(stool->mpsegrp->normal_mpse); - - int pattern_id = 1; - stool->add("the", 3, pattern_id); - CHECK(stool->max_len == 3); - - pattern_id = 77; - stool->add("tuba", 4, pattern_id); - CHECK(stool->max_len == 4); - - pattern_id = 78; - stool->add("uba", 3, pattern_id); - CHECK(stool->max_len == 4); - - pattern_id = 2112; - stool->add("away", 4, pattern_id); - CHECK(stool->max_len == 4); - - pattern_id = 1000; - stool->add("nothere", 7, pattern_id); - CHECK(stool->max_len == 7); - - stool->prep(); - - } - void teardown() override - { - delete stool; - } -}; - -TEST(search_tool_bnfa, search) -{ - // 0 1 2 3 - // 0123456789012345678901234567890 - const char* datastr = "the tuba ran away with the tuna"; - const ExpectedMatch xm[] = - { - { 1, 3 }, - { 78, 8 }, - { 2112, 17 }, - { 1, 26 }, - { 0, 0 } - }; - - s_expect = xm; - s_found = 0; - - int result = stool->find(datastr, strlen(datastr), Test_SearchStrFound); - - CHECK(result == 4); - CHECK(s_found == 4); -} - -TEST(search_tool_bnfa, search_all) -{ - // 0 1 2 3 - // 0123456789012345678901234567890 - const char* datastr = "the tuba ran away with the tuna"; - const ExpectedMatch xm[] = - { - { 1, 3 }, - { 78, 8 }, - { 2112, 17 }, - { 1, 26 }, - { 0, 0 } - }; - - s_expect = xm; - s_found = 0; - - int result = stool->find_all(datastr, strlen(datastr), Test_SearchStrFound); - - CHECK(result == 4); - CHECK(s_found == 4); -} +const MpseApi* get_test_api() +{ return (const MpseApi*) se_ac_full; } //------------------------------------------------------------------------- // ac_full tests @@ -305,11 +60,7 @@ TEST_GROUP(search_tool_full) void setup() override { - CHECK(se_ac_full); - SearchTool::set_conf(snort_conf); - stool = new SearchTool("ac_full", true); - SearchTool::set_conf(nullptr); - + stool = new SearchTool; CHECK(stool->mpsegrp->normal_mpse); int pattern_id = 1; @@ -345,47 +96,49 @@ TEST(search_tool_full, search) { // 0 1 2 3 // 0123456789012345678901234567890 - const char* datastr = "the tuba ran away with the tuna"; + const char* datastr = "the tuba ran away with the the tuna"; const ExpectedMatch xm[] = { { 1, 3 }, { 78, 8 }, { 2112, 17 }, { 1, 26 }, + { 1, 30 }, { 0, 0 } }; s_expect = xm; s_found = 0; - int result = stool->find(datastr, strlen(datastr), Test_SearchStrFound); + int result = stool->find(datastr, strlen(datastr), check_mpse_match); - CHECK(result == 4); - CHECK(s_found == 4); + CHECK(result == 5); + CHECK(s_found == 5); } TEST(search_tool_full, search_all) { // 0 1 2 3 - // 0123456789012345678901234567890 - const char* datastr = "the tuba ran away with the tuna"; + // 01234567890123456789012345678901234 + const char* datastr = "the the tuba ran away with the tuna"; const ExpectedMatch xm[] = { { 1, 3 }, - { 78, 8 }, - { 77, 8 }, - { 2112, 17 }, - { 1, 26 }, + { 1, 7 }, + { 78, 12 }, + { 77, 12 }, + { 2112, 21 }, + { 1, 30 }, { 0, 0 } }; s_expect = xm; s_found = 0; - int result = stool->find_all(datastr, strlen(datastr), Test_SearchStrFound); + int result = stool->find_all(datastr, strlen(datastr), check_mpse_match); - CHECK(result == 5); - CHECK(s_found == 5); + CHECK(result == 6); + CHECK(s_found == 6); } //------------------------------------------------------------------------- @@ -394,6 +147,7 @@ TEST(search_tool_full, search_all) int main(int argc, char** argv) { + ((MpseApi*)se_ac_full)->init(); return CommandLineTestRunner::RunAllTests(argc, argv); } diff --git a/tools/snort2lua/config_states/config_deleted.cc b/tools/snort2lua/config_states/config_deleted.cc index b406c3d60..fc4e0ec84 100644 --- a/tools/snort2lua/config_states/config_deleted.cc +++ b/tools/snort2lua/config_states/config_deleted.cc @@ -600,6 +600,19 @@ static const ConvertMap sidechannel_api = const ConvertMap* sidechannel_map = &sidechannel_api; +/************************************************* + ******************* stateful ****************** + *************************************************/ + +static const std::string stateful = "stateful"; +static const ConvertMap stateful_api = +{ + stateful, + deleted_ctor<& stateful>, +}; + +const ConvertMap* stateful_map = &stateful_api; + /************************************************* ***************** no_promisc ******************* *************************************************/ diff --git a/tools/snort2lua/config_states/config_detection.cc b/tools/snort2lua/config_states/config_detection.cc index 4fc38d5b0..62e755f7a 100644 --- a/tools/snort2lua/config_states/config_detection.cc +++ b/tools/snort2lua/config_states/config_detection.cc @@ -65,8 +65,7 @@ bool Detection::convert(std::istringstream& data_stream) else if (keyword == "search-optimize") { - table_api.add_diff_option_comment("search-optimize", "search_optimize"); - tmpval = table_api.add_option("search_optimize", true); + table_api.add_deleted_comment("search-optimize is always true"); } else if (keyword == "split-any-any") { @@ -194,23 +193,23 @@ bool Detection::convert(std::istringstream& data_stream) } else if (method == "ac-std") { - table_api.add_diff_option_comment("ac-std", "ac_std"); - tmpval = table_api.add_option("search_method", "ac_std"); + table_api.add_diff_option_comment("ac-std", "ac_full"); + tmpval = table_api.add_option("search_method", "ac_full"); } else if (method == "ac-banded") { - table_api.add_diff_option_comment("ac-banded", "ac_banded"); - tmpval = table_api.add_option("search_method", "ac_banded"); + table_api.add_diff_option_comment("ac-banded", "ac_full"); + tmpval = table_api.add_option("search_method", "ac_full"); } else if (method == "acs") { - table_api.add_diff_option_comment("acs", "ac_sparse"); - tmpval = table_api.add_option("search_method", "ac_sparse"); + table_api.add_diff_option_comment("acs", "ac_full"); + tmpval = table_api.add_option("search_method", "ac_full"); } else if (method == "ac-sparsebands") { - table_api.add_diff_option_comment("ac-sparsebands", "ac_sparse_bands"); - tmpval = table_api.add_option("search_method", "ac_sparse_bands"); + table_api.add_diff_option_comment("ac-sparsebands", "ac_full"); + tmpval = table_api.add_option("search_method", "ac_full"); } else if (method == "lowmem") { diff --git a/tools/snort2lua/config_states/config_no_option.cc b/tools/snort2lua/config_states/config_no_option.cc index 24028d84c..ec66d55b4 100644 --- a/tools/snort2lua/config_states/config_no_option.cc +++ b/tools/snort2lua/config_states/config_no_option.cc @@ -302,19 +302,6 @@ static const ConvertMap show_year_api = const ConvertMap* show_year_map = &show_year_api; -/************************************************* - ******************* stateful ****************** - *************************************************/ - -static const std::string stateful = "stateful"; -static const ConvertMap stateful_api = -{ - stateful, - config_true_no_opt_ctor<& stateful, & alerts> -}; - -const ConvertMap* stateful_map = &stateful_api; - /************************************************* ********************* utc ********************* *************************************************/