From: Russ Combs (rucombs) Date: Tue, 19 Feb 2019 14:54:16 +0000 (-0500) Subject: Merge pull request #1521 in SNORT/snort3 from ~RUCOMBS/snort3:rxp_next to master X-Git-Tag: 3.0.0-251~41 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4e80165a3489d34112cca71f5460afd1748519c5;p=thirdparty%2Fsnort3.git Merge pull request #1521 in SNORT/snort3 from ~RUCOMBS/snort3:rxp_next to master Squashed commit of the following: commit 2557b2399f4bf28852585f513c74ca639e5f237b Author: russ Date: Mon Feb 18 16:21:32 2019 -0500 cppcheck: fix some basic warnings commit 17be370de205506baec9d37d828090e9c3e61dd6 Author: russ Date: Mon Feb 11 16:34:13 2019 -0500 RegexOffload: refactor into mode-specific subclasses commit 5f8adc6efd1029b98076d90d6864f8a2d248b499 Author: russ Date: Mon Feb 11 14:04:47 2019 -0500 MpseBatch: refactor into separate files commit d3ae7cebbacc7524276dd6aa2d11a7c67d9b72e8 Author: William Cochrane Date: Fri Oct 19 14:12:33 2018 +0100 Addition and use of offload search method/engine We are providing the user the ability to configure alongsides the (normal) fast pattern search method an offload search method. As a result 2 search engines may be created for each search engine group. Because the capability of each search engine may differ (e.g. one search engine may support regex and the other may not) the detection option tree that gets created cannot be shared amongst the normal and offload search engines but will be created unique for each search engine. To avoid duplication of search engines an offload search engine will only be instantiated if the offload search method is different to that of the normal search method. Offload search requests will now use the offload search method if it is configured and different to the normal search method, otherwise the normal search method will be used. If a request to the offload search engine fails this search is retried by sending it to the (normal) search engine. Also if a search tool request exceeds the offload limit and the offload search method is configured then a synchronous search using the offload search engine is attempted and if it fails this search will be retried to the (normal) search engine commit c9b69c52a8db4dc83833cc0c4059a7120a8daebd Author: Jonathan McDowell Date: Wed Jan 9 15:04:35 2019 +0000 Enable asyncronous searching using RegexOffload Turn RegexOffload into a more basic MPSE offloader, only executing searches in the offload thread rather than a full packet evaluation. Additionally allow for the option that the MPSE supports asynchronous operation and does not require separate threads to achieve this. --- diff --git a/src/detection/detection_engine.cc b/src/detection/detection_engine.cc index ecc11e4e9..0cc78b38d 100644 --- a/src/detection/detection_engine.cc +++ b/src/detection/detection_engine.cc @@ -36,6 +36,7 @@ #include "main/snort_debug.h" #include "main/thread.h" #include "managers/inspector_manager.h" +#include "managers/mpse_manager.h" #include "packet_io/active.h" #include "parser/parser.h" #include "profiler/profiler_defs.h" @@ -62,7 +63,25 @@ using namespace snort; //-------------------------------------------------------------------------- void DetectionEngine::thread_init() -{ offloader = new RegexOffload(SnortConfig::get_conf()->offload_threads); } +{ + SnortConfig* sc = SnortConfig::get_conf(); + FastPatternConfig* fp = sc->fast_pattern_config; + const MpseApi* offload_search_api = fp->get_offload_search_api(); + + // Note: offload_threads is really the maximum number of offload_requests + if (offload_search_api and MpseManager::is_async_capable(offload_search_api)) + { + // If the search method is async capable then the searches will be performed directly + // by the search engine, without requiring a processing thread. + offloader = RegexOffload::get_offloader(sc->offload_threads, false); + } + else + { + // If the search method is not async capable then offloaded searches will be performed + // in a separate processing thread that the RegexOffload instance needs to create. + offloader = RegexOffload::get_offloader(sc->offload_threads, true); + } +} void DetectionEngine::thread_term() { delete offloader; } @@ -338,6 +357,9 @@ void DetectionEngine::do_offload(Packet* p) p->context->conf = SnortConfig::get_conf(); p->set_offloaded(); + // Build the searches list in the packet context + fp_partial(p); + offloader->put(p); pc.offloads++; } @@ -359,6 +381,7 @@ bool DetectionEngine::offload(Packet* p) if ( depends_on_suspended ) { fp_partial(p); + p->context->searches.search_sync(); sw->suspend(); return true; } @@ -377,8 +400,6 @@ void DetectionEngine::idle() trace_logf(detection, TRACE_DETECTION_ENGINE, "(wire) %" PRIu64 " de::sleep\n", get_packet_number()); - const struct timespec blip = { 0, 1 }; - nanosleep(&blip, nullptr); onload(); } trace_logf(detection, TRACE_DETECTION_ENGINE, "(wire) %" PRIu64 " de::idle (r=%d)\n", diff --git a/src/detection/fp_config.cc b/src/detection/fp_config.cc index 4afb8f409..22b1d8530 100644 --- a/src/detection/fp_config.cc +++ b/src/detection/fp_config.cc @@ -66,6 +66,25 @@ const char* FastPatternConfig::get_search_method() return search_api->base.name; } +bool FastPatternConfig::set_offload_search_method(const char* method) +{ + const MpseApi* api = MpseManager::get_search_api(method); + + if ( !api ) + return false; + + offload_search_api = api; + return true; +} + +const char* FastPatternConfig::get_offload_search_method() +{ + if ( !offload_search_api ) + return nullptr; + + return offload_search_api->base.name; +} + void FastPatternConfig::set_max_pattern_len(unsigned int max_len) { if (max_pattern_len != 0) diff --git a/src/detection/fp_config.h b/src/detection/fp_config.h index 3e269b09b..ca998ea69 100644 --- a/src/detection/fp_config.h +++ b/src/detection/fp_config.h @@ -125,11 +125,17 @@ public: bool set_search_method(const char*); const char* get_search_method(); + bool set_offload_search_method(const char*); + const char* get_offload_search_method(); + void set_max_pattern_len(unsigned); const snort::MpseApi* get_search_api() { return search_api; } + const snort::MpseApi* get_offload_search_api() + { return offload_search_api; } + bool get_trim() { return trim; } @@ -145,7 +151,8 @@ public: unsigned set_max(unsigned bytes); private: - const snort::MpseApi* search_api; + const snort::MpseApi* search_api = nullptr; + const snort::MpseApi* offload_search_api = nullptr; bool inspect_stream_insert = true; bool trim; diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc index 11ed2cbf9..b3c5e4a2a 100644 --- a/src/detection/fp_create.cc +++ b/src/detection/fp_create.cc @@ -35,6 +35,7 @@ #include "fp_create.h" #include "framework/mpse.h" +#include "framework/mpse_batch.h" #include "hash/ghash.h" #include "log/messages.h" #include "main/snort_config.h" @@ -59,12 +60,13 @@ using namespace snort; using namespace std; static unsigned mpse_count = 0; +static unsigned offload_mpse_count = 0; static const char* s_group = ""; static void fpDeletePMX(void* data); -static int fpGetFinalPattern( - FastPatternConfig*, PatternMatchData*, const char*& ret_pattern, unsigned& ret_bytes); +static int fpGetFinalPattern(FastPatternConfig*, PatternMatchData*, const char*& ret_pattern, + unsigned& ret_bytes, Mpse::MpseType mpse_type); static void print_nfp_info(const char*, OptTreeNode*); static void print_fp_info(const char*, const OptTreeNode*, const PatternMatchData*, @@ -110,7 +112,7 @@ static bool new_sig(int num_children, detection_option_tree_node_t** nodes, OptT return true; } -static int otn_create_tree(OptTreeNode* otn, void** existing_tree) +static int otn_create_tree(OptTreeNode* otn, void** existing_tree, Mpse::MpseType mpse_type) { detection_option_tree_node_t* node = nullptr, * child; bool need_leaf = false; @@ -150,7 +152,7 @@ static int otn_create_tree(OptTreeNode* otn, void** existing_tree) /* Don't add contents that are only for use in the * fast pattern matcher */ - if ( is_fast_pattern_only(opt_fp) ) + if ( is_fast_pattern_only(opt_fp, mpse_type) ) { opt_fp = opt_fp->next; continue; @@ -344,7 +346,7 @@ static void neg_list_free(void** list) *list = nullptr; } -static int pmx_create_tree(SnortConfig* sc, void* id, void** existing_tree) +static int pmx_create_tree(SnortConfig* sc, void* id, void** existing_tree, Mpse::MpseType mpse_type) { assert(existing_tree); @@ -363,51 +365,36 @@ static int pmx_create_tree(SnortConfig* sc, void* id, void** existing_tree) if (!*existing_tree) *existing_tree = new_root(otn); - return otn_create_tree(otn, existing_tree); + return otn_create_tree(otn, existing_tree, mpse_type); } -static int fpFinishPortGroupRule( - SnortConfig* sc, PortGroup* pg, - OptTreeNode* otn, PatternMatchData* pmd, FastPatternConfig* fp) +static int pmx_create_tree_normal(SnortConfig* sc, void* id, void** existing_tree) { - if ( !pmd ) - { - pg->add_nfp_rule(otn); - print_nfp_info(s_group, otn); - return 0; - } - if ( !pg->mpse[pmd->pm_type] ) - { - static MpseAgent agent = - { - pmx_create_tree, add_patrn_to_neg_list, - fpDeletePMX, free_detection_option_root, neg_list_free - }; - - pg->mpse[pmd->pm_type] = MpseManager::get_search_engine( - sc, fp->get_search_api(), &agent); - - if ( !pg->mpse[pmd->pm_type] ) - { - ParseError("Failed to create pattern matcher for %d", pmd->pm_type); - return -1; - } - mpse_count++; - - if ( fp->get_search_opt() ) - pg->mpse[pmd->pm_type]->set_opt(1); - } - if (pmd->is_negated()) - pg->add_nfp_rule(otn); + return pmx_create_tree(sc, id, existing_tree, Mpse::MPSE_TYPE_NORMAL); +} - else - pg->add_rule(); +static int pmx_create_tree_offload(SnortConfig* sc, void* id, void** existing_tree) +{ + return pmx_create_tree(sc, id, existing_tree, Mpse::MPSE_TYPE_OFFLOAD); +} +static int fpFinishPortGroupRule( + SnortConfig* sc, snort::Mpse* mpse, OptTreeNode* otn, PatternMatchData* pmd, + FastPatternConfig* fp, Mpse::MpseType mpse_type, bool get_final_pat) +{ const char* pattern; unsigned pattern_length; - if (fpGetFinalPattern(fp, pmd, pattern, pattern_length) == -1) - return -1; + if (get_final_pat) + { + if (fpGetFinalPattern(fp, pmd, pattern, pattern_length, mpse_type) == -1) + return -1; + } + else + { + pattern = pmd->pattern_buf; + pattern_length = pmd->pattern_size; + } if ( fp->get_debug_print_fast_patterns() ) print_fp_info(s_group, otn, pmd, pattern, pattern_length); @@ -419,7 +406,7 @@ static int fpFinishPortGroupRule( Mpse::PatternDescriptor desc( pmd->is_no_case(), pmd->is_negated(), pmd->is_literal(), pmd->mpse_flags); - pg->mpse[pmd->pm_type]->add_pattern(sc, (const uint8_t*)pattern, pattern_length, desc, pmx); + mpse->add_pattern(sc, (const uint8_t*)pattern, pattern_length, desc, pmx); return 0; } @@ -435,24 +422,56 @@ static int fpFinishPortGroup( for (i = PM_TYPE_PKT; i < PM_TYPE_MAX; i++) { - if (pg->mpse[i] != nullptr) + if (pg->mpsegrp[i] != nullptr) { - if (pg->mpse[i]->get_pattern_count() != 0) + if (pg->mpsegrp[i]->normal_mpse != nullptr) { - if ( !sc->test_mode() or sc->mem_check() ) + if (pg->mpsegrp[i]->normal_mpse->get_pattern_count() != 0) + { + if ( !sc->test_mode() or sc->mem_check() ) + { + if ( pg->mpsegrp[i]->normal_mpse->prep_patterns(sc) != 0 ) + FatalError("Failed to compile port group patterns for normal " + "search engine.\n"); + } + + if (fp->get_debug_mode()) + pg->mpsegrp[i]->normal_mpse->print_info(); + rules = 1; + } + else { - if ( pg->mpse[i]->prep_patterns(sc) != 0 ) - FatalError("Failed to compile port group patterns.\n"); + MpseManager::delete_search_engine(pg->mpsegrp[i]->normal_mpse); + pg->mpsegrp[i]->normal_mpse = nullptr; } + } + if (pg->mpsegrp[i]->offload_mpse != nullptr) + { + if (pg->mpsegrp[i]->offload_mpse->get_pattern_count() != 0) + { + if ( !sc->test_mode() or sc->mem_check() ) + { + if ( pg->mpsegrp[i]->offload_mpse->prep_patterns(sc) != 0 ) + FatalError("Failed to compile port group patterns for offload " + "search engine.\n"); + } - if (fp->get_debug_mode()) - pg->mpse[i]->print_info(); - rules = 1; + if (fp->get_debug_mode()) + pg->mpsegrp[i]->offload_mpse->print_info(); + rules = 1; + } + else + { + MpseManager::delete_search_engine(pg->mpsegrp[i]->offload_mpse); + pg->mpsegrp[i]->offload_mpse = nullptr; + } } - else + + if ((pg->mpsegrp[i]->normal_mpse == nullptr) and + (pg->mpsegrp[i]->offload_mpse == nullptr)) { - MpseManager::delete_search_engine(pg->mpse[i]); - pg->mpse[i] = nullptr; + delete pg->mpsegrp[i]; + pg->mpsegrp[i] = nullptr; } } } @@ -464,7 +483,7 @@ static int fpFinishPortGroup( for (ruleNode = pg->nfp_head; ruleNode; ruleNode = ruleNode->rnNext) { OptTreeNode* otn = (OptTreeNode*)ruleNode->rnRuleData; - otn_create_tree(otn, &pg->nfp_tree); + otn_create_tree(otn, &pg->nfp_tree, Mpse::MPSE_TYPE_NORMAL); } finalize_detection_option_tree(sc, (detection_option_tree_root_t*)pg->nfp_tree); @@ -483,27 +502,21 @@ static int fpFinishPortGroup( return 0; } -static void fpAddAlternatePatterns(SnortConfig* sc, PortGroup* pg, - OptTreeNode* otn, PatternMatchData* pmd, FastPatternConfig* fp) +static void fpAddAlternatePatterns(SnortConfig* sc, snort::Mpse* mpse, + OptTreeNode* otn, PatternMatchData* pmd, FastPatternConfig* fp, Mpse::MpseType mpse_type) { - if ( fp->get_debug_print_fast_patterns() ) - print_fp_info(s_group, otn, pmd, pmd->pattern_buf, pmd->pattern_size); - - PMX* pmx = (PMX*)snort_calloc(sizeof(PMX)); - pmx->rule_node.rnRuleData = otn; - pmx->pmd = pmd; - - Mpse::PatternDescriptor desc( - pmd->is_no_case(), pmd->is_negated(), pmd->is_literal(), pmd->mpse_flags); - - pg->mpse[pmd->pm_type]->add_pattern( - sc, (const uint8_t*)pmd->pattern_buf, pmd->pattern_size, desc, pmx); + fpFinishPortGroupRule(sc, mpse, otn, pmd, fp, mpse_type, false); } static int fpAddPortGroupRule( SnortConfig* sc, PortGroup* pg, OptTreeNode* otn, FastPatternConfig* fp, bool srvc) { + const MpseApi* search_api = nullptr; + const MpseApi* offload_search_api = nullptr; PatternMatchVector pmv; + OptFpList* next = nullptr; + bool only_literal; + bool exclude; // skip builtin rules, continue for text and so rules if ( otn->sigInfo.builtin ) @@ -513,31 +526,167 @@ static int fpAddPortGroupRule( if ( !otn->enabled ) return -1; - OptFpList* next = nullptr; - bool only_literal = !MpseManager::is_regex_capable(fp->get_search_api()); - bool exclude; + search_api = fp->get_search_api(); + assert(search_api); + + only_literal = !MpseManager::is_regex_capable(search_api); pmv = get_fp_content(otn, next, srvc, only_literal, exclude); if ( !pmv.empty() ) { - PatternMatchData* main_pmd = pmv.back(); - pmv.pop_back(); + PatternMatchVector pmv_ol; + OptFpList* next_ol = nullptr; + bool add_to_offload = false; + bool cont = true; + PatternMatchData* ol_pmd = nullptr; - if ( !main_pmd->is_relative() && !main_pmd->is_negated() && main_pmd->fp_only >= 0 && - // FIXIT-L no_case consideration is mpse specific, delegate - !main_pmd->offset && !main_pmd->depth && main_pmd->is_no_case() ) + offload_search_api = fp->get_offload_search_api(); + + // Only add rule to the offload search engine if the offload search engine + // is different to the normal search engine. + if (offload_search_api and (offload_search_api != search_api)) { - if ( !next || !next->ips_opt || !next->ips_opt->is_relative() ) - main_pmd->fp_only = 1; + bool exclude_ol; + bool only_literal_ol = !MpseManager::is_regex_capable(offload_search_api); + pmv_ol = get_fp_content(otn, next_ol, srvc, only_literal_ol, exclude_ol); + + // If we can get a fast_pattern for the normal search engine but not for the + // offload search engine then add rule to the non fast pattern list + if (!pmv_ol.empty()) + add_to_offload = true; + else + cont = false; } - if (fpFinishPortGroupRule(sc, pg, otn, main_pmd, fp) == 0) + // From here on we will create the mpses that are needed and add the patterns + if (cont) { - if (main_pmd->pattern_size > otn->longestPatternLen) - otn->longestPatternLen = main_pmd->pattern_size; - for (auto p : pmv) - fpAddAlternatePatterns(sc, pg, otn, p, fp); + PatternMatchData* main_pmd = pmv.back(); + pmv.pop_back(); + + if ( !main_pmd->is_relative() && !main_pmd->is_negated() && main_pmd->fp_only >= 0 && + // FIXIT-L no_case consideration is mpse specific, delegate + !main_pmd->offset && !main_pmd->depth && main_pmd->is_no_case() ) + { + if ( !next || !next->ips_opt || !next->ips_opt->is_relative() ) + main_pmd->fp_only |= (1 << Mpse::MPSE_TYPE_NORMAL); + } + + static MpseAgent agent = + { + pmx_create_tree_normal, add_patrn_to_neg_list, + fpDeletePMX, free_detection_option_root, neg_list_free + }; + + if ( pg->mpsegrp[main_pmd->pm_type] == nullptr ) + { + pg->mpsegrp[main_pmd->pm_type] = new MpseGroup; + if ( pg->mpsegrp[main_pmd->pm_type] == nullptr ) + { + ParseError("Failed to create pattern matcher for %d", main_pmd->pm_type); + return -1; + } + } + + if (pg->mpsegrp[main_pmd->pm_type]->normal_mpse == nullptr) + { + if (!pg->mpsegrp[main_pmd->pm_type]->create_normal_mpse(sc, &agent)) + { + ParseError("Failed to create normal pattern matcher for %d", main_pmd->pm_type); + return -1; + } + + mpse_count++; + if ( fp->get_search_opt() ) + pg->mpsegrp[main_pmd->pm_type]->normal_mpse->set_opt(1); + } + + if (add_to_offload) + { + ol_pmd = pmv_ol.back(); + pmv_ol.pop_back(); + + if ( !ol_pmd->is_relative() && !ol_pmd->is_negated() && ol_pmd->fp_only >= 0 && + // FIXIT-L no_case consideration is mpse specific, delegate + !ol_pmd->offset && !ol_pmd->depth && ol_pmd->is_no_case() ) + { + if ( !next_ol || !next_ol->ips_opt || !next_ol->ips_opt->is_relative() ) + ol_pmd->fp_only |= (1 << Mpse::MPSE_TYPE_OFFLOAD); + } + + static MpseAgent agent_offload = + { + pmx_create_tree_offload, add_patrn_to_neg_list, + fpDeletePMX, free_detection_option_root, neg_list_free + }; + + // Keep the created mpse alongside the same pm type as the main pmd + if (pg->mpsegrp[main_pmd->pm_type]->offload_mpse == nullptr) + { + if (!pg->mpsegrp[main_pmd->pm_type]->create_offload_mpse(sc, &agent_offload)) + { + ParseError("Failed to create offload pattern matcher for %d", + main_pmd->pm_type); + return -1; + } + + offload_mpse_count++; + if ( fp->get_search_opt() ) + pg->mpsegrp[main_pmd->pm_type]->offload_mpse->set_opt(1); + } + } + + bool add_rule = false; + bool add_nfp_rule = false; + + if (pg->mpsegrp[main_pmd->pm_type]->normal_mpse) + { + add_rule = true; + if (main_pmd->is_negated()) + add_nfp_rule = true; + + // Now add patterns + if (fpFinishPortGroupRule(sc, pg->mpsegrp[main_pmd->pm_type]->normal_mpse, + otn, main_pmd, fp, Mpse::MPSE_TYPE_NORMAL, true) == 0) + { + if (main_pmd->pattern_size > otn->longestPatternLen) + otn->longestPatternLen = main_pmd->pattern_size; + + // Add Alternative patterns + for (auto p : pmv) + fpAddAlternatePatterns(sc, pg->mpsegrp[main_pmd->pm_type]->normal_mpse, + otn, p, fp, Mpse::MPSE_TYPE_NORMAL); + } + } + + if (ol_pmd and pg->mpsegrp[main_pmd->pm_type]->offload_mpse) + { + add_rule = true; + if (ol_pmd->is_negated()) + add_nfp_rule = true; + + // Now add patterns + if (fpFinishPortGroupRule(sc, pg->mpsegrp[main_pmd->pm_type]->offload_mpse, + otn, ol_pmd, fp, Mpse::MPSE_TYPE_OFFLOAD, true) == 0) + { + if (ol_pmd->pattern_size > otn->longestPatternLen) + otn->longestPatternLen = ol_pmd->pattern_size; + + // Add Alternative patterns + for (auto p : pmv_ol) + fpAddAlternatePatterns(sc, pg->mpsegrp[main_pmd->pm_type]->offload_mpse, + otn, p, fp, Mpse::MPSE_TYPE_OFFLOAD); + } + } + + if (add_rule) + { + if (add_nfp_rule) + pg->add_nfp_rule(otn); + else + pg->add_rule(); + } return 0; } @@ -547,8 +696,8 @@ static int fpAddPortGroupRule( return 0; // no fast pattern added - if (fpFinishPortGroupRule(sc, pg, otn, nullptr, fp) != 0) - return -1; + pg->add_nfp_rule(otn); + print_nfp_info(s_group, otn); return 0; } @@ -755,7 +904,7 @@ static void fpFreeRuleMaps(SnortConfig* sc) static int fpGetFinalPattern( FastPatternConfig* fp, PatternMatchData* pmd, - const char*& ret_pattern, unsigned& ret_bytes) + const char*& ret_pattern, unsigned& ret_bytes, Mpse::MpseType mpse_type) { if ( !fp or !pmd ) { @@ -779,7 +928,9 @@ static int fpGetFinalPattern( // 3. non-literals like regex - truncation could invalidate the // expression. - if ( pmd->fp_only > 0 or pmd->is_negated() or !pmd->is_literal() ) + assert((mpse_type == Mpse::MPSE_TYPE_NORMAL) or (mpse_type == Mpse::MPSE_TYPE_OFFLOAD)); + + if ( (pmd->fp_only & (1 << mpse_type)) or pmd->is_negated() or !pmd->is_literal() ) { ret_pattern = pattern; ret_bytes = bytes; @@ -842,14 +993,23 @@ static void fpPortGroupPrintRuleCount(PortGroup* pg, const char* what) for (type = PM_TYPE_PKT; type < PM_TYPE_MAX; type++) { - int count = pg->mpse[type] ? pg->mpse[type]->get_pattern_count() : 0; + if (pg->mpsegrp[type]) + { + int count = pg->mpsegrp[type]->normal_mpse ? + pg->mpsegrp[type]->normal_mpse->get_pattern_count() : 0; + int count_ol = pg->mpsegrp[type]->offload_mpse ? + pg->mpsegrp[type]->offload_mpse->get_pattern_count() : 0; + + if ( count ) + LogMessage("\tNormal Pattern Matcher %s: %d\n", pm_type_strings[type], count); - if ( count ) - LogMessage("\t%s: %d\n", pm_type_strings[type], count); + if ( count_ol ) + LogMessage("\tOffload Pattern Matcher %s: %d\n", pm_type_strings[type], count_ol); + } } if ( pg->nfp_rule_count ) - LogMessage("\tNo content: %u\n", pg->nfp_rule_count); + LogMessage("\tNormal Pattern Matcher No content: %u\n", pg->nfp_rule_count); } static void fpDeletePMX(void* pv) @@ -1375,7 +1535,8 @@ static void fp_sum_port_groups(PortGroup* pg, unsigned c[PM_TYPE_MAX]) return; for ( int i = PM_TYPE_PKT; i < PM_TYPE_MAX; ++i ) - if ( pg->mpse[i] and pg->mpse[i]->get_pattern_count() ) + if ( pg->mpsegrp[i] and pg->mpsegrp[i]->normal_mpse and + pg->mpsegrp[i]->normal_mpse->get_pattern_count() ) c[i]++; } @@ -1531,6 +1692,7 @@ int fpCreateFastPacketDetection(SnortConfig* sc) } mpse_count = 0; + offload_mpse_count = 0; MpseManager::start_search_engine(fp->get_search_api()); @@ -1576,6 +1738,11 @@ int fpCreateFastPacketDetection(SnortConfig* sc) LogLabel("search engine"); MpseManager::print_mpse_summary(fp->get_search_api()); } + if ( offload_mpse_count and (fp->get_offload_search_api())) + { + LogLabel("offload search engine"); + MpseManager::print_mpse_summary(fp->get_offload_search_api()); + } if ( fp->get_num_patterns_truncated() ) LogMessage("%25.25s: %-12u\n", "truncated patterns", fp->get_num_patterns_truncated()); diff --git a/src/detection/fp_detect.cc b/src/detection/fp_detect.cc index 609d0bf5c..422548898 100644 --- a/src/detection/fp_detect.cc +++ b/src/detection/fp_detect.cc @@ -874,9 +874,9 @@ static int rule_tree_queue( } static inline int batch_search( - Mpse* so, OtnxMatchData* omd, const uint8_t* buf, unsigned len, PegCount& cnt) + MpseGroup* so, OtnxMatchData* omd, const uint8_t* buf, unsigned len, PegCount& cnt) { - assert(so->get_pattern_count() > 0); + assert(so->get_normal_mpse()->get_pattern_count() > 0); cnt++; //FIXIT-P Batch outer UDP payload searches for teredo set and the outer header @@ -886,7 +886,7 @@ static inline int batch_search( int start_state = 0; MpseStash* stash = omd->p->context->stash; stash->init(); - so->search(buf, len, rule_tree_queue, omd, &start_state); + so->get_normal_mpse()->search(buf, len, rule_tree_queue, omd, &start_state); stash->process(rule_tree_match, omd); } else @@ -908,7 +908,8 @@ static inline int search_buffer( { if ( gadget->get_fp_buf(ibt, omd->p, buf) ) { - if ( Mpse* so = omd->pg->mpse[pmt] ) + // Depending on where we are searching we call the appropriate mpse + if ( MpseGroup* so = omd->pg->mpsegrp[pmt] ) { // FIXIT-H DELETE ME done - get the context packet number trace_logf(detection, TRACE_FP_SEARCH, "%" PRIu64 " fp %s.%s[%d]\n", @@ -937,7 +938,7 @@ static int fp_search( if ( (!user_mode or type < 2) and p->data and p->dsize ) { // ports search raw packet only - if ( Mpse* so = port_group->mpse[PM_TYPE_PKT] ) + if ( MpseGroup* so = port_group->mpsegrp[PM_TYPE_PKT] ) { if ( uint16_t pattern_match_size = p->get_detect_limit() ) { @@ -972,7 +973,7 @@ static int fp_search( if ( !user_mode or type > 0 ) { // file searches file only - if ( Mpse* so = port_group->mpse[PM_TYPE_FILE] ) + if ( MpseGroup* so = port_group->mpsegrp[PM_TYPE_FILE] ) { // FIXIT-M file data should be obtained from // inspector gadget as is done with search_buffer @@ -1315,12 +1316,8 @@ void fp_full(Packet* p) c->searches.context = c->otnx; fpEvalPacket(p, FPTask::BOTH); - if (c->searches.items.size() > 0) { - c->searches.search(c->searches); - while (c->searches.items.size() > 0) - c->searches.receive_responses(c->searches); + if ( c->searches.search_sync() ) stash->process(rule_tree_match, c->otnx); - } fpFinalSelectEvent(c->otnx, p); } @@ -1336,11 +1333,6 @@ void fp_partial(Packet* p) c->searches.mf = rule_tree_queue; c->searches.context = c->otnx; fpEvalPacket(p, FPTask::FP); - - if (c->searches.items.size() > 0) { - Mpse* so = c->searches.items.begin()->second.so[0]; - so->search(c->searches); - } } void fp_complete(Packet* p) @@ -1348,8 +1340,6 @@ void fp_complete(Packet* p) IpsContext* c = p->context; MpseStash* stash = c->stash; stash->enable_process(); - while (c->searches.items.size() > 0) - c->searches.items.begin()->second.so[0]->receive_responses(c->searches); stash->process(rule_tree_match, c->otnx); fpEvalPacket(p, FPTask::NON_FP); fpFinalSelectEvent(c->otnx, p); diff --git a/src/detection/fp_utils.cc b/src/detection/fp_utils.cc index 6616b8b61..c9cf278df 100644 --- a/src/detection/fp_utils.cc +++ b/src/detection/fp_utils.cc @@ -127,6 +127,18 @@ PatternMatchData* get_pmd(OptFpList* ofl, SnortProtocolId snort_protocol_id, Rul return ofl->ips_opt->get_pattern(snort_protocol_id, direction); } +bool is_fast_pattern_only(OptFpList* ofl, Mpse::MpseType mpse_type) +{ + PatternMatchData* pmd = get_pmd(ofl, UNKNOWN_PROTOCOL_ID, RULE_WO_DIR); + + if ( !pmd ) + return false; + + assert((mpse_type == Mpse::MPSE_TYPE_NORMAL) or (mpse_type == Mpse::MPSE_TYPE_OFFLOAD)); + + return (pmd->fp_only & (1 << mpse_type)); +} + bool is_fast_pattern_only(OptFpList* ofl) { PatternMatchData* pmd = get_pmd(ofl, UNKNOWN_PROTOCOL_ID, RULE_WO_DIR); @@ -249,7 +261,10 @@ bool FpSelector::is_better_than( if ( pmd->is_fast_pattern() ) { ParseWarning(WARN_RULES, "content ineligible for fast_pattern matcher - ignored"); - pmd->flags &= ~PatternMatchData::FAST_PAT; + // When we have a normal search engine we do not wish to invalidate the user + // indicated fast pattern as this may be a valid fast pattern for use in the offload + // search engine + // pmd->flags &= ~PatternMatchData::FAST_PAT; } return false; } diff --git a/src/detection/fp_utils.h b/src/detection/fp_utils.h index 6b3241675..5e0ac19e3 100644 --- a/src/detection/fp_utils.h +++ b/src/detection/fp_utils.h @@ -24,11 +24,13 @@ // fast pattern utilities #include #include "framework/ips_option.h" +#include "framework/mpse.h" struct OptFpList; struct OptTreeNode; struct PatternMatchData* get_pmd(OptFpList*, SnortProtocolId, snort::RuleDirection); +bool is_fast_pattern_only(OptFpList*, snort::Mpse::MpseType); bool is_fast_pattern_only(OptFpList*); void validate_fast_pattern(OptTreeNode*); diff --git a/src/detection/ips_context.h b/src/detection/ips_context.h index a6cd91763..cd3ea9a1d 100644 --- a/src/detection/ips_context.h +++ b/src/detection/ips_context.h @@ -28,6 +28,7 @@ #include "main/snort_types.h" #include "framework/codec.h" #include "framework/mpse.h" +#include "framework/mpse_batch.h" // required to get a decent decl of pkth #include "protocols/packet.h" diff --git a/src/detection/regex_offload.cc b/src/detection/regex_offload.cc index 10fe2b288..71ad8c186 100644 --- a/src/detection/regex_offload.cc +++ b/src/detection/regex_offload.cc @@ -33,12 +33,13 @@ #include #include -#include "main/snort_config.h" -#include "latency/packet_latency.h" -#include "latency/rule_latency.h" #include "fp_detect.h" #include "ips_context.h" +#include "latency/packet_latency.h" +#include "latency/rule_latency.h" +#include "main/snort_config.h" +// FIXIT-L this could be offloader specific struct RegexRequest { snort::Packet* packet = nullptr; @@ -52,8 +53,16 @@ struct RegexRequest bool go = true; }; +RegexOffload* RegexOffload::get_offloader(unsigned max, bool async) +{ + if ( async ) + return new ThreadRegexOffload(max); + + return new MpseRegexOffload(max); +} + //-------------------------------------------------------------------------- -// regex offload implementation +// base offload implementation //-------------------------------------------------------------------------- RegexOffload::RegexOffload(unsigned max) @@ -61,7 +70,6 @@ RegexOffload::RegexOffload(unsigned max) for ( unsigned i = 0; i < max; ++i ) { RegexRequest* req = new RegexRequest; - req->thread = new std::thread(worker, req, snort::SnortConfig::get_conf()); idle.emplace_back(req); } } @@ -71,82 +79,137 @@ RegexOffload::~RegexOffload() assert(busy.empty()); for ( auto* req : idle ) - { - req->thread->join(); - delete req->thread; delete req; - } } void RegexOffload::stop() { assert(busy.empty()); +} - for ( auto* req : idle ) +bool RegexOffload::on_hold(snort::Flow* f) const +{ + for ( auto* req : busy ) { - std::unique_lock lock(req->mutex); - req->go = false; - req->cond.notify_one(); + if ( req->packet->flow == f ) + return true; } + return false; } -void RegexOffload::worker(RegexRequest* req, snort::SnortConfig* initial_config) +//-------------------------------------------------------------------------- +// synchronous (ie non) offload implementation +//-------------------------------------------------------------------------- + +MpseRegexOffload::MpseRegexOffload(unsigned max) : RegexOffload(max) { } + +void MpseRegexOffload::put(snort::Packet* p) { - snort::SnortConfig::set_conf(initial_config); + assert(p); + assert(!idle.empty()); - while ( true ) - { - { - std::unique_lock lock(req->mutex); - req->cond.wait_for(lock, std::chrono::seconds(1)); + RegexRequest* req = idle.front(); + idle.pop_front(); // FIXIT-H use splice to move instead + busy.emplace_back(req); - // setting conf is somewhat expensive, checking the conf is not - // this occurs here to take advantage if idling - if ( req->packet and req->packet->context->conf != snort::SnortConfig::get_conf() ) - snort::SnortConfig::set_conf(req->packet->context->conf); + req->packet = p; - if ( !req->go ) - break; + if (p->context->searches.items.size() > 0) + p->context->searches.offload_search(); +} - if ( !req->offload ) +bool MpseRegexOffload::get(snort::Packet*& p) +{ + assert(!busy.empty()); + + for ( auto i = busy.begin(); i != busy.end(); i++ ) + { + RegexRequest* req = *i; + snort::IpsContext* c = req->packet->context; + + if ( c->searches.items.size() > 0 ) + { + snort::Mpse::MpseRespType resp_ret = c->searches.receive_offload_responses(); + + if (resp_ret == snort::Mpse::MPSE_RESP_NOT_COMPLETE) continue; + + else if (resp_ret == snort::Mpse::MPSE_RESP_COMPLETE_FAIL) + { + if (!c->searches.can_fallback()) + { + // FIXIT-M Add peg counts to record offload search fallback attempts + c->searches.search_sync(); + } + // FIXIT-M else Add peg counts to record offload search failures + } + c->searches.items.clear(); } - assert(req->packet); - assert(req->packet->is_offloaded()); - fp_partial(req->packet); + p = req->packet; + req->packet = nullptr; - req->offload = false; + busy.erase(i); + idle.emplace_back(req); + + return true; } - tterm(); + + p = nullptr; + return false; } -void RegexOffload::tterm() +//-------------------------------------------------------------------------- +// async (threads) offload implementation +//-------------------------------------------------------------------------- + +ThreadRegexOffload::ThreadRegexOffload(unsigned max) : RegexOffload(max) { - // FIXIT-M break this over-coupling. In reality we shouldn't be evaluating latency in offload. - PacketLatency::tterm(); - RuleLatency::tterm(); + for ( auto* req : idle ) + req->thread = new std::thread(worker, req, snort::SnortConfig::get_conf()); +} + +ThreadRegexOffload::~ThreadRegexOffload() +{ + for ( auto* req : idle ) + { + req->thread->join(); + delete req->thread; + } } -void RegexOffload::put(snort::Packet* p) +void ThreadRegexOffload::stop() +{ + RegexOffload::stop(); + + for ( auto* req : idle ) + { + std::unique_lock lock(req->mutex); + req->go = false; + req->cond.notify_one(); + } +} + +void ThreadRegexOffload::put(snort::Packet* p) { assert(p); assert(!idle.empty()); RegexRequest* req = idle.front(); - idle.pop_front(); // FIXIT-H use splice to move instead busy.emplace_back(req); std::unique_lock lock(req->mutex); - req->packet = p; - req->offload = true; - req->cond.notify_one(); + if (p->context->searches.items.size() > 0) + { + req->offload = true; + req->cond.notify_one(); + } } -bool RegexOffload::get(snort::Packet*& p) +bool ThreadRegexOffload::get(snort::Packet*& p) { assert(!busy.empty()); @@ -165,18 +228,63 @@ bool RegexOffload::get(snort::Packet*& p) return true; } - + p = nullptr; return false; } -bool RegexOffload::on_hold(snort::Flow* f) +void ThreadRegexOffload::worker(RegexRequest* req, snort::SnortConfig* initial_config) { - for ( auto* req : busy ) + snort::SnortConfig::set_conf(initial_config); + + while ( true ) { - if ( req->packet->flow == f ) - return true; + { + std::unique_lock lock(req->mutex); + req->cond.wait_for(lock, std::chrono::seconds(1)); + + // setting conf is somewhat expensive, checking the conf is not + // this occurs here to take advantage if idling + if ( req->packet and req->packet->context->conf != snort::SnortConfig::get_conf() ) + snort::SnortConfig::set_conf(req->packet->context->conf); + + if ( !req->go ) + break; + + if ( !req->offload ) + continue; + } + + assert(req->packet); + assert(req->packet->is_offloaded()); + assert(req->packet->context->searches.items.size() > 0); + + snort::MpseBatch& batch = req->packet->context->searches; + batch.offload_search(); + snort::Mpse::MpseRespType resp_ret; + + do + { + resp_ret = batch.receive_offload_responses(); + } + while (resp_ret == snort::Mpse::MPSE_RESP_NOT_COMPLETE); + + if (resp_ret == snort::Mpse::MPSE_RESP_COMPLETE_FAIL) + { + if (!batch.can_fallback()) + { + // FIXIT-M Add peg counts to record offload search fallback attempts + batch.search_sync(); + } + // FIXIT-M else Add peg counts to record offload search failures + } + + batch.items.clear(); + req->offload = false; } - return false; + + // FIXIT-M break this over-coupling. In reality we shouldn't be evaluating latency in offload. + PacketLatency::tterm(); + RuleLatency::tterm(); } diff --git a/src/detection/regex_offload.h b/src/detection/regex_offload.h index 3c5fb964a..0351f8ffb 100644 --- a/src/detection/regex_offload.h +++ b/src/detection/regex_offload.h @@ -22,12 +22,11 @@ #define REGEX_OFFLOAD_H // RegexOffload provides an interface to fast pattern search accelerators. -// currently implemented as a simple thread offload, but will become an -// abstract base class with true hardware offload subclasses. for starters -// the thread offload will "cheat" and tightly interface with fp_detect but -// eventually morph into such a proper subclass as the offload api emerges. -// presently all offload is per packet thread; packet threads do not share -// offload resources. +// There are two flavors: MPSE and thread. The MpseRegexOffload interfaces to +// an MPSE that is capable of regex offload such as the RXP whereas +// ThreadRegexOffload implements the regex search in auxiliary threads w/o +// requiring extra MPSE instances. presently all offload is per packet thread; +// packet threads do not share offload resources. #include #include @@ -45,30 +44,53 @@ struct RegexRequest; class RegexOffload { public: - RegexOffload(unsigned max); - ~RegexOffload(); + static RegexOffload* get_offloader(unsigned max, bool async); + virtual ~RegexOffload(); + + virtual void stop(); - void stop(); + virtual void put(snort::Packet*) = 0; + virtual bool get(snort::Packet*&) = 0; - unsigned available() + unsigned available() const { return idle.size(); } - unsigned count() + unsigned count() const { return busy.size(); } - void put(snort::Packet*); - bool get(snort::Packet*&); - - bool on_hold(snort::Flow*); + bool on_hold(snort::Flow*) const; -private: - static void worker(RegexRequest*, snort::SnortConfig*); - static void tterm(); +protected: + RegexOffload(unsigned max); -private: +protected: std::list busy; std::list idle; }; +class MpseRegexOffload : public RegexOffload +{ +public: + MpseRegexOffload(unsigned max); + + void put(snort::Packet*) override; + bool get(snort::Packet*&) override; +}; + +class ThreadRegexOffload : public RegexOffload +{ +public: + ThreadRegexOffload(unsigned max); + ~ThreadRegexOffload(); + + void stop() override; + + void put(snort::Packet*) override; + bool get(snort::Packet*&) override; + +private: + static void worker(RegexRequest*, snort::SnortConfig*); +}; + #endif diff --git a/src/file_api/file_log.cc b/src/file_api/file_log.cc index 73543cb47..5eb08341a 100644 --- a/src/file_api/file_log.cc +++ b/src/file_api/file_log.cc @@ -97,7 +97,7 @@ void LogHandler::log_file_name(TextLog* log, FileContext* file) { std::string& name = file->get_file_name(); - if (name.length() <= 0) + if ( name.empty() ) return; size_t fname_len = name.length(); diff --git a/src/framework/CMakeLists.txt b/src/framework/CMakeLists.txt index 4de094cb5..bd7747121 100644 --- a/src/framework/CMakeLists.txt +++ b/src/framework/CMakeLists.txt @@ -15,6 +15,7 @@ set (FRAMEWORK_INCLUDES lua_api.h module.h mpse.h + mpse_batch.h parameter.h range.h so_rule.h @@ -32,6 +33,7 @@ add_library ( framework OBJECT parameter.cc module.cc mpse.cc + mpse_batch.cc range.cc value.cc ) diff --git a/src/framework/mpse.cc b/src/framework/mpse.cc index df0df79b9..b6e41633d 100644 --- a/src/framework/mpse.cc +++ b/src/framework/mpse.cc @@ -23,9 +23,16 @@ #include "mpse.h" +#include + #include "profiler/profiler_defs.h" #include "search_engines/pat_stats.h" +#include "managers/mpse_manager.h" +#include "managers/module_manager.h" +#include "main/snort_config.h" +#include "detection/fp_config.h" +#include "mpse_batch.h" using namespace std; @@ -62,20 +69,35 @@ int Mpse::search_all( return _search(T, n, match, context, current_state); } -void Mpse::search(MpseBatch& batch) +void Mpse::search(MpseBatch& batch, MpseType mpse_type) { int start_state; for ( auto& item : batch.items ) { + if (item.second.done) + continue; + + item.second.error = false; + item.second.matches = 0; + for ( auto& so : item.second.so ) { start_state = 0; - so->search(item.first.buf, item.first.len, batch.mf, batch.context, &start_state); + switch (mpse_type) + { + case MPSE_TYPE_NORMAL: + item.second.matches += so->get_normal_mpse()->search(item.first.buf, + item.first.len, batch.mf, batch.context, &start_state); + break; + case MPSE_TYPE_OFFLOAD: + item.second.matches += so->get_offload_mpse()->search(item.first.buf, + item.first.len, batch.mf, batch.context, &start_state); + break; + } } item.second.done = true; } - batch.items.clear(); } } diff --git a/src/framework/mpse.h b/src/framework/mpse.h index 319cf9dcf..63b13e9ba 100644 --- a/src/framework/mpse.h +++ b/src/framework/mpse.h @@ -25,6 +25,7 @@ // machine from the patterns, and search either a single buffer or a set // of (related) buffers for patterns. +#include #include #include #include @@ -45,48 +46,22 @@ struct MpseApi; struct MpseBatch; struct ProfileStats; -template -struct MpseBatchKey +class SO_PUBLIC Mpse { - BUF buf; - LEN len; - MpseBatchKey(BUF b, LEN n) - { - this->buf = b; - this->len = n; - } - - bool operator==(const MpseBatchKey &k) const +public: + enum MpseType { - return buf == k.buf && len == k.len; - } -}; + MPSE_TYPE_NORMAL = 0, + MPSE_TYPE_OFFLOAD = 1 + }; -struct MpseBatchKeyHash -{ - template - std::size_t operator()(const MpseBatchKey &k) const + enum MpseRespType { - std::size_t h1 = std::hash()(k.buf); - std::size_t h2 = std::hash()(k.len); - - return h1 ^ h2; - } -}; - -class MpseBatchItem -{ -public: - std::vector so; - bool done; - - MpseBatchItem(Mpse* s = nullptr) - { if (s) so.push_back(s); done = false; } -}; + MPSE_RESP_COMPLETE_FAIL = -1, + MPSE_RESP_NOT_COMPLETE = 0, + MPSE_RESP_COMPLETE_SUCCESS = 1 + }; -class SO_PUBLIC Mpse -{ -public: virtual ~Mpse() = default; struct PatternDescriptor @@ -113,9 +88,10 @@ public: virtual int search_all( const uint8_t* T, int n, MpseMatch, void* context, int* current_state); - virtual void search(MpseBatch& batch); + virtual void search(MpseBatch& batch, MpseType mpse_type); - virtual bool receive_responses(MpseBatch&) { return true; } + virtual MpseRespType receive_responses(MpseBatch&) + { return MPSE_RESP_COMPLETE_SUCCESS; } virtual void set_opt(int) { } virtual int print_info() { return 0; } @@ -139,19 +115,6 @@ private: const MpseApi* api; }; -struct MpseBatch -{ - MpseMatch mf; - void* context; - std::unordered_map, MpseBatchItem, MpseBatchKeyHash> items; - - void search(MpseBatch& batch) - { items.begin()->second.so[0]->search(batch); } - - bool receive_responses(MpseBatch& batch) - { return items.begin()->second.so[0]->receive_responses(batch); } -}; - extern THREAD_LOCAL ProfileStats mpsePerfStats; typedef void (* MpseOptFunc)(SnortConfig*); @@ -165,6 +128,7 @@ typedef void (* MpseDelFunc)(Mpse*); #define MPSE_BASE 0x00 #define MPSE_TRIM 0x01 #define MPSE_REGEX 0x02 +#define MPSE_ASYNC 0x04 struct MpseApi { diff --git a/src/framework/mpse_batch.cc b/src/framework/mpse_batch.cc new file mode 100644 index 000000000..84a8c4737 --- /dev/null +++ b/src/framework/mpse_batch.cc @@ -0,0 +1,179 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2018-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// mpse_batch.cc author Titan IC Systems + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "mpse_batch.h" + +#include "profiler/profiler_defs.h" +#include "search_engines/pat_stats.h" +#include "managers/mpse_manager.h" +#include "managers/module_manager.h" +#include "main/snort_config.h" +#include "detection/fp_config.h" + +#include + +using namespace std; + +namespace snort +{ + +//------------------------------------------------------------------------- +// batch stuff +//------------------------------------------------------------------------- + +bool MpseBatch::search_sync() +{ + bool searches = false; + + if (items.size() > 0) + { + Mpse::MpseRespType resp_ret; + searches = true; + + search(); + do + { + resp_ret = receive_responses(); + } + while (resp_ret == Mpse::MPSE_RESP_NOT_COMPLETE); + + // Assumption here is that a normal search engine will never fail + } + items.clear(); + + return searches; +} + +//------------------------------------------------------------------------- +// group stuff +//------------------------------------------------------------------------- + +MpseGroup::~MpseGroup() +{ + if (normal_mpse) + { + MpseManager::delete_search_engine(normal_mpse); + normal_mpse = nullptr; + } + if (offload_mpse) + { + MpseManager::delete_search_engine(offload_mpse); + offload_mpse = nullptr; + } +} + +bool MpseGroup::create_normal_mpse(SnortConfig* sc, const MpseAgent* agent) +{ + FastPatternConfig* fp = sc->fast_pattern_config; + const MpseApi* search_api = nullptr; + + if (fp) + search_api = fp->get_search_api(); + + if (search_api != nullptr) + { + normal_mpse = MpseManager::get_search_engine(sc, search_api, agent); + return true; + } + else + { + normal_mpse = nullptr; + return false; + } +} + +bool MpseGroup::create_normal_mpse(const char* type) +{ + if ( !type and SnortConfig::get_conf()->fast_pattern_config ) + type = SnortConfig::get_conf()->fast_pattern_config->get_search_method(); + + if ( !type ) + type = "ac_bnfa"; + + const MpseApi* search_api = MpseManager::get_search_api(type); + + if (search_api) + { + Module* mod = ModuleManager::get_module(search_api->base.name); + normal_mpse = search_api->ctor(nullptr, mod, nullptr); + normal_mpse->set_api(search_api); + return true; + } + else + { + normal_mpse = nullptr; + return false; + } +} + +bool MpseGroup::create_offload_mpse(SnortConfig* sc, const MpseAgent* agent) +{ + FastPatternConfig* fp = sc->fast_pattern_config; + const MpseApi* search_api = nullptr; + const MpseApi* offload_search_api = nullptr; + + if (fp) + { + search_api = fp->get_search_api(); + offload_search_api = fp->get_offload_search_api(); + } + + if (offload_search_api and (offload_search_api != search_api)) + { + offload_mpse = MpseManager::get_search_engine(sc, offload_search_api, agent); + return true; + } + else + { + offload_mpse = nullptr; + return false; + } +} + +bool MpseGroup::create_offload_mpse() +{ + const MpseApi* search_api = nullptr; + const MpseApi* offload_search_api = nullptr; + + if (SnortConfig::get_conf()->fast_pattern_config ) + { + search_api = SnortConfig::get_conf()->fast_pattern_config->get_search_api(); + offload_search_api = SnortConfig::get_conf()->fast_pattern_config->get_offload_search_api(); + } + + if (offload_search_api and (offload_search_api != search_api)) + { + Module* mod = ModuleManager::get_module(offload_search_api->base.name); + offload_mpse = offload_search_api->ctor(nullptr, mod, nullptr); + offload_mpse->set_api(offload_search_api); + return true; + } + else + { + offload_mpse = nullptr; + return false; + } +} + +} + diff --git a/src/framework/mpse_batch.h b/src/framework/mpse_batch.h new file mode 100644 index 000000000..0903e4c76 --- /dev/null +++ b/src/framework/mpse_batch.h @@ -0,0 +1,155 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2018-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// mpse_batch.h author Titan IC Systems + +#ifndef MPSE_BATCH_H +#define MPSE_BATCH_H + +#include "framework/mpse.h" +#include "main/snort_types.h" + +namespace snort +{ +class SO_PUBLIC MpseGroup +{ +public: + MpseGroup() + { normal_mpse = nullptr; offload_mpse = nullptr; } + + MpseGroup(Mpse* normal) + { normal_mpse = normal; offload_mpse = nullptr; } + + ~MpseGroup(); + + Mpse* get_normal_mpse() const + { return normal_mpse; } + + // Offload will only be actioned if the offload_search_api is configured. + // If the offload_search_api is the same as the normal_search_api then + // only the normal mpse will get created and thus if the offload mpse + // is requested the normal mpse will be returned otherwise the offload_mpse + // will get returned + Mpse* get_offload_mpse() const + { return offload_mpse ? offload_mpse : normal_mpse; } + + bool create_normal_mpse(SnortConfig*, const MpseAgent* agent); + bool create_normal_mpse(const char*); + + bool create_offload_mpse(SnortConfig*, const MpseAgent* agent); + bool create_offload_mpse(); + + inline bool can_fallback() const + { return get_offload_mpse() != normal_mpse; } + +public: // FIXIT-L privatize + Mpse* normal_mpse; + Mpse* offload_mpse; +}; + +template +struct MpseBatchKey +{ + BUF buf; + LEN len; + MpseBatchKey(BUF b, LEN n) + { + this->buf = b; + this->len = n; + } + + bool operator==(const MpseBatchKey &k) const + { + return buf == k.buf && len == k.len; + } +}; + +struct MpseBatchKeyHash +{ + template + std::size_t operator()(const MpseBatchKey &k) const + { + std::size_t h1 = std::hash()(k.buf); + std::size_t h2 = std::hash()(k.len); + + return h1 ^ h2; + } +}; + +class MpseBatchItem +{ +public: + std::vector so; + bool done; + bool error; + int matches; + + MpseBatchItem(MpseGroup* s = nullptr) + { if (s) so.push_back(s); done = false; error = false; matches = 0; } +}; + +struct MpseBatch +{ + MpseMatch mf; + void* context; + std::unordered_map, MpseBatchItem, MpseBatchKeyHash> items; + + void search(); + Mpse::MpseRespType receive_responses(); + + void offload_search(); + Mpse::MpseRespType receive_offload_responses(); + + bool search_sync(); + bool can_fallback() const; +}; + +inline void MpseBatch::search() +{ + items.begin()->second.so[0]->get_normal_mpse()-> + search(*this, Mpse::MPSE_TYPE_NORMAL); +} + +inline Mpse::MpseRespType MpseBatch::receive_responses() +{ + return items.begin()->second.so[0]->get_normal_mpse()->receive_responses(*this); +} + +inline void MpseBatch::offload_search() +{ + assert(items.begin()->second.so[0]->get_offload_mpse()); + + items.begin()->second.so[0]->get_offload_mpse()-> + search(*this, Mpse::MPSE_TYPE_OFFLOAD); +} + +inline Mpse::MpseRespType MpseBatch::receive_offload_responses() +{ + assert(items.begin()->second.so[0]->get_offload_mpse()); + + return items.begin()->second.so[0]->get_offload_mpse()-> + receive_responses(*this); +} + +inline bool MpseBatch::can_fallback() const +{ + return items.begin()->second.so[0]->can_fallback(); +} +} + +#endif + diff --git a/src/hash/lru_cache_shared.h b/src/hash/lru_cache_shared.h index beeb19317..695350277 100644 --- a/src/hash/lru_cache_shared.h +++ b/src/hash/lru_cache_shared.h @@ -149,7 +149,7 @@ bool LruCacheShared::set_max_size(size_t newsize) { LruListIter list_iter; - if (newsize <= 0) + if (newsize == 0) return false; // Not allowed to set size to zero. std::lock_guard cache_lock(cache_mutex); diff --git a/src/helpers/base64_encoder.cc b/src/helpers/base64_encoder.cc index de4b679bf..4e2a99f0f 100644 --- a/src/helpers/base64_encoder.cc +++ b/src/helpers/base64_encoder.cc @@ -126,7 +126,7 @@ unsigned Base64Encoder::finish(char* buf) #ifdef UNIT_TEST TEST_CASE("b64 decode", "[Base64Encoder]") { - Base64Encoder b64; + Base64Encoder b64e; const char* text = "The quick brown segment jumped over the lazy dogs.\n"; const char* code = "VGhlIHF1aWNrIGJyb3duIHNlZ21lbnQganVtcGVkIG92ZXIgdGhlIGxhenkgZG9ncy4K"; @@ -135,17 +135,17 @@ TEST_CASE("b64 decode", "[Base64Encoder]") SECTION("no decode") { - CHECK(!b64.finish(buf)); + CHECK(!b64e.finish(buf)); } SECTION("null data") { - CHECK(!b64.encode(nullptr, 0, buf)); - CHECK(!b64.finish(buf)); + CHECK(!b64e.encode(nullptr, 0, buf)); + CHECK(!b64e.finish(buf)); } SECTION("zero length data") { - CHECK(!b64.encode((const uint8_t*)"ignore", 0, buf)); - CHECK(!b64.finish(buf)); + CHECK(!b64e.encode((const uint8_t*)"ignore", 0, buf)); + CHECK(!b64e.finish(buf)); } SECTION("finish states") { @@ -156,20 +156,20 @@ TEST_CASE("b64 decode", "[Base64Encoder]") for ( unsigned i = 0; i < to_do; ++i ) { - unsigned n = b64.encode((const uint8_t*)txt[i], strlen(txt[i]), buf); - n += b64.finish(buf+n); + unsigned n = b64e.encode((const uint8_t*)txt[i], strlen(txt[i]), buf); + n += b64e.finish(buf+n); REQUIRE(n < sizeof(buf)); buf[n] = 0; CHECK(!strcmp(buf, exp[i])); - b64.reset(); + b64e.reset(); } } SECTION("one shot") { - unsigned n = b64.encode((const uint8_t*)text, strlen(text), buf); - n += b64.finish(buf+n); + unsigned n = b64e.encode((const uint8_t*)text, strlen(text), buf); + n += b64e.finish(buf+n); REQUIRE(n < sizeof(buf)); buf[n] = 0; @@ -189,16 +189,16 @@ TEST_CASE("b64 decode", "[Base64Encoder]") while ( offset < len ) { unsigned k = (offset + chunk > len) ? len - offset : chunk; - n += b64.encode((const uint8_t*)text+offset, k, buf+n); + n += b64e.encode((const uint8_t*)text+offset, k, buf+n); offset += k; } - n += b64.finish(buf+n); + n += b64e.finish(buf+n); REQUIRE(n < sizeof(buf)); buf[n] = 0; CHECK(!strcmp(buf, code)); - b64.reset(); + b64e.reset(); } } } diff --git a/src/ips_options/ips_sd_pattern.cc b/src/ips_options/ips_sd_pattern.cc index 909b46208..c29b4867c 100644 --- a/src/ips_options/ips_sd_pattern.cc +++ b/src/ips_options/ips_sd_pattern.cc @@ -327,8 +327,8 @@ public: private: SdPatternConfig config; - static void scratch_setup(SnortConfig* sc); - static void scratch_cleanup(SnortConfig* sc); + static void scratch_setup(SnortConfig*); + static void scratch_cleanup(SnortConfig*); }; bool SdPatternModule::begin(const char*, int, SnortConfig*) @@ -337,7 +337,7 @@ bool SdPatternModule::begin(const char*, int, SnortConfig*) return true; } -bool SdPatternModule::set(const char*, Value& v, SnortConfig* sc) +bool SdPatternModule::set(const char*, Value& v, SnortConfig*) { if ( v.is("~pattern") ) { diff --git a/src/loggers/unified2.cc b/src/loggers/unified2.cc index f33acbb08..bbc78de71 100644 --- a/src/loggers/unified2.cc +++ b/src/loggers/unified2.cc @@ -658,9 +658,11 @@ static void _AlertIP4_v2(Packet* p, const char*, Unified2Config* config, const E alertdata.sport_itype = htons(p->ptrs.icmph->type); alertdata.dport_icode = htons(p->ptrs.icmph->code); } - - alertdata.sport_itype = htons(p->ptrs.sp); - alertdata.dport_icode = htons(p->ptrs.dp); + else + { + alertdata.sport_itype = htons(p->ptrs.sp); + alertdata.dport_icode = htons(p->ptrs.dp); + } if ( p->proto_bits & PROTO_BIT__MPLS ) alertdata.mpls_label = htonl(p->ptrs.mplsHdr.label); @@ -744,9 +746,11 @@ static void _AlertIP6_v2(Packet* p, const char*, Unified2Config* config, const E alertdata.sport_itype = htons(p->ptrs.icmph->type); alertdata.dport_icode = htons(p->ptrs.icmph->code); } - - alertdata.sport_itype = htons(p->ptrs.sp); - alertdata.dport_icode = htons(p->ptrs.dp); + else + { + alertdata.sport_itype = htons(p->ptrs.sp); + alertdata.dport_icode = htons(p->ptrs.dp); + } if ( p->proto_bits & PROTO_BIT__MPLS ) alertdata.mpls_label = htonl(p->ptrs.mplsHdr.label); diff --git a/src/main/modules.cc b/src/main/modules.cc index 02a9ca7f5..2911f7657 100755 --- a/src/main/modules.cc +++ b/src/main/modules.cc @@ -252,6 +252,9 @@ static const Parameter search_engine_params[] = { "search_method", Parameter::PT_DYNAMIC, (void*)&get_search_methods, "ac_bnfa", "set fast pattern algorithm - choose available search engine" }, + { "offload_search_method", Parameter::PT_DYNAMIC, (void*)&get_search_methods, nullptr, + "set fast pattern offload algorithm - choose available search engine" }, + { "search_optimize", Parameter::PT_BOOL, nullptr, "true", "tweak state machine construction for better performance" }, @@ -356,6 +359,11 @@ bool SearchEngineModule::set(const char*, Value& v, SnortConfig* sc) if ( !fp->set_search_method(v.get_string()) ) return false; } + else if ( v.is("offload_search_method") ) + { + if ( !fp->set_offload_search_method(v.get_string()) ) + return false; + } else if ( v.is("search_optimize") ) fp->set_search_opt(v.get_bool()); diff --git a/src/main/snort.cc b/src/main/snort.cc index d996be838..ee65962b5 100644 --- a/src/main/snort.cc +++ b/src/main/snort.cc @@ -331,8 +331,14 @@ void Snort::init(int argc, char** argv) SnortConfig::get_conf()->post_setup(); - MpseManager::activate_search_engine( - SnortConfig::get_conf()->fast_pattern_config->get_search_api(), SnortConfig::get_conf()); + const MpseApi* search_api = SnortConfig::get_conf()->fast_pattern_config->get_search_api(); + const MpseApi* offload_search_api = SnortConfig::get_conf()->fast_pattern_config-> + get_offload_search_api(); + + MpseManager::activate_search_engine(search_api, SnortConfig::get_conf()); + + if ((offload_search_api != nullptr) and (offload_search_api != search_api)) + MpseManager::activate_search_engine(offload_search_api, SnortConfig::get_conf()); SFAT_Start(); diff --git a/src/managers/mpse_manager.cc b/src/managers/mpse_manager.cc index a7a7973c7..18ae48d8b 100644 --- a/src/managers/mpse_manager.cc +++ b/src/managers/mpse_manager.cc @@ -171,6 +171,12 @@ bool MpseManager::search_engine_trim(const MpseApi* api) return (api->flags & MPSE_TRIM) != 0; } +bool MpseManager::is_async_capable(const MpseApi* api) +{ + assert(api); + return (api->flags & MPSE_ASYNC) != 0; +} + bool MpseManager::is_regex_capable(const MpseApi* api) { assert(api); diff --git a/src/managers/mpse_manager.h b/src/managers/mpse_manager.h index 859bda6da..d6a84616b 100644 --- a/src/managers/mpse_manager.h +++ b/src/managers/mpse_manager.h @@ -76,6 +76,7 @@ public: static void start_search_engine(const snort::MpseApi*); static void stop_search_engine(const snort::MpseApi*); static bool search_engine_trim(const snort::MpseApi*); + static bool is_async_capable(const snort::MpseApi*); static bool is_regex_capable(const snort::MpseApi*); static void print_mpse_summary(const snort::MpseApi*); static void print_search_engine_stats(); diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc index 81d6de560..5f4284b5a 100644 --- a/src/network_inspectors/appid/appid_discovery.cc +++ b/src/network_inspectors/appid/appid_discovery.cc @@ -358,12 +358,9 @@ static bool set_network_attributes(AppIdSession* asd, Packet* p, IpProtocol& pro static bool is_packet_ignored(AppIdSession* asd, Packet* p, AppidSessionDirection direction) { -// FIXIT-M - Need to convert this _dpd stream api call to the correct snort++ method #ifdef REMOVED_WHILE_NOT_IN_USE - bool is_http2 = false; // _dpd.streamAPI->is_session_http2(p->flow); -#else - bool is_http2 = false; -#endif + bool is_http2 = false; // FIXIT-M _dpd.streamAPI->is_session_http2(p->flow); + if (is_http2) { if (asd) @@ -375,7 +372,9 @@ static bool is_packet_ignored(AppIdSession* asd, Packet* p, AppidSessionDirectio return true; } } - else if ( p->is_rebuilt() && !p->flow->is_proxied() ) + else +#endif + if ( p->is_rebuilt() && !p->flow->is_proxied() ) { // FIXIT-M: In snort2x, a rebuilt packet was ignored whether it had a session or not. // Here, we are ignoring rebuilt packet only if it has a session. Why? diff --git a/src/network_inspectors/appid/appid_utils/network_set.cc b/src/network_inspectors/appid/appid_utils/network_set.cc index 3e683462e..84753b57d 100644 --- a/src/network_inspectors/appid/appid_utils/network_set.cc +++ b/src/network_inspectors/appid/appid_utils/network_set.cc @@ -972,12 +972,12 @@ int NetworkSetManager::reduce(NetworkSet* network_set) } network_set->pnetwork = (Network**)snort_calloc(count * sizeof(Network*)); SF_LNODE* iter = nullptr; - int i = 0; + int k = 0; for (network = (Network*)sflist_first(&network_set->networks, &iter); - network && i < count; + network && k < count; network = (Network*)sflist_next(&iter)) { - network_set->pnetwork[i++] = network; + network_set->pnetwork[k++] = network; } /* bubble sort this array */ for (int i = (count - 1); i >= 0; i--) @@ -1035,12 +1035,12 @@ int NetworkSetManager::reduce(NetworkSet* network_set) } network_set->pnetwork6 = (Network6**)snort_calloc(count * sizeof(Network6*)); SF_LNODE* iter = nullptr; - int i = 0; + int k = 0; for (network6 = (Network6*)sflist_first(&network_set->networks6, &iter); - network6 && i < count; + network6 && k < count; network6 = (Network6*)sflist_next(&iter)) { - network_set->pnetwork6[i++] = network6; + network_set->pnetwork6[k++] = network6; } /* bubble sort this array */ for (int i = (count - 1); i >= 0; i--) diff --git a/src/parser/vars.cc b/src/parser/vars.cc index 24e2d91d3..321c0bed0 100644 --- a/src/parser/vars.cc +++ b/src/parser/vars.cc @@ -511,8 +511,7 @@ VarEntry* VarDefine( /* Check if this is a variable that stores an IP */ else if (*value == '$') { - sfip_var_t* var; - if ((var = sfvt_lookup_var(ip_vartable, value)) != nullptr) + if ( sfvt_lookup_var(ip_vartable, value) ) { sfvt_define(ip_vartable, name, value); return nullptr; diff --git a/src/ports/port_group.cc b/src/ports/port_group.cc index 2b4b17928..24c1876bc 100644 --- a/src/ports/port_group.cc +++ b/src/ports/port_group.cc @@ -24,7 +24,8 @@ #include "port_group.h" #include "detection/detection_options.h" -#include "managers/mpse_manager.h" +#include "framework/mpse.h" +#include "framework/mpse_batch.h" #include "utils/util.h" void PortGroup::add_rule() @@ -41,10 +42,10 @@ void PortGroup::free(PortGroup* pg) for (int i = PM_TYPE_PKT; i < PM_TYPE_MAX; i++) { - if (pg->mpse[i] != nullptr) + if (pg->mpsegrp[i]) { - MpseManager::delete_search_engine(pg->mpse[i]); - pg->mpse[i] = nullptr; + delete pg->mpsegrp[i]; + pg->mpsegrp[i] = nullptr; } } diff --git a/src/ports/port_group.h b/src/ports/port_group.h index ef7fda0d6..6521ee9b5 100644 --- a/src/ports/port_group.h +++ b/src/ports/port_group.h @@ -27,7 +27,7 @@ namespace snort { - class Mpse; + class MpseGroup; } // PortGroup contains a set of fast patterns in the form of an MPSE and a @@ -65,7 +65,7 @@ struct PortGroup RULE_NODE* nfp_head, * nfp_tail; // pattern matchers - snort::Mpse* mpse[PM_TYPE_MAX]; + snort::MpseGroup* mpsegrp[PM_TYPE_MAX]; // detection option tree void* nfp_tree; diff --git a/src/protocols/packet_manager.cc b/src/protocols/packet_manager.cc index 4d8485706..e424245f1 100644 --- a/src/protocols/packet_manager.cc +++ b/src/protocols/packet_manager.cc @@ -685,7 +685,7 @@ int PacketManager::encode_format( bool update_ip4_len = false; uint8_t num_layers = p->num_layers; - if ( num_layers <= 0 ) + if ( num_layers == 0 ) return -1; c->reset(); @@ -716,7 +716,7 @@ int PacketManager::encode_format( update_ip4_len = true; } - if (num_layers <= 0) + if (num_layers == 0) return -1; } diff --git a/src/search_engines/search_tool.cc b/src/search_engines/search_tool.cc index ab53ddb84..5a25d3bfb 100644 --- a/src/search_engines/search_tool.cc +++ b/src/search_engines/search_tool.cc @@ -21,26 +21,48 @@ #include "config.h" #endif -#include "search_tool.h" - #include -#include "managers/mpse_manager.h" +#include "detection/fp_config.h" +#include "framework/mpse.h" +#include "framework/mpse_batch.h" +#include "main/snort_config.h" +#include "search_tool.h" namespace snort { SearchTool::SearchTool(const char* method, bool dfa) { - mpse = MpseManager::get_search_engine(method); - assert(mpse); + mpsegrp = new MpseGroup; + + // When no method is passed in, a normal search engine mpse will be created + // with the search method the same as that configured for the fast pattern + // normal search method, and also an offload search engine mpse will be + // created with the search method the same as that configured for the fast + // pattern offload search method. If a method is passed in then an offload + // search engine will not be created + + if (mpsegrp->create_normal_mpse(method)) + { + if( dfa ) + mpsegrp->normal_mpse->set_opt(1); + } + + if (method == nullptr) + { + if (mpsegrp->create_offload_mpse()) + { + if ( dfa ) + mpsegrp->offload_mpse->set_opt(1); + } + } + max_len = 0; - if( dfa ) - mpse->set_opt(1); } SearchTool::~SearchTool() { - MpseManager::delete_search_engine(mpse); + delete mpsegrp; } void SearchTool::add(const char* pat, unsigned len, int id, bool no_case) @@ -62,8 +84,10 @@ void SearchTool::add(const uint8_t* pat, unsigned len, void* id, bool no_case) { Mpse::PatternDescriptor desc(no_case, false, true); - if ( mpse ) - mpse->add_pattern(nullptr, pat, len, desc, id); + if ( mpsegrp->normal_mpse ) + mpsegrp->normal_mpse->add_pattern(nullptr, pat, len, desc, id); + if ( mpsegrp->offload_mpse ) + mpsegrp->offload_mpse->add_pattern(nullptr, pat, len, desc, id); if ( len > max_len ) max_len = len; @@ -71,8 +95,10 @@ void SearchTool::add(const uint8_t* pat, unsigned len, void* id, bool no_case) void SearchTool::prep() { - if ( mpse ) - mpse->prep_patterns(nullptr); + if ( mpsegrp->normal_mpse ) + mpsegrp->normal_mpse->prep_patterns(nullptr); + if ( mpsegrp->offload_mpse ) + mpsegrp->offload_mpse->prep_patterns(nullptr); } int SearchTool::find( @@ -83,6 +109,10 @@ int SearchTool::find( bool confine, void* user_data) { + int num = 0; + SnortConfig* sc = SnortConfig::get_conf(); + FastPatternConfig* fp = sc->fast_pattern_config; + if ( confine && max_len > 0 ) { if ( max_len < len ) @@ -91,7 +121,21 @@ int SearchTool::find( if ( !user_data ) user_data = (void*)str; - int num = mpse->search((const uint8_t*)str, len, mf, user_data, &state); + if ( fp and fp->get_offload_search_api() and (len >= sc->offload_limit) and + (mpsegrp->get_offload_mpse() != mpsegrp->get_normal_mpse()) ) + { + num = mpsegrp->get_offload_mpse()->search((const uint8_t*)str, len, mf, user_data, &state); + + if ( num < 0 ) + num = mpsegrp->get_normal_mpse()->search((const uint8_t*)str, len, mf, user_data, + &state); + } + else + num = mpsegrp->get_normal_mpse()->search((const uint8_t*)str, len, mf, user_data, &state); + + // SeachTool::find expects the number found to be returned so if we have a failure return 0 + if ( num < 0 ) + num = 0; return num; } @@ -114,6 +158,10 @@ int SearchTool::find_all( bool confine, void* user_data) { + int num = 0; + SnortConfig* sc = SnortConfig::get_conf(); + FastPatternConfig* fp = sc->fast_pattern_config; + if ( confine && max_len > 0 ) { if ( max_len < len ) @@ -124,7 +172,23 @@ int SearchTool::find_all( int state = 0; - int num = mpse->search_all((const uint8_t*)str, len, mf, user_data, &state); + if ( fp and fp->get_offload_search_api() and (len >= sc->offload_limit) and + (mpsegrp->get_offload_mpse() != mpsegrp->get_normal_mpse()) ) + { + num = mpsegrp->get_offload_mpse()->search_all((const uint8_t*)str, len, mf, user_data, + &state); + + if ( num < 0 ) + num = mpsegrp->get_normal_mpse()->search_all((const uint8_t*)str, len, mf, user_data, + &state); + } + else + num = mpsegrp->get_normal_mpse()->search_all((const uint8_t*)str, len, mf, user_data, + &state); + + // SeachTool::find expects the number found to be returned so if we have a failure return 0 + if ( num < 0 ) + num = 0; return num; } diff --git a/src/search_engines/search_tool.h b/src/search_engines/search_tool.h index 00c8e6cde..dd8a41821 100644 --- a/src/search_engines/search_tool.h +++ b/src/search_engines/search_tool.h @@ -49,7 +49,7 @@ public: bool confine = false, void* user_data = nullptr); private: - class Mpse* mpse; + class MpseGroup* mpsegrp; unsigned max_len; }; } // namespace snort diff --git a/src/search_engines/test/hyperscan_test.cc b/src/search_engines/test/hyperscan_test.cc index cac67bdf6..9d95fb831 100644 --- a/src/search_engines/test/hyperscan_test.cc +++ b/src/search_engines/test/hyperscan_test.cc @@ -26,6 +26,7 @@ #include "framework/base_api.h" #include "framework/mpse.h" +#include "framework/mpse_batch.h" #include "main/snort_config.h" // must appear after snort_config.h to avoid broken c++ map include @@ -55,20 +56,35 @@ int Mpse::search_all( return _search(T, n, match, context, current_state); } -void Mpse::search(MpseBatch& batch) +void Mpse::search(MpseBatch& batch, MpseType mpse_type) { int start_state; for ( auto& item : batch.items ) { + if (item.second.done) + continue; + + item.second.error = false; + item.second.matches = 0; + for ( auto& so : item.second.so ) { start_state = 0; - so->search(item.first.buf, item.first.len, batch.mf, batch.context, &start_state); + switch (mpse_type) + { + case MPSE_TYPE_NORMAL: + item.second.matches += so->normal_mpse->search(item.first.buf, item.first.len, + batch.mf, batch.context, &start_state); + break; + case MPSE_TYPE_OFFLOAD: + item.second.matches += so->offload_mpse->search(item.first.buf, item.first.len, + batch.mf, batch.context, &start_state); + break; + } } item.second.done = true; } - batch.items.clear(); } SnortConfig s_conf; diff --git a/src/search_engines/test/search_tool_test.cc b/src/search_engines/test/search_tool_test.cc index b86367ca2..89d90a83b 100644 --- a/src/search_engines/test/search_tool_test.cc +++ b/src/search_engines/test/search_tool_test.cc @@ -31,8 +31,9 @@ #include "framework/base_api.h" #include "framework/mpse.h" -#include "managers/mpse_manager.h" +#include "framework/mpse_batch.h" #include "main/snort_config.h" +#include "managers/mpse_manager.h" // must appear after snort_config.h to avoid broken c++ map include #include @@ -111,21 +112,37 @@ int Mpse::search_all( return _search(T, n, match, context, current_state); } -void Mpse::search(MpseBatch& batch) +void Mpse::search(MpseBatch& batch, MpseType mpse_type) { int start_state; for ( auto& item : batch.items ) { + if (item.second.done) + continue; + + item.second.error = false; + item.second.matches = 0; + for ( auto& so : item.second.so ) { start_state = 0; - so->search(item.first.buf, item.first.len, batch.mf, batch.context, &start_state); + switch (mpse_type) + { + case MPSE_TYPE_NORMAL: + item.second.matches += so->normal_mpse->search(item.first.buf, item.first.len, + batch.mf, batch.context, &start_state); + break; + case MPSE_TYPE_OFFLOAD: + item.second.matches += so->offload_mpse->search(item.first.buf, item.first.len, + batch.mf, batch.context, &start_state); + break; + } } item.second.done = true; } - batch.items.clear(); } + } extern const BaseApi* se_ac_bnfa; @@ -160,6 +177,33 @@ void MpseManager::delete_search_engine(Mpse* eng) api->dtor(eng); } +MpseGroup::~MpseGroup() +{ + if (normal_mpse) + { + MpseManager::delete_search_engine(normal_mpse); + normal_mpse = nullptr; + } + if (offload_mpse) + { + MpseManager::delete_search_engine(offload_mpse); + offload_mpse = nullptr; + } +} + +bool MpseGroup::create_normal_mpse(const char* type) +{ + normal_mpse = MpseManager::get_search_engine(type); + + return true; +} + +bool MpseGroup::create_offload_mpse() +{ + offload_mpse = nullptr; + return false; +} + struct ExpectedMatch { int id; @@ -198,7 +242,7 @@ TEST_GROUP(search_tool_bnfa) CHECK(se_ac_bnfa); stool = new SearchTool("ac_bnfa"); - CHECK(stool->mpse); + CHECK(stool->mpsegrp->normal_mpse); int pattern_id = 1; stool->add("the", 3, pattern_id); @@ -288,7 +332,7 @@ TEST_GROUP(search_tool_full) CHECK(se_ac_full); stool = new SearchTool("ac_full", true); - CHECK(stool->mpse); + CHECK(stool->mpsegrp->normal_mpse); int pattern_id = 1; stool->add("the", 3, pattern_id); diff --git a/src/service_inspectors/dce_rpc/dce_smb2.cc b/src/service_inspectors/dce_rpc/dce_smb2.cc index 51720f75b..7cb72a2b5 100644 --- a/src/service_inspectors/dce_rpc/dce_smb2.cc +++ b/src/service_inspectors/dce_rpc/dce_smb2.cc @@ -707,7 +707,6 @@ void DCE2_Smb2Process(DCE2_SmbSsnData* ssd) Packet* p = DetectionEngine::get_current_packet(); const uint8_t* data_ptr = p->data; uint16_t data_len = p->dsize; - uint32_t next_command_offset = 0; /*Check header length*/ if (data_len < sizeof(NbssHdr) + SMB2_HEADER_LENGTH) @@ -722,6 +721,7 @@ void DCE2_Smb2Process(DCE2_SmbSsnData* ssd) if (p->is_pdu_start()) { const Smb2Hdr* smb_hdr = (const Smb2Hdr*)(data_ptr + sizeof(NbssHdr)); + uint32_t next_command_offset; /* SMB protocol allows multiple smb commands to be grouped in a single packet. So loop through to parse all the smb commands. Reference: https://msdn.microsoft.com/en-us/library/cc246614.aspx diff --git a/src/stream/ip/ip_defrag.cc b/src/stream/ip/ip_defrag.cc index 6943a4419..5dc984977 100644 --- a/src/stream/ip/ip_defrag.cc +++ b/src/stream/ip/ip_defrag.cc @@ -403,8 +403,8 @@ static inline int FragCheckFirstLast( } trace_logf(stream_ip, "Frag Status: %s:%s\n", - ft->frag_flags&FRAG_GOT_FIRST ? "FIRST" : "No FIRST", - ft->frag_flags&FRAG_GOT_LAST ? "LAST" : "No LAST"); + (ft->frag_flags&FRAG_GOT_FIRST) ? "FIRST" : "No FIRST", + (ft->frag_flags&FRAG_GOT_LAST) ? "LAST" : "No LAST"); return retVal; } diff --git a/tools/snort2lua/rule_states/rule_pcre.cc b/tools/snort2lua/rule_states/rule_pcre.cc index 6a654619f..4833ee47a 100644 --- a/tools/snort2lua/rule_states/rule_pcre.cc +++ b/tools/snort2lua/rule_states/rule_pcre.cc @@ -39,7 +39,6 @@ public: bool Pcre::convert(std::istringstream& data_stream) { - std::string keyword; bool sticky_buffer_set = false; std::string buffer = "pkt_data";