From: Russ Combs (rucombs) Date: Tue, 8 Dec 2015 19:09:39 +0000 (-0500) Subject: Merge pull request #171 in SNORT/snort3 from crc/fp3 to master X-Git-Tag: 3.0.0-233~695 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=46241bdb55e2b7d84014990eb2cd896e83b11fba;p=thirdparty%2Fsnort3.git Merge pull request #171 in SNORT/snort3 from crc/fp3 to master Squashed commit of the following: commit 89df5b0420788b40e1ed089a15ccc1388ca16920 Author: Russ Combs Date: Tue Dec 8 12:48:38 2015 -0500 updates per review commit cef10abb0ab1af11b9ef6e6970952ce1768c0899 Author: Russ Combs Date: Fri Dec 4 10:34:32 2015 -0500 more fast pattern refactoring refactor mpse queuing and remove _q flavors of mpse convert mpse match index to next byte after match; not start of match bonus - update catch foo --- diff --git a/extra/src/search_engines/CMakeLists.txt b/extra/src/search_engines/CMakeLists.txt index 8525347a3..6ac5d2b9a 100644 --- a/extra/src/search_engines/CMakeLists.txt +++ b/extra/src/search_engines/CMakeLists.txt @@ -1,7 +1,6 @@ add_example_library(lowmem search_engines lowmem.cc - lowmem_q.cc pat_stats.cc sfksearch.cc sfksearch.h diff --git a/extra/src/search_engines/Makefile.am b/extra/src/search_engines/Makefile.am index c3d21923b..a39b800dd 100644 --- a/extra/src/search_engines/Makefile.am +++ b/extra/src/search_engines/Makefile.am @@ -6,7 +6,6 @@ liblowmem_la_LDFLAGS = -export-dynamic -shared liblowmem_la_SOURCES = \ lowmem.cc \ -lowmem_q.cc \ sfksearch.cc \ sfksearch.h \ trie_api.cc diff --git a/extra/src/search_engines/lowmem.cc b/extra/src/search_engines/lowmem.cc index 5db98fa0f..7bcb9c174 100644 --- a/extra/src/search_engines/lowmem.cc +++ b/extra/src/search_engines/lowmem.cc @@ -65,7 +65,7 @@ public: int prep_patterns(SnortConfig* sc) override { - return KTrieCompileWithSnortConf(sc, obj); + return KTrieCompile(sc, obj); } int _search( diff --git a/extra/src/search_engines/lowmem_q.cc b/extra/src/search_engines/lowmem_q.cc deleted file mode 100644 index 048da6ba7..000000000 --- a/extra/src/search_engines/lowmem_q.cc +++ /dev/null @@ -1,209 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2015 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2002-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- -/* -* An abstracted interface to the Multi-Pattern Matching routines, -* thats why we're passing 'void *' objects around. -* -* Marc A Norton -* -* Updates: -* 3/06 - Added AC_BNFA search -*/ - -// lowmem_q.cc author Russ Combs - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include - -#include "sfksearch.h" -#include "main/snort_debug.h" -#include "main/snort_types.h" -#include "framework/mpse.h" -#include "framework/module.h" -#include "log/messages.h" -#include "profiler/profiler.h" - -using namespace std; - -static string s_var; - -static const char* s_name = "lowmem_q"; -static const char* s_help = "MPSE that minimizes memory used"; - -//------------------------------------------------------------------------- -// module stuff -//------------------------------------------------------------------------- - -static const Parameter s_params[] = -{ - { "var", Parameter::PT_STRING, nullptr, nullptr, - "additional print text" }, - - { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } -}; - -class LowmemQModule : public Module -{ -public: - LowmemQModule() : Module(s_name, s_help, s_params) { } - - bool set(const char*, Value&, SnortConfig*) override; - bool begin(const char*, int, SnortConfig*) override; - -public: - string var; -}; - -bool LowmemQModule::set(const char*, Value& v, SnortConfig*) -{ - if ( v.is("var") ) - var = v.get_string(); - - else - return false; - - return true; -} - -bool LowmemQModule::begin(const char*, int, SnortConfig*) -{ - var.clear(); - return true; -} - -//------------------------------------------------------------------------- -// "lowmem_q" -//------------------------------------------------------------------------- - -class LowmemQMpse : public Mpse -{ -private: - KTRIE_STRUCT* obj; - -public: - LowmemQMpse(bool use_gc, const MpseAgent* agent) : Mpse(s_name, use_gc) - { - obj = KTrieNew(1, agent); - } - - ~LowmemQMpse() - { - if (obj) - KTrieDelete(obj); - } - - int add_pattern( - SnortConfig*, const uint8_t* P, unsigned m, - bool noCase, bool negative, void* user) override - { - return KTrieAddPattern(obj, P, m, noCase, negative, user); - } - - int prep_patterns(SnortConfig* sc) override - { - return KTrieCompileWithSnortConf(sc, obj); - } - - int _search( - const uint8_t* T, int n, MpseMatch match, - void* context, int* current_state) override - { - *current_state = 0; - return KTrieSearchQ(obj, T, n, match, context); - } - - int get_pattern_count() override - { - return KTriePatternCount(obj); - } -}; - -//------------------------------------------------------------------------- -// api -//------------------------------------------------------------------------- - -static Module* mod_ctor() -{ return new LowmemQModule; } - -static void mod_dtor(Module* m) -{ delete m; } - -static Mpse* lmq_ctor(SnortConfig*, class Module* mod, bool use_gc, const MpseAgent* agent) -{ - LowmemQModule* lmqm = (LowmemQModule*)mod; - s_var = lmqm->var; - return new LowmemQMpse(use_gc, agent); -} - -static void lmq_dtor(Mpse* p) -{ - delete p; -} - -static void lmq_init() -{ - KTrie_init_xlatcase(); - KTrieInitMemUsed(); -} - -static void lmq_print() -{ - if ( !KTrieMemUsed() ) - return; - - if ( !s_var.empty() ) - LogMessage("lowmemq.var = %s\n", s_var.c_str()); - - double x = (double)KTrieMemUsed(); - - LogMessage("[ LowMem Search-Method Memory Used : %g %s ]\n", - (x > 1.e+6) ? x/1.e+6 : x/1.e+3, - (x > 1.e+6) ? "MBytes" : "KBytes"); -} - -static const MpseApi lmq_api = -{ - { - PT_SEARCH_ENGINE, - sizeof(MpseApi), - SEAPI_VERSION, - 0, - API_RESERVED, - API_OPTIONS, - s_name, - "Keyword Trie (low memory, moderate performance) MPSE with queued events", - mod_ctor, - mod_dtor - }, - false, - nullptr, - nullptr, - nullptr, - nullptr, - lmq_ctor, - lmq_dtor, - lmq_init, - lmq_print, -}; - -const BaseApi* se_lowmem_q = &lmq_api.base; - diff --git a/extra/src/search_engines/sfksearch.cc b/extra/src/search_engines/sfksearch.cc index 42fe54481..a23e3dd4e 100644 --- a/extra/src/search_engines/sfksearch.cc +++ b/extra/src/search_engines/sfksearch.cc @@ -44,8 +44,6 @@ #include "sfksearch.h" #include "search_engines/pat_stats.h" -#define SFKSEARCH_TRACK_Q - static void KTrieFree(KTRIENODE* n); static unsigned int mtot = 0; @@ -484,7 +482,7 @@ static void Build_Bad_Character_Shifts(KTRIE_STRUCT* kt) } } -static int KTrieBuildMatchStateNodeWithSnortConf( +static int KTrieBuildMatchStateNode( SnortConfig* sc, KTRIENODE* root, KTRIE_STRUCT* ts) { int cnt = 0; @@ -520,20 +518,20 @@ static int KTrieBuildMatchStateNodeWithSnortConf( /* for child of this root */ if (root->child) { - cnt += KTrieBuildMatchStateNodeWithSnortConf(sc, root->child, ts); + cnt += KTrieBuildMatchStateNode(sc, root->child, ts); } /* 1st sibling of this root -- other siblings will be processed from * within the processing for root->sibling. */ if (root->sibling) { - cnt += KTrieBuildMatchStateNodeWithSnortConf(sc, root->sibling, ts); + cnt += KTrieBuildMatchStateNode(sc, root->sibling, ts); } return cnt; } -static int KTrieBuildMatchStateTreesWithSnortConf(SnortConfig* sc, KTRIE_STRUCT* ts) +static int KTrieBuildMatchStateTrees(SnortConfig* sc, KTRIE_STRUCT* ts) { int i, cnt = 0; KTRIENODE* root; @@ -545,7 +543,7 @@ static int KTrieBuildMatchStateTreesWithSnortConf(SnortConfig* sc, KTRIE_STRUCT* /* each and every prefix match at this root*/ if ( root and ts->agent ) { - cnt += KTrieBuildMatchStateNodeWithSnortConf(sc, root, ts); + cnt += KTrieBuildMatchStateNode(sc, root, ts); } } @@ -585,7 +583,7 @@ static inline int _KTrieCompile(KTRIE_STRUCT* ts) return 0; } -int KTrieCompileWithSnortConf(SnortConfig* sc, KTRIE_STRUCT* ts) +int KTrieCompile(SnortConfig* sc, KTRIE_STRUCT* ts) { int rval; @@ -593,136 +591,14 @@ int KTrieCompileWithSnortConf(SnortConfig* sc, KTRIE_STRUCT* ts) return rval; if ( ts->agent ) - KTrieBuildMatchStateTreesWithSnortConf(sc, ts); + KTrieBuildMatchStateTrees(sc, ts); return 0; } void sfksearch_print_qinfo(void) { -#ifdef SFKSEARCH_TRACK_Q - print_pat_stats("sfksearch", SFK_MAX_INQ); -#endif -} - -static inline void _init_queue(SFK_PMQ* b) -{ - b->inq=0; - b->inq_flush=0; -} - -/* uniquely insert into q */ -static inline int _add_queue(SFK_PMQ* b, void* p) -{ - int i; - -#ifdef SFKSEARCH_TRACK_Q - pmqs.tot_inq_inserts++; -#endif - - for (i=(int)(b->inq)-1; i>=0; i--) - if ( p == b->q[i] ) - return 0; - -#ifdef SFKSEARCH_TRACK_Q - pmqs.tot_inq_uinserts++; -#endif - - if ( b->inq < SFK_MAX_INQ ) - { - b->q[ b->inq++ ] = p; - } - - if ( b->inq == SFK_MAX_INQ ) - { -#ifdef SFKSEARCH_TRACK_Q - b->inq_flush++; -#endif - return 1; - } - return 0; -} - -static inline unsigned _process_queue(SFK_PMQ* q, MpseMatch match, void* context) -{ - KTRIEPATTERN* pk; - unsigned int i; - -#ifdef SFKSEARCH_TRACK_Q - if ( q->inq > pmqs.max_inq ) - pmqs.max_inq = q->inq; - pmqs.tot_inq_flush += q->inq_flush; -#endif - - for ( i=0; iinq; i++ ) - { - pk = (KTRIEPATTERN*)q->q[i]; - if (pk) - { - if (match(pk->user, pk->rule_option_tree, 0, context, pk->neg_list) > 0) - { - q->inq=0; - return 1; - } - } - } - q->inq=0; - return 0; -} - -static inline int KTriePrefixMatchQ( - KTRIE_STRUCT* kt, const uint8_t* T, int n, MpseMatch match, void* context) -{ - KTRIENODE* root; - //KTRIEPATTERN * pk; - //int index ; - - root = kt->root[ xlatcase[*T] ]; - - if ( !root ) - return 0; - - while ( n ) - { - if ( root->edge == xlatcase[*T] ) - { - T++; - n--; - - if ( root->pkeyword ) - { - if ( _add_queue(&kt->q, root->pkeyword) ) - { - if ( _process_queue(&kt->q,match,context) ) - { - return 1; - } - } - } - - if ( n && root->child ) - { - root = root->child; - } - else /* cannot continue -- match is over */ - { - break; - } - } - else - { - if ( root->sibling ) - { - root = root->sibling; - } - else /* cannot continue */ - { - break; - } - } - } - - return 0; + print_pat_stats("lowmem", 0); } /* @@ -765,7 +641,7 @@ static inline int KTriePrefixMatch( pk = root->pkeyword; if (pk) { - index = (int)(T - bT - pk->n ); + index = (int)(T - bT); nfound++; if (match (pk->user, pk->rule_option_tree, index, context, pk->neg_list) > 0) { @@ -798,52 +674,6 @@ static inline int KTriePrefixMatch( return nfound; } -int KTrieSearchQ( - KTRIE_STRUCT* ks, const uint8_t* T, int n, MpseMatch match, void* context) -{ - _init_queue(&ks->q); - while ( n > 0 ) - { - if ( KTriePrefixMatchQ(ks, T++, n--, match, context) ) - return 0; - } - _process_queue(&ks->q,match,context); - - return 0; -} - -static inline int KTrieSearchQBC( - KTRIE_STRUCT* ks, const uint8_t* T, int n, MpseMatch match, void* context) -{ - int tshift; - const uint8_t* Tend; - short* bcShift = (short*)ks->bcShift; - int bcSize = ks->bcSize; - - _init_queue(&ks->q); - - Tend = T + n - bcSize; - - bcSize--; - - for (; T <= Tend; n--, T++ ) - { - while ( (tshift = bcShift[ T[bcSize] ]) > 0 ) - { - T += tshift; - if ( T > Tend ) - return 0; - } - - if ( KTriePrefixMatchQ(ks, T, n, match, context) ) - return 0; - } - - _process_queue(&ks->q,match,context); - - return 0; -} - /* * */ @@ -970,7 +800,7 @@ int main(int argc, char** argv) printf("Patterns added \n"); - KTrieCompileWithSnortConf(nullptr, ts); + KTrieCompile(nullptr, ts); printf("Patterns compiled \n"); printf("--> %d characters, %d patterns, %d bytes allocated\n",ts->nchars,ts->npats,ts->memory); diff --git a/extra/src/search_engines/sfksearch.h b/extra/src/search_engines/sfksearch.h index 918dd5db3..0d7bd4784 100644 --- a/extra/src/search_engines/sfksearch.h +++ b/extra/src/search_engines/sfksearch.h @@ -26,11 +26,6 @@ #include #include "search_engines/search_common.h" -#define ALPHABET_SIZE 256 - -#define KTRIEMETHOD_STD 0 -#define KTRIEMETHOD_QUEUE 1 - struct KTRIEPATTERN { KTRIEPATTERN* next; /* global list of all patterns*/ @@ -60,15 +55,6 @@ struct KTRIENODE #define KTRIE_ROOT_NODES 256 -#define SFK_MAX_INQ 32 - -struct SFK_PMQ -{ - unsigned inq; - unsigned inq_flush; - void* q[SFK_MAX_INQ]; -}; - struct KTRIE_STRUCT { KTRIEPATTERN* patrn; /* List of patterns, built as they are added*/ @@ -85,8 +71,6 @@ struct KTRIE_STRUCT int bcSize; unsigned short bcShift[KTRIE_ROOT_NODES]; - - SFK_PMQ q; }; void KTrie_init_xlatcase(); @@ -97,10 +81,9 @@ int KTrieAddPattern( KTRIE_STRUCT*, const uint8_t* P, unsigned n, bool nocase, bool negative, void* id); -int KTrieCompileWithSnortConf(struct SnortConfig*, KTRIE_STRUCT*); +int KTrieCompile(struct SnortConfig*, KTRIE_STRUCT*); int KTrieSearch(KTRIE_STRUCT*, const uint8_t* T, int n, MpseMatch, void* context); -int KTrieSearchQ(KTRIE_STRUCT*, const uint8_t* T, int n, MpseMatch, void* context); unsigned int KTrieMemUsed(); void KTrieInitMemUsed(); diff --git a/extra/src/search_engines/trie_api.cc b/extra/src/search_engines/trie_api.cc index d79bdf215..23617e24c 100644 --- a/extra/src/search_engines/trie_api.cc +++ b/extra/src/search_engines/trie_api.cc @@ -21,12 +21,10 @@ #include "framework/mpse.h" extern const BaseApi* se_lowmem; -extern const BaseApi* se_lowmem_q; SO_PUBLIC const BaseApi* snort_plugins[] = { se_lowmem, - se_lowmem_q, nullptr }; diff --git a/src/catch/unit_test.cc b/src/catch/unit_test.cc index 53a92bc55..4e043a897 100644 --- a/src/catch/unit_test.cc +++ b/src/catch/unit_test.cc @@ -17,18 +17,21 @@ //-------------------------------------------------------------------------- // unit_test.h author Russ Combs +#include "unit_test.h" + #include #include #include #include -#include "unit_test.h" +#define CATCH_CONFIG_RUNNER +#include "catch.hpp" static bool s_catch = false; static std::vector test_tags; -void unit_test_catch_test_filter(const char* s) +void catch_set_filter(const char* s) { if ( s && strcmp(s, "all") ) test_tags.push_back( s ); @@ -41,12 +44,6 @@ bool catch_enabled() return s_catch; } -// check defines fail, so we must squash that because -// catch uses stream and that has a fail method -#undef fail -#define CATCH_CONFIG_RUNNER -#include "catch.hpp" - static bool run_catch() { Catch::Session session; diff --git a/src/catch/unit_test.h b/src/catch/unit_test.h index a0246166d..23a6ce6c3 100644 --- a/src/catch/unit_test.h +++ b/src/catch/unit_test.h @@ -22,7 +22,7 @@ // Unit test interface -void unit_test_catch_test_filter(const char* s); +void catch_set_filter(const char* s); bool catch_enabled(); diff --git a/src/detection/detection_options.cc b/src/detection/detection_options.cc index 49cff5ead..fa73bfa68 100644 --- a/src/detection/detection_options.cc +++ b/src/detection/detection_options.cc @@ -452,12 +452,6 @@ int detection_option_node_evaluate( // Add the match for this otn to the queue. { OptTreeNode* otn = (OptTreeNode*)node->option_data; - PatternMatchData* pmd = (PatternMatchData*)eval_data->pmd; - - int pattern_size = 0; - if ( pmd ) - pattern_size = pmd->pattern_size; - int16_t app_proto = p->get_application_protocol(); int check_ports = 1; @@ -508,8 +502,11 @@ int detection_option_node_evaluate( otn->state[get_instance_id()].matches++; if ( !eval_data->flowbit_noalert ) + { + PatternMatchData* pmd = (PatternMatchData*)eval_data->pmd; + int pattern_size = pmd ? pmd->pattern_size : 0; fpAddMatch((OTNX_MATCH_DATA*)pomd, pattern_size, otn); - + } result = rval = DETECTION_OPTION_MATCH; } } diff --git a/src/detection/fp_config.cc b/src/detection/fp_config.cc index f35c25cd4..becc917ca 100644 --- a/src/detection/fp_config.cc +++ b/src/detection/fp_config.cc @@ -42,7 +42,7 @@ FastPatternConfig::FastPatternConfig() max_queue_events = 5; bleedover_port_limit = 1024; - search_api = MpseManager::get_search_api("ac_bnfa_q"); + search_api = MpseManager::get_search_api("ac_bnfa"); assert(search_api); trim = MpseManager::search_engine_trim(search_api); } diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc index 1fa104f76..a4cfaab8b 100644 --- a/src/detection/fp_create.cc +++ b/src/detection/fp_create.cc @@ -69,11 +69,11 @@ static unsigned mpse_count = 0; static void fpDeletePMX(void* data); -static int fpGetFinalPattern(FastPatternConfig*, PatternMatchData* pmd, - char** ret_pattern, int* ret_bytes); +static int fpGetFinalPattern( + FastPatternConfig*, PatternMatchData*, char** ret_pattern, int* ret_bytes); -static void PrintFastPatternInfo(OptTreeNode* otn, PatternMatchData* pmd, - const char* pattern, int pattern_length); +static void PrintFastPatternInfo( + OptTreeNode*, PatternMatchData*, const char* pattern, int pattern_length); static const char* const pm_type_strings[PM_TYPE_MAX] = { @@ -384,28 +384,10 @@ static int FLP_Trim(char* p, int plen, char** buff) static bool pmd_can_be_fp(PatternMatchData* pmd, CursorActionType cat) { - if ( !pmd->pattern_buf || !pmd->pattern_size ) - return false; - if ( cat <= CAT_SET_OTHER ) return false; - if ( !pmd->negated ) - return true; - - /* Negative contents can only be considered if they are not relative - * and don't have any offset or depth. This is because the pattern - * matcher does not take these into consideration and may find the - * content in a non-relevant section of the payload and thus disable - * the rule when it shouldn't be. - * Also case sensitive patterns cannot be considered since patterns - * are inserted into the pattern matcher without case which may - * lead to false negatives */ - if ( pmd->relative || !pmd->no_case || - pmd->offset || pmd->depth ) - return false; - - return true; + return pmd->can_be_fp(); } struct FpFoo diff --git a/src/detection/fp_detect.cc b/src/detection/fp_detect.cc index 7add1e9c0..297788a15 100644 --- a/src/detection/fp_detect.cc +++ b/src/detection/fp_detect.cc @@ -40,6 +40,8 @@ #include "config.h" #endif +#include + #include "detect.h" #include "fp_config.h" #include "fp_create.h" @@ -76,6 +78,7 @@ #include "protocols/tcp.h" #include "protocols/udp.h" #include "protocols/icmp4.h" +#include "search_engines/pat_stats.h" THREAD_LOCAL ProfileStats rulePerfStats; THREAD_LOCAL ProfileStats ruleRTNEvalPerfStats; @@ -462,6 +465,11 @@ static int rule_tree_match( { PMX* pmx = (PMX*)user; OTNX_MATCH_DATA* pomd = (OTNX_MATCH_DATA*)context; + + unsigned sz = pmx->pmd->pattern_size; + assert(sz <= (unsigned)index and (unsigned)index <= pomd->size); + assert(!strncasecmp((char*)pmx->pmd->pattern_buf, (char*)pomd->data+index-sz, sz)); + detection_option_tree_root_t* root = (detection_option_tree_root_t*)tree; detection_option_eval_data_t eval_data; NCListNode* ncl; @@ -855,6 +863,101 @@ static inline int fpFinalSelectEvent(OTNX_MATCH_DATA* o, Packet* p) return 0; } +class MpseStash +{ +public: + static const unsigned max = 32; + + void init() + { count = flushed = 0; } + + bool push(void* user, void* tree, int index, void* list); + bool process(MpseMatch, void*); + +private: + unsigned count; + unsigned flushed; + + struct Node { + void* user; + void* tree; + void* list; + int index; + } queue[max]; +}; + +static THREAD_LOCAL MpseStash stash; + +// uniquely insert into q, should splay elements for performance +// return true if maxed out to trigger a flush +bool MpseStash::push(void* user, void* tree, int index, void* list) +{ + pmqs.tot_inq_inserts++; + + for ( int i = (int)(count) - 1; i >= 0; --i ) + { + if ( tree == queue[i].tree ) + return false; + } + + if ( count < max ) + { + Node& node = queue[count++]; + node.user = user; + node.tree = tree; + node.index = index; + node.list = list; + pmqs.tot_inq_uinserts++; + } + + if ( count == max ) + { + flushed++; + return true; + } + + return false; +} + +bool MpseStash::process(MpseMatch match, void* context) +{ + if ( count > pmqs.max_inq ) + pmqs.max_inq = count; + + pmqs.tot_inq_flush += flushed; + + for ( unsigned i = 0; i < count; ++i ) + { + Node& node = queue[i]; + + // process a pattern - case is handled by otn processing + int res = match(node.user, node.tree, node.index, context, node.list); + + if ( res > 0 ) + { + /* terminate matching */ + count = 0; + return true; + } + } + count = 0; + return false; +} + +// rule_tree_match() could be used instead to bypass the queuing +static int rule_tree_queue( + void* user, void* tree, int index, void* context, void* list) +{ + if ( stash.push(user, tree, index, list) ) + { + if ( stash.process(rule_tree_match, context) ) + { + return 1; + } + } + return 0; +} + #ifdef PPM_MGR #define CHECK_PPM() \ if (PPM_PACKET_ABORT_FLAG()) \ @@ -868,7 +971,10 @@ static inline int fpFinalSelectEvent(OTNX_MATCH_DATA* o, Packet* p) assert(so->get_pattern_count() > 0); \ int start_state = 0; \ cnt++; \ - so->search(buf, len, rule_tree_match, omd, &start_state); \ + omd->data = buf; omd->size = len; \ + stash.init(); \ + so->search(buf, len, rule_tree_queue, omd, &start_state); \ + stash.process(rule_tree_match, omd); \ CHECK_PPM() \ } diff --git a/src/detection/fp_detect.h b/src/detection/fp_detect.h index 783fb084d..a3688084e 100644 --- a/src/detection/fp_detect.h +++ b/src/detection/fp_detect.h @@ -90,6 +90,10 @@ struct OTNX_MATCH_DATA { PortGroup* pg; Packet* p; + + const uint8_t* data; + unsigned size; + int check_ports; MATCH_INFO* matchInfo; diff --git a/src/detection/pattern_match_data.h b/src/detection/pattern_match_data.h index 70ae0a0e8..9c52bd667 100644 --- a/src/detection/pattern_match_data.h +++ b/src/detection/pattern_match_data.h @@ -70,6 +70,30 @@ struct PatternMatchData bool unbounded() { return !depth; } + + bool can_be_fp() + { + if ( !pattern_buf || !pattern_size ) + return false; + + if ( !negated ) + return true; + + // Negative contents can only be considered if they are not + // relative and don't have any offset or depth. This is because + // the pattern matcher does not take these into consideration and + // may find the content in a non-relevant section of the payload + // and thus disable the rule when it shouldn't be. + + // Also case sensitive patterns cannot be considered since patterns + // are inserted into the pattern matcher without case which may + // lead to false negatives. + + if ( relative || !no_case || offset || depth ) + return false; + + return true; + } }; #endif diff --git a/src/ips_options/ips_hash.cc b/src/ips_options/ips_hash.cc index cf873faec..0945382a6 100644 --- a/src/ips_options/ips_hash.cc +++ b/src/ips_options/ips_hash.cc @@ -71,7 +71,7 @@ class HashOption : public IpsOption { public: HashOption(const char* s, HashPsIdx hpi, HashMatchData* c, HashFunc f, unsigned n) : - IpsOption(s, RULE_OPTION_TYPE_OTHER) + IpsOption(s) { config = c; hashf = f; size = n; idx = hpi; assert(n <= MAX_HASH_SIZE); } ~HashOption() { delete config; } diff --git a/src/ips_options/ips_ip_proto.cc b/src/ips_options/ips_ip_proto.cc index 056a88818..ff2b8e27e 100644 --- a/src/ips_options/ips_ip_proto.cc +++ b/src/ips_options/ips_ip_proto.cc @@ -58,7 +58,7 @@ class IpProtoOption : public IpsOption { public: IpProtoOption(const IpProtoData& c) : - IpsOption(s_name, RULE_OPTION_TYPE_OTHER) + IpsOption(s_name) { config = c; } uint32_t hash() const override; diff --git a/src/ips_options/ips_pcre.cc b/src/ips_options/ips_pcre.cc index a5e1d0c1a..b6b09f81e 100644 --- a/src/ips_options/ips_pcre.cc +++ b/src/ips_options/ips_pcre.cc @@ -462,6 +462,9 @@ public: uint32_t hash() const override; bool operator==(const IpsOption&) const override; + CursorActionType get_cursor_type() const override + { return CAT_ADJUST; } + bool is_relative() override { return (config->options & SNORT_PCRE_RELATIVE) != 0; } diff --git a/src/ips_options/ips_regex.cc b/src/ips_options/ips_regex.cc index c45b0117a..0de6aebba 100644 --- a/src/ips_options/ips_regex.cc +++ b/src/ips_options/ips_regex.cc @@ -86,6 +86,9 @@ public: uint32_t hash() const override; bool operator==(const IpsOption&) const override; + CursorActionType get_cursor_type() const override + { return CAT_ADJUST; } + bool is_relative() override { return config.relative; } diff --git a/src/main/modules.cc b/src/main/modules.cc index d2e5f39d5..71239d472 100644 --- a/src/main/modules.cc +++ b/src/main/modules.cc @@ -219,7 +219,7 @@ static const Parameter search_engine_params[] = { "inspect_stream_inserts", Parameter::PT_BOOL, nullptr, "false", "inspect reassembled payload - disabling is good for performance, bad for detection" }, - { "search_method", Parameter::PT_DYNAMIC, (void*)get_search_methods, "ac_bnfa_q", + { "search_method", Parameter::PT_DYNAMIC, (void*)get_search_methods, "ac_bnfa", "set fast pattern algorithm - choose available search engine" }, { "split_any_any", Parameter::PT_BOOL, nullptr, "false", diff --git a/src/main/snort_module.cc b/src/main/snort_module.cc index 28a18b15d..ce3608cdb 100644 --- a/src/main/snort_module.cc +++ b/src/main/snort_module.cc @@ -830,7 +830,7 @@ bool SnortModule::set(const char*, Value& v, SnortConfig* sc) #ifdef UNIT_TEST else if ( v.is("--catch-test") ) - unit_test_catch_test_filter(v.get_string()); + catch_set_filter(v.get_string()); #endif else if ( v.is("--version") ) help_version(sc, v.get_string()); diff --git a/src/managers/mpse_manager.cc b/src/managers/mpse_manager.cc index fb49d8a14..59632586b 100644 --- a/src/managers/mpse_manager.cc +++ b/src/managers/mpse_manager.cc @@ -155,7 +155,6 @@ bool MpseManager::search_engine_trim(const MpseApi* api) void MpseManager::print_qinfo() { sfksearch_print_qinfo(); - bnfa_print_qinfo(); acsmx2_print_qinfo(); } diff --git a/src/search_engines/CMakeLists.txt b/src/search_engines/CMakeLists.txt index 336ae5764..1d8195047 100644 --- a/src/search_engines/CMakeLists.txt +++ b/src/search_engines/CMakeLists.txt @@ -13,7 +13,6 @@ set (ACSMX_SOURCES set (ACSMX2_SOURCES ac_banded.cc ac_full.cc - ac_full_q.cc ac_sparse.cc ac_sparse_bands.cc acsmx2.cc @@ -22,7 +21,6 @@ set (ACSMX2_SOURCES set (BNFA_SOURCES ac_bnfa.cc - ac_bnfa_q.cc bnfa_search.cc bnfa_search.h ) diff --git a/src/search_engines/Makefile.am b/src/search_engines/Makefile.am index 6169360c4..4e8d0fd05 100644 --- a/src/search_engines/Makefile.am +++ b/src/search_engines/Makefile.am @@ -15,7 +15,6 @@ acsmx.h acsmx2_sources = \ ac_banded.cc \ ac_full.cc \ -ac_full_q.cc \ ac_sparse.cc \ ac_sparse_bands.cc \ acsmx2.cc \ @@ -23,7 +22,6 @@ acsmx2.h bnfa_sources = \ ac_bnfa.cc \ -ac_bnfa_q.cc \ bnfa_search.cc \ bnfa_search.h diff --git a/src/search_engines/ac_bnfa_q.cc b/src/search_engines/ac_bnfa_q.cc deleted file mode 100644 index 7b3eef87e..000000000 --- a/src/search_engines/ac_bnfa_q.cc +++ /dev/null @@ -1,154 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2015 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2002-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- -/* -* An abstracted interface to the Multi-Pattern Matching routines, -* thats why we're passing 'void *' objects around. -* -* Marc A Norton -* -* Updates: -* 3/06 - Added AC_BNFA search -*/ - -#include "bnfa_search.h" - -#include "main/snort_debug.h" -#include "main/snort_types.h" -#include "main/snort_config.h" -#include "utils/util.h" -#include "profiler/profiler.h" -#include "framework/mpse.h" - -//------------------------------------------------------------------------- -// "ac_bnfa_q" -//------------------------------------------------------------------------- - -class AcBnfaQMpse : public Mpse -{ -private: - bnfa_struct_t* obj; - -public: - AcBnfaQMpse(SnortConfig*, bool use_gc, const MpseAgent* agent) - : Mpse("ac_bnfa_q", use_gc) - { - obj = bnfaNew(agent); - - if (obj) - obj->bnfaMethod = 0; - } - - ~AcBnfaQMpse() - { - if (obj) - bnfaFree(obj); - } - - void set_opt(int flag) override - { - if (obj) - bnfaSetOpt(obj, flag); - } - - int add_pattern( - SnortConfig*, const uint8_t* P, unsigned m, - bool noCase, bool negative, void* user) override - { - return bnfaAddPattern(obj, P, m, noCase, negative, user); - } - - int prep_patterns(SnortConfig* sc) override - { - return bnfaCompile(sc, obj); - } - - int _search( - const uint8_t* T, int n, MpseMatch match, - void* context, int* current_state) override - { - /* return is actually the state */ - return _bnfa_search_csparse_nfa_q( - obj, T, n, match, context, 0 /* start-state */, current_state); - } - - int print_info() override - { - bnfaPrintInfo(obj); - return 0; - } - - int get_pattern_count() override - { - return bnfaPatternCount(obj); - } -}; - -//------------------------------------------------------------------------- -// api -//------------------------------------------------------------------------- - -static Mpse* bnfaq_ctor( - SnortConfig* sc, class Module*, bool use_gc, const MpseAgent* agent) -{ - return new AcBnfaQMpse(sc, use_gc, agent); -} - -static void bnfaq_dtor(Mpse* p) -{ - delete p; -} - -static void bnfaq_init() -{ - bnfa_init_xlatcase(); - bnfaInitSummary(); -} - -static void bnfaq_print() -{ - bnfaPrintSummary(); -} - -static const MpseApi bnfaq_api = -{ - { - PT_SEARCH_ENGINE, - sizeof(MpseApi), - SEAPI_VERSION, - 0, - API_RESERVED, - API_OPTIONS, - "ac_bnfa_q", - "Aho-Corasick Binary NFA (low memory, high performance) with queued events", - nullptr, - nullptr - }, - false, - nullptr, - nullptr, - nullptr, - nullptr, - bnfaq_ctor, - bnfaq_dtor, - bnfaq_init, - bnfaq_print, -}; - -const BaseApi* se_ac_bnfa_q = &bnfaq_api.base; - diff --git a/src/search_engines/ac_full_q.cc b/src/search_engines/ac_full_q.cc deleted file mode 100644 index 5c48c9ecd..000000000 --- a/src/search_engines/ac_full_q.cc +++ /dev/null @@ -1,149 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2015 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2013-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -#include "acsmx.h" -#include "acsmx2.h" - -#include "main/snort_debug.h" -#include "main/snort_types.h" -#include "main/snort_config.h" -#include "utils/util.h" -#include "profiler/profiler.h" -#include "framework/mpse.h" - -//------------------------------------------------------------------------- -// "ac_full_q" -//------------------------------------------------------------------------- - -class AcfQMpse : public Mpse -{ -private: - ACSM_STRUCT2* obj; - -public: - AcfQMpse(SnortConfig*, bool use_gc, const MpseAgent* agent) - : Mpse("ac_full_q", use_gc) - { - obj = acsmNew2(agent); - if (obj) acsmSelectFormat2(obj, ACF_FULLQ); - } - - ~AcfQMpse() - { - if (obj) - acsmFree2(obj); - } - - void set_opt(int flag) override - { - if (obj) - acsmCompressStates(obj, flag); - } - - int add_pattern( - SnortConfig*, const uint8_t* P, unsigned m, - bool noCase, bool negative, void* user) override - { - return acsmAddPattern2(obj, P, m, noCase, negative, user); - } - - int prep_patterns(SnortConfig* sc) override - { - return acsmCompile2(sc, obj); - } - - int _search( - const uint8_t* T, int n, MpseMatch match, - void* context, int* current_state) override - { - return acsmSearchSparseDFA_Full_q(obj, T, n, match, context, current_state); - } - - int search_all( - const uint8_t* T, int n, MpseMatch match, - void* context, int* current_state) override - { - return acsmSearchSparseDFA_Full_q_all(obj, T, n, match, context, current_state); - } - - int print_info() override - { - return acsmPrintDetailInfo2(obj); - } - - int get_pattern_count() override - { - return acsmPatternCount2(obj); - } -}; - -//------------------------------------------------------------------------- -// api -//------------------------------------------------------------------------- - -static Mpse* acfq_ctor( - SnortConfig* sc, class Module*, bool use_gc, const MpseAgent* agent) -{ - return new AcfQMpse(sc, use_gc, agent); -} - -static void acfq_dtor(Mpse* p) -{ - delete p; -} - -static void acfq_init() -{ - acsmx2_init_xlatcase(); - acsm_init_summary(); -} - -static void acfq_print() -{ - acsmPrintSummaryInfo2(); -} - -static const MpseApi acfq_api = -{ - { - PT_SEARCH_ENGINE, - sizeof(MpseApi), - SEAPI_VERSION, - 0, - API_RESERVED, - API_OPTIONS, - "ac_full_q", - "Aho-Corasick Full (high memory, best performance) with queued events," - " implements search_all()", - nullptr, - nullptr - }, - false, - nullptr, - nullptr, - nullptr, - nullptr, - acfq_ctor, - acfq_dtor, - acfq_init, - acfq_print, -}; - -const BaseApi* se_ac_full_q = &acfq_api.base; - diff --git a/src/search_engines/acsmx.cc b/src/search_engines/acsmx.cc index 906bef231..6791caaaa 100644 --- a/src/search_engines/acsmx.cc +++ b/src/search_engines/acsmx.cc @@ -470,7 +470,7 @@ int acsmSearch( if ( StateTable[state].MatchList != NULL ) { mlist = StateTable[state].MatchList; - index = T - mlist->n + 1 - Tc; + index = T + 1 - Tc; nfound++; if (match(mlist->udata->id, mlist->rule_option_tree, index, context, mlist->neg_list) > 0) diff --git a/src/search_engines/acsmx2.cc b/src/search_engines/acsmx2.cc index 96a2ad76a..17b226737 100644 --- a/src/search_engines/acsmx2.cc +++ b/src/search_engines/acsmx2.cc @@ -109,18 +109,6 @@ ** 8/28/06 ** man - Sparse and SparseBands - fixed off by one in calculating matching index ** SparseBands changed ps increment to 2+n to increment between bands. -** -** 01/2008 -** man - added 2 phase pattern matcher using a pattern match queue. -** -** Matching states are queued, duplicate matches are dropped, -** and after the complete buffer scan the queued matches are -** processed. This improves cacheing performance, and reduces -** duplicate rule processing. The queue is limited in size and -** is flushed if it becomes full during the scan. This allows -** simple insertions. Tracking queue ops is optional, as this can -** impose a modest performance hit of a few percent. -** */ #include "acsmx2.h" @@ -2101,7 +2089,7 @@ int acsmSearchSparseDFA( mlist = MatchList[state]; if (mlist) { - index = T - mlist->n - Tc + 1; + index = T - Tc + 1; nfound++; if (match (mlist->udata, mlist->rule_option_tree, index, context, mlist->neg_list) > 0) @@ -2121,263 +2109,10 @@ int acsmSearchSparseDFA( void acsmx2_print_qinfo(void) { #ifdef ACSMX2_TRACK_Q - print_pat_stats("acsmx2", AC_MAX_INQ); + print_pat_stats("acsmx2", 0); #endif } -static inline void _init_queue(PMQ* b) -{ - b->inq=0; - b->inq_flush=0; -} - -/* uniquely insert into q, should splay elements for performance */ -static inline int _add_queue(PMQ* b, void* p) -{ - int i; - -#ifdef ACSMX2_TRACK_Q - pmqs.tot_inq_inserts++; -#endif - - for (i=(int)(b->inq)-1; i>=0; i--) - if ( p == b->q[i] ) - return 0; - -#ifdef ACSMX2_TRACK_Q - pmqs.tot_inq_uinserts++; -#endif - - if ( b->inq < AC_MAX_INQ ) - { - b->q[ b->inq++ ] = p; - } - - if ( b->inq == AC_MAX_INQ ) - { -#ifdef ACSMX2_TRACK_Q - b->inq_flush++; -#endif - return 1; - } - return 0; -} - -static inline unsigned _process_queue(PMQ* q, MpseMatch match, void* context) -{ - ACSM_PATTERN2* mlist; - unsigned int i; - -#ifdef ACSMX2_TRACK_Q - if ( q->inq > pmqs.max_inq ) - pmqs.max_inq = q->inq; - pmqs.tot_inq_flush += q->inq_flush; -#endif - - for ( i=0; iinq; i++ ) - { - mlist = (ACSM_PATTERN2*)q->q[i]; - if (mlist) - { - if (match (mlist->udata, mlist->rule_option_tree, 0, context, mlist->neg_list) > 0) - { - q->inq = 0; - return 1; - } - } - } - q->inq=0; - return 0; -} - -/* - * Matching states are queued, duplicate matches are dropped, - * and after the complete buffer scan, the queued matches are - * processed. This improves cacheing performance, and reduces - * duplicate rule processing. The queue is limited in size and - * is flushed if it becomes full during the scan. This allows - * simple insertions. Tracking queue ops is optional, as this can - * impose a modest performance hit of a few percent. - */ -#define AC_SEARCH_Q \ - for (; T < Tend; T++) \ - { \ - ps = NextState[state]; \ - sindex = xlatcase[T[0]]; \ - if (ps[1]) \ - { \ - if (MatchList[state]) \ - { \ - if (_add_queue(&acsm->q,MatchList[state])) \ - { \ - if (_process_queue(&acsm->q, match, context)) \ - { \ - *current_state = state; \ - return 1; \ - } \ - } \ - } \ - } \ - state = ps[2 + sindex]; \ - } - -int acsmSearchSparseDFA_Full_q( - ACSM_STRUCT2* acsm, const uint8_t* T, int n, MpseMatch match, - void* context, int* current_state) -{ - const uint8_t* Tend; - int sindex; - acstate_t state; - ACSM_PATTERN2** MatchList = acsm->acsmMatchList; - - Tend = T + n; - - if (current_state == NULL) - return 0; - - _init_queue(&acsm->q); - - state = *current_state; - - switch (acsm->sizeofstate) - { - case 1: - { - uint8_t* ps; - uint8_t** NextState = (uint8_t**)acsm->acsmNextState; - AC_SEARCH_Q; - } - break; - case 2: - { - uint16_t* ps; - uint16_t** NextState = (uint16_t**)acsm->acsmNextState; - AC_SEARCH_Q; - } - break; - default: - { - acstate_t* ps; - acstate_t** NextState = acsm->acsmNextState; - AC_SEARCH_Q; - } - break; - } - - *current_state = state; - - if (MatchList[state]) - _add_queue(&acsm->q,MatchList[state]); - - _process_queue(&acsm->q, match, context); - - return 0; -} - -/* - * Matching states are queued, duplicate matches are dropped, - * and after the complete buffer scan, the queued matches are - * processed. This improves cacheing performance, and reduces - * duplicate rule processing. The queue is limited in size and - * is flushed if it becomes full during the scan. This allows - * simple insertions. Tracking queue ops is optional, as this can - * impose a modest performance hit of a few percent. - */ -#define AC_SEARCH_Q_ALL \ - for (; T < Tend; T++) \ - { \ - ps = NextState[state]; \ - sindex = xlatcase[T[0]]; \ - if (ps[1]) \ - { \ - for ( mlist = MatchList[state]; \ - mlist!= NULL; \ - mlist = mlist->next ) \ - { \ - if ( mlist->nocase || (memcmp (mlist->casepatrn, T - mlist->n, mlist->n) == 0)) \ - { \ - if (_add_queue(&acsm->q,mlist)) \ - { \ - if (_process_queue(&acsm->q, match, context)) \ - { \ - *current_state = state; \ - return 1; \ - } \ - } \ - } \ - } \ - } \ - state = ps[2 + sindex]; \ - } - -int acsmSearchSparseDFA_Full_q_all( - ACSM_STRUCT2* acsm, const uint8_t* T, int n, MpseMatch match, - void* context, int* current_state) -{ - const uint8_t* Tend; - int sindex; - acstate_t state; - ACSM_PATTERN2** MatchList = acsm->acsmMatchList; - ACSM_PATTERN2* mlist; - - Tend = T + n; - - if (current_state == NULL) - return 0; - - _init_queue(&acsm->q); - - state = *current_state; - - switch (acsm->sizeofstate) - { - case 1: - { - uint8_t* ps; - uint8_t** NextState = (uint8_t**)acsm->acsmNextState; - AC_SEARCH_Q_ALL; - } - break; - case 2: - { - uint16_t* ps; - uint16_t** NextState = (uint16_t**)acsm->acsmNextState; - AC_SEARCH_Q_ALL; - } - break; - default: - { - acstate_t* ps; - acstate_t** NextState = acsm->acsmNextState; - AC_SEARCH_Q_ALL; - } - break; - } - - *current_state = state; - - for ( mlist = MatchList[state]; - mlist!= NULL; - mlist = mlist->next ) - { - if ( mlist->nocase || (memcmp (mlist->casepatrn, T - mlist->n, mlist->n) == 0)) - { - if (_add_queue(&acsm->q,mlist)) - { - if (_process_queue(&acsm->q, match, context)) - { - *current_state = state; - return 1; - } - } - } - } - - _process_queue(&acsm->q, match, context); - - return 0; -} - /* * Full format DFA search * Do not change anything here without testing, caching and prefetching @@ -2398,7 +2133,7 @@ int acsmSearchSparseDFA_Full_q_all( mlist = MatchList[state]; \ if (mlist) \ { \ - index = T - mlist->n - Tx; \ + index = T - Tx; \ nfound++; \ if (match (mlist->udata, mlist->rule_option_tree, index, context, \ mlist->neg_list) > 0) \ @@ -2462,7 +2197,7 @@ int acsmSearchSparseDFA_Full( mlist = MatchList[state]; if (mlist) { - index = T - mlist->n - Tx; + index = T - Tx; nfound++; if (match(mlist->udata, mlist->rule_option_tree, index, context, mlist->neg_list) > 0) { @@ -2496,7 +2231,7 @@ int acsmSearchSparseDFA_Full( mlist!= NULL; \ mlist = mlist->next ) \ { \ - index = T - mlist->n - Tx; \ + index = T - Tx; \ if ( mlist->nocase || (memcmp (mlist->casepatrn, Tx + index, mlist->n) == 0)) \ { \ nfound++; \ @@ -2563,7 +2298,7 @@ int acsmSearchSparseDFA_Full_All( mlist!= NULL; mlist = mlist->next ) { - index = T - mlist->n - Tx; + index = T - Tx; if ( mlist->nocase || (memcmp (mlist->casepatrn, Tx + index, mlist->n) == 0)) { @@ -2627,7 +2362,7 @@ int acsmSearchSparseDFA_Banded( mlist = MatchList[state]; if (mlist) { - index = T - mlist->n - Tx; + index = T - Tx; nfound++; if (match (mlist->udata, mlist->rule_option_tree, index, context, mlist->neg_list) > 0) @@ -2650,7 +2385,7 @@ int acsmSearchSparseDFA_Banded( mlist = MatchList[state]; if (mlist) { - index = T - mlist->n - Tx; + index = T - Tx; nfound++; if (match (mlist->udata, mlist->rule_option_tree, index, context, mlist->neg_list) > 0) { @@ -2706,7 +2441,7 @@ int acsmSearchSparseNFA( mlist = MatchList[state]; if (mlist) { - index = T - mlist->n - Tx; + index = T - Tx; nfound++; if (match (mlist->udata, mlist->rule_option_tree, index, context, mlist->neg_list) > 0) { diff --git a/src/search_engines/acsmx2.h b/src/search_engines/acsmx2.h index 703807117..9c0b503f6 100644 --- a/src/search_engines/acsmx2.h +++ b/src/search_engines/acsmx2.h @@ -107,14 +107,6 @@ enum FSA_DFA }; -#define AC_MAX_INQ 32 -struct PMQ -{ - unsigned inq; - unsigned inq_flush; - void* q[AC_MAX_INQ]; -}; - /* * Aho-Corasick State Machine Struct - one per group of pattterns */ @@ -145,8 +137,6 @@ struct ACSM_STRUCT2 int sizeofstate; int compress_states; - - PMQ q; }; /* @@ -165,9 +155,6 @@ int acsmCompile2(struct SnortConfig*, ACSM_STRUCT2*); int acsmSearchSparseDFA_Full( ACSM_STRUCT2*, const uint8_t* T, int n, MpseMatch, void* context, int* current_state); -int acsmSearchSparseDFA_Full_q( - ACSM_STRUCT2*, const uint8_t* T, int n, MpseMatch, void* context, int* current_state); - int acsmSearchSparseDFA_Banded( ACSM_STRUCT2*, const uint8_t* T, int n, MpseMatch, void* context, int* current_state); @@ -180,9 +167,6 @@ int acsmSearchSparseNFA( int acsmSearchSparseDFA_Full_All( ACSM_STRUCT2*, const uint8_t* Tx, int n, MpseMatch, void* context, int* current_state); -int acsmSearchSparseDFA_Full_q_all( - ACSM_STRUCT2*, const uint8_t* T, int n, MpseMatch, void* context, int* current_state); - void acsmFree2(ACSM_STRUCT2* acsm); int acsmPatternCount2(ACSM_STRUCT2* acsm); void acsmCompressStates(ACSM_STRUCT2*, int); diff --git a/src/search_engines/bnfa_search.cc b/src/search_engines/bnfa_search.cc index 2d9e20b04..d74062cc3 100644 --- a/src/search_engines/bnfa_search.cc +++ b/src/search_engines/bnfa_search.cc @@ -153,15 +153,8 @@ #include -#define BNFA_TRACK_Q - -#ifdef BNFA_TRACK_Q -# include "main/snort_config.h" -#endif - #include "search_common.h" #include "pat_stats.h" - #include "main/snort_types.h" #include "main/snort_debug.h" #include "utils/stats.h" @@ -1401,13 +1394,6 @@ static inline int _bnfaCompile(bnfa_struct_t* bnfa) BNFA_FREE(tmpMatchList,sizeof(void*) * bnfa->bnfaMaxStates,bnfa->matchlist_memory); -#ifdef MATCH_LIST_CNT - bnfa->bnfaMatchListCnt=(unsigned*)calloc(sizeof(unsigned) * bnfa->bnfaNumStates); - if (!bnfa->bnfaMatchListCnt) - { - return -1; - } -#endif /* Alloc a failure state table - only need NumStates */ bnfa->bnfaFailState =(bnfa_state_t*)BNFA_MALLOC(sizeof(bnfa_state_t) * bnfa->bnfaNumStates, bnfa->failstate_memory); @@ -1550,10 +1536,7 @@ static inline unsigned _bnfa_search_full_nfa( continue; } patrn = (bnfa_pattern_t*)mlist->data; - if ( ( T - Tx) < patrn->n ) - index = 0; - else - index = T - Tx - patrn->n + 1; + index = T - Tx + 1; nfound++; /* Don't do anything specific for case sensitive patterns and not, * since that will be covered by the rule tree itself. Each tree @@ -1635,10 +1618,7 @@ static inline unsigned _bnfa_search_full_nfa_case( continue; } patrn = (bnfa_pattern_t*)mlist->data; - if ( ( T - Tx) < patrn->n ) - index = 0; - else - index = T - Tx - patrn->n + 1; + index = T - Tx + 1; nfound++; /* Don't do anything specific for case (in)sensitive patterns * since that will be covered by the rule tree itself. Each @@ -1720,10 +1700,7 @@ static inline unsigned _bnfa_search_full_nfa_nocase( continue; } patrn = (bnfa_pattern_t*)mlist->data; - if ( ( T - Tx) < patrn->n ) - index = 0; - else - index = T - Tx - patrn->n + 1; + index = T - Tx + 1; /* Don't do anything specific for case sensitive patterns and not, * since that will be covered by the rule tree itself. Each tree * might have both case sensitive & case insensitive patterns. @@ -1791,74 +1768,6 @@ static inline int _bnfa_binearch(bnfa_state_t* a, int a_len, int val) return -1; } -#ifdef BNFA_MAIN -/* -* Sparse format for state table using single array storage -* -* word 1: state -* word 2: control-word = cb<<24| fs -* cb : control-byte -* : mb | fb | nt -* mb : bit 8 set if match state, zero otherwise -* fb : bit 7 set if using full format, zero otherwise -* nt : number of transitions 0..63 (more than 63 requires full format) -* fs: failure-transition-state -* word 3+: byte-value(0-255) << 24 | transition-state -*/ -static inline unsigned _bnfa_get_next_state_csparse_nfa_qx( - bnfa_state_t* pcx, unsigned sindex, unsigned input) -{ - int k; - int nc; - int index; - bnfa_state_t* pcs; - - for (;; ) - { - pcs = pcx + sindex + 1; /* skip state-id == 1st word */ - - if ( pcs[0] & BNFA_SPARSE_FULL_BIT ) - { - if ( sindex == 0 ) - { - return pcs[1+input] & BNFA_SPARSE_MAX_STATE; - } - else - { - if ( pcs[1+input] & BNFA_SPARSE_MAX_STATE ) - return pcs[1+input] & BNFA_SPARSE_MAX_STATE; - } - } - else - { - nc = (pcs[0]>>BNFA_SPARSE_COUNT_SHIFT) & BNFA_SPARSE_MAX_ROW_TRANSITIONS; - if ( nc > BNFA_SPARSE_LINEAR_SEARCH_LIMIT ) - { - /* binary search... */ - index = _bnfa_binearch(pcs+1, nc, input); - if ( index >= 0 ) - { - return pcs[index+1] & BNFA_SPARSE_MAX_STATE; - } - } - else - { - /* linear search... */ - for ( k=0; k>BNFA_SPARSE_VALUE_SHIFT) == input ) - { - return pcs[k+1] & BNFA_SPARSE_MAX_STATE; - } - } - } - } - - return 0; /* no transition keyword match failed */ - } -} -#endif - /* * Sparse format for state table using single array storage * @@ -1927,191 +1836,14 @@ static inline unsigned _bnfa_get_next_state_csparse_nfa( } /* - * Per Pattern case search, case is on per pattern basis - * standard snort search - * note: index is not used by snort, so it's commented - * TRACK_Q can impose a modest couple % performance difference in the - * pattern matching rate. + * Per Pattern case search, case is on per pattern basis standard snort + * search note: index is not used by snort, so it's commented */ -/* Queue whole pattern groups at end states in AC */ -void bnfa_print_qinfo(void) -{ -#ifdef BNFA_TRACK_Q - print_pat_stats("bnfa", MAX_INQ); -#endif -} - -static inline void _init_queue(bnfa_struct_t* b) -{ - b->inq=0; - b->inq_flush=0; -} - -/* uniquely insert into q, should splay elements for performance */ -static inline int _add_queue(bnfa_struct_t* b, bnfa_match_node_t* p) -{ - int i; - -#ifdef BNFA_TRACK_Q - pmqs.tot_inq_inserts++; -#endif - - for (i=(int)(b->inq)-1; i>=0; i--) - if ( p == b->q[i] ) - return 0; - -#ifdef BNFA_TRACK_Q - pmqs.tot_inq_uinserts++; -#endif - - if ( b->inq < MAX_INQ ) - { - b->q[ b->inq++ ] = p; - } - - if ( b->inq == MAX_INQ ) - { -#ifdef BNFA_TRACK_Q - b->inq_flush++; -#endif - return 1; - } - - return 0; -} - -static inline unsigned _process_queue( - bnfa_struct_t* bnfa, MpseMatch match, void* context) -{ - bnfa_match_node_t* mlist; - bnfa_pattern_t* patrn; - int res; - unsigned int i; - -#ifdef BNFA_TRACK_Q - if ( bnfa->inq > pmqs.max_inq ) - pmqs.max_inq = bnfa->inq; - pmqs.tot_inq_flush += bnfa->inq_flush; -#endif - - for ( i=0; iinq; i++ ) - { - mlist = (bnfa_match_node_t*)bnfa->q[i]; - if (mlist) - { - patrn = (bnfa_pattern_t*)mlist->data; - /*process a pattern - case is handled by otn processing */ - res = match(patrn->userdata, mlist->rule_option_tree, 0, context, - mlist->neg_list); - if ( res > 0 ) - { - /* terminate matching */ - bnfa->inq=0; /* clear the q */ - return 1; - } - } - } - bnfa->inq=0; /* clear the q */ - return 0; -} - -#ifdef BNFA_MAIN -static inline unsigned _bnfa_search_csparse_nfa_qx( - bnfa_struct_t* bnfa, uint8_t* T, int n, MpseMatch match, void* context) -{ - bnfa_match_node_t* mlist; - uint8_t* Tend; - bnfa_match_node_t** MatchList = bnfa->bnfaMatchList; - bnfa_state_t* transList = bnfa->bnfaTransList; - unsigned sindex=0; - - Tend = T + n; - - for (; TbnfaMatchList; - bnfa_state_t* transList = bnfa->bnfaTransList; - unsigned last_sindex; - - Tend = T + n; - - _init_queue(bnfa); - - for (; TbnfaMatchTestCnt; -#endif + T = Tx; Tend = T + n; @@ -2151,21 +1881,13 @@ unsigned _bnfa_search_csparse_nfa( last_match_saved = last_match; last_match = sindex; -#ifdef MATCH_LIST_CNT - if ( MatchList[ transList[sindex] ] ) - MatchTestCnt[ transList[index] ]++; -#endif - { mlist = MatchList[ transList[sindex] ]; if ( !mlist ) return nfound; patrn = (bnfa_pattern_t*)mlist->data; - if ( ( T - Tx) < patrn->n ) - index = 0; - else - index = T - Tx - patrn->n + 1; + index = T - Tx + 1; nfound++; /* Don't do anything specific for case sensitive patterns and not, * since that will be covered by the rule tree itself. Each tree @@ -2231,10 +1953,7 @@ static inline unsigned _bnfa_search_csparse_nfa_case( { mlist = MatchList[ transList[sindex] ]; patrn = (bnfa_pattern_t*)mlist->data; - if ( ( T - Tx) < patrn->n ) - index = 0; - else - index = T - Tx - patrn->n + 1; + index = T - Tx + 1; nfound++; /* Don't do anything specific for case sensitive patterns and not, * since that will be covered by the rule tree itself. Each tree @@ -2302,10 +2021,7 @@ static inline unsigned _bnfa_search_csparse_nfa_nocase( { mlist = MatchList[ transList[sindex] ]; patrn = (bnfa_pattern_t*)mlist->data; - if ( ( T - Tx) < patrn->n ) - index = 0; - else - index = T - Tx - patrn->n + 1; + index = T - Tx + 1; nfound++; /* Don't do anything specific for case sensitive patterns and not, * since that will be covered by the rule tree itself. Each tree @@ -2412,35 +2128,6 @@ void bnfaAccumInfo(bnfa_struct_t* p) px->failstate_memory += p->failstate_memory; } -#ifdef MATCH_LIST_CNT -void bnfaPrintMatchListCnt(bnfa_struct_t* p) -{ - unsigned* cnt = p->bnfaMatchListCnt; - int i; - bnfa_match_node_t* mn; - bnfa_pattern_t* patrn; - - printf("[ MatchListCnt for ac-bnfa state machine\n ]"); - - for (i=0; ibnfaNumStates; i++) - { - if ( cnt[i] ) - { - printf("state[%d] cnt=%d",i,cnt[i]); - mn = bnfa->MatchList[i]; - if ( mn ) - { - patrn =(bnfa_pattern_t*)mn->data; - //xprintOTNSidGid(cnt,patrn->userdata); - } - printf("\n"); - fflush(stdout); - } - } -} - -#endif - #ifdef BNFA_MAIN #include /* @@ -2462,22 +2149,6 @@ void bnfaPrintMatchListCnt(bnfa_struct_t* p) * The state or sindex of the state machine. This can than be passed back * in on the next search, if desired. */ -static unsigned bnfaSearchX( - bnfa_struct_t* bnfa, uint8_t* T, int n, MpseMatch match, - void* context, unsigned, int*) -{ - int ret; - - _init_queue(bnfa); - while ( n > 0) - { - ret = _bnfa_search_csparse_nfa_qx(bnfa, T++, n--, match, context); - - if ( ret ) - return 0; - } - return _process_queue(bnfa, match, context); -} static unsigned bnfaSearch( bnfa_struct_t* bnfa, uint8_t* Tx, int n, MpseMatch match, diff --git a/src/search_engines/bnfa_search.h b/src/search_engines/bnfa_search.h index 3623464a0..fb936f594 100644 --- a/src/search_engines/bnfa_search.h +++ b/src/search_engines/bnfa_search.h @@ -149,11 +149,6 @@ struct bnfa_struct_t int nextstate_memory; int failstate_memory; int matchlist_memory; - -#define MAX_INQ 32 - unsigned inq; - unsigned inq_flush; - void* q[MAX_INQ]; }; /* @@ -177,10 +172,6 @@ unsigned _bnfa_search_csparse_nfa( bnfa_struct_t * pstruct, const uint8_t* t, int tlen, MpseMatch, void* context, unsigned sindex, int* current_state); -unsigned _bnfa_search_csparse_nfa_q( - bnfa_struct_t * pstruct, const uint8_t* t, int tlen, MpseMatch, - void* context, unsigned sindex, int* current_state); - int bnfaPatternCount(bnfa_struct_t* p); void bnfaPrint(bnfa_struct_t* pstruct); /* prints the nfa states-verbose!! */ diff --git a/src/search_engines/dev_notes.txt b/src/search_engines/dev_notes.txt index d55df4896..855346202 100644 --- a/src/search_engines/dev_notes.txt +++ b/src/search_engines/dev_notes.txt @@ -3,12 +3,15 @@ Builtin fast pattern matching algorithms are implemented here. * MPSE = multi-pattern search engine * DFA = deterministic finite automaton * NFA = non-DFA +* HFA = hybrid FA -This code has has evolved through 3 major versions: +This code has has evolved through 4 major versions: 1. acsmx.cc: ac_std -2. acsmx2.cc: ac_full, ac_full_q, ac_sparse, ac_banded, ac_sparse_bands -3. bnfa_search.cc: ac_bnfa, ac_bnfa_q +2. acsmx2.cc: ac_full, ac_sparse, ac_banded, ac_sparse_bands +3. bnfa_search.cc: ac_bnfa + intel_cpm.cc: intel_cpm was added later based on ac_bnfa +4. hyperscan.cc: support of regex fast patterns Check the comments at the start of the above files for details on the implementation. @@ -25,10 +28,27 @@ DFA flavors try to reduce memory for transition storage by various schemes: transitions are not stored * sparse bands - a list of bands -The *_q flavors use a match queue to defer rule tree evaluation until after -the full buffer is searched in order to keep the cache warm. This aspect -should be orthogonal such that any method can be used with or w/o a match -queue. +Version 4 entails a number of refactoring changes to support regex fast +patterns using hyperscan, an HFA. A key change is to return the offset of +the end of match the way hyperscan does to support relative matches to fast +pattern only contents and regexes. + +Version 4 also includes refactoring the match queues. The *_q flavors were +deleted as this aspect is orthogonal to the algorithm. Instead a match +queue is always used to defer rule tree evaluation until after the full +buffer is searched in order to keep the cache warm. This is a development +decision based on overall performance. + +Note that hyperscan essentially results in single branch detection option +trees because from a client view each match state is unique - one per rule. +This is a potential negative impact on performance but does not yet seem +significant. Furthermore, regex based fast patterns may obviate the need +for the tree. However, the tree remains as it is essential for other +algorithms. + +intel_cpm will likely be deleted as it requires a license and does not +perform as well as hyperscan. It remains pending further performance +evaluations. SearchTool makes it easy to use ac_bnfa. This is used by http, pop, imap, and smtp. diff --git a/src/search_engines/hyperscan.cc b/src/search_engines/hyperscan.cc index c4bcf1ec9..00b2da373 100644 --- a/src/search_engines/hyperscan.cc +++ b/src/search_engines/hyperscan.cc @@ -220,10 +220,6 @@ int HyperscanMpse::prep_patterns(SnortConfig* sc) return 0; } -// FIXIT-P first cut does not queue matches which will likley be required -// to improve cache performance. for now each match results in an -// immediate callback. - int HyperscanMpse::match(unsigned id, unsigned long long to) { assert(id < pvector.size()); diff --git a/src/search_engines/intel_soft_cpm.cc b/src/search_engines/intel_soft_cpm.cc index 5c0f7410a..81753892e 100644 --- a/src/search_engines/intel_soft_cpm.cc +++ b/src/search_engines/intel_soft_cpm.cc @@ -140,6 +140,7 @@ static inline unsigned int IntelPmProcessQueue( IntelPmMatchState* mstate = (IntelPmMatchState*)q->q[i]; if (mstate != NULL) { + // FIXIT-L delete soft-cpm or delete queue and get index if (match(mstate->user_data, mstate->rule_option_tree, 0, context, mstate->neg_list) > 0) { diff --git a/src/search_engines/search_engines.cc b/src/search_engines/search_engines.cc index cc77cfb96..612d79fd3 100644 --- a/src/search_engines/search_engines.cc +++ b/src/search_engines/search_engines.cc @@ -25,7 +25,6 @@ struct BaseApi; extern const BaseApi* se_ac_bnfa; -extern const BaseApi* se_ac_bnfa_q; #ifdef INTEL_SOFT_CPM extern const BaseApi* se_intel_cpm; @@ -38,7 +37,6 @@ extern const BaseApi* se_hyperscan; #ifdef STATIC_SEARCH_ENGINES extern const BaseApi* se_ac_banded; extern const BaseApi* se_ac_full; -extern const BaseApi* se_ac_full_q; extern const BaseApi* se_ac_sparse; extern const BaseApi* se_ac_sparse_bands; extern const BaseApi* se_ac_std; @@ -47,7 +45,6 @@ extern const BaseApi* se_ac_std; const BaseApi* search_engines[] = { se_ac_bnfa, - se_ac_bnfa_q, #ifdef INTEL_SOFT_CPM se_intel_cpm, @@ -60,7 +57,6 @@ const BaseApi* search_engines[] = #ifdef STATIC_SEARCH_ENGINES se_ac_banded, se_ac_full, - se_ac_full_q, se_ac_sparse, se_ac_sparse_bands, se_ac_std,