add_example_library(lowmem search_engines
lowmem.cc
- lowmem_q.cc
pat_stats.cc
sfksearch.cc
sfksearch.h
liblowmem_la_SOURCES = \
lowmem.cc \
-lowmem_q.cc \
sfksearch.cc \
sfksearch.h \
trie_api.cc
int prep_patterns(SnortConfig* sc) override
{
- return KTrieCompileWithSnortConf(sc, obj);
+ return KTrieCompile(sc, obj);
}
int _search(
+++ /dev/null
-//--------------------------------------------------------------------------
-// Copyright (C) 2014-2015 Cisco and/or its affiliates. All rights reserved.
-// Copyright (C) 2002-2013 Sourcefire, Inc.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License Version 2 as published
-// by the Free Software Foundation. You may not use, modify or distribute
-// this program under any other version of the GNU General Public License.
-//
-// This program is distributed in the hope that it will be useful, but
-// WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, write to the Free Software Foundation, Inc.,
-// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-//--------------------------------------------------------------------------
-/*
-* An abstracted interface to the Multi-Pattern Matching routines,
-* thats why we're passing 'void *' objects around.
-*
-* Marc A Norton <mnorton@sourcefire.com>
-*
-* Updates:
-* 3/06 - Added AC_BNFA search
-*/
-
-// lowmem_q.cc author Russ Combs <rucombs@cisco.com>
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <string>
-
-#include "sfksearch.h"
-#include "main/snort_debug.h"
-#include "main/snort_types.h"
-#include "framework/mpse.h"
-#include "framework/module.h"
-#include "log/messages.h"
-#include "profiler/profiler.h"
-
-using namespace std;
-
-static string s_var;
-
-static const char* s_name = "lowmem_q";
-static const char* s_help = "MPSE that minimizes memory used";
-
-//-------------------------------------------------------------------------
-// module stuff
-//-------------------------------------------------------------------------
-
-static const Parameter s_params[] =
-{
- { "var", Parameter::PT_STRING, nullptr, nullptr,
- "additional print text" },
-
- { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
-};
-
-class LowmemQModule : public Module
-{
-public:
- LowmemQModule() : Module(s_name, s_help, s_params) { }
-
- bool set(const char*, Value&, SnortConfig*) override;
- bool begin(const char*, int, SnortConfig*) override;
-
-public:
- string var;
-};
-
-bool LowmemQModule::set(const char*, Value& v, SnortConfig*)
-{
- if ( v.is("var") )
- var = v.get_string();
-
- else
- return false;
-
- return true;
-}
-
-bool LowmemQModule::begin(const char*, int, SnortConfig*)
-{
- var.clear();
- return true;
-}
-
-//-------------------------------------------------------------------------
-// "lowmem_q"
-//-------------------------------------------------------------------------
-
-class LowmemQMpse : public Mpse
-{
-private:
- KTRIE_STRUCT* obj;
-
-public:
- LowmemQMpse(bool use_gc, const MpseAgent* agent) : Mpse(s_name, use_gc)
- {
- obj = KTrieNew(1, agent);
- }
-
- ~LowmemQMpse()
- {
- if (obj)
- KTrieDelete(obj);
- }
-
- int add_pattern(
- SnortConfig*, const uint8_t* P, unsigned m,
- bool noCase, bool negative, void* user) override
- {
- return KTrieAddPattern(obj, P, m, noCase, negative, user);
- }
-
- int prep_patterns(SnortConfig* sc) override
- {
- return KTrieCompileWithSnortConf(sc, obj);
- }
-
- int _search(
- const uint8_t* T, int n, MpseMatch match,
- void* context, int* current_state) override
- {
- *current_state = 0;
- return KTrieSearchQ(obj, T, n, match, context);
- }
-
- int get_pattern_count() override
- {
- return KTriePatternCount(obj);
- }
-};
-
-//-------------------------------------------------------------------------
-// api
-//-------------------------------------------------------------------------
-
-static Module* mod_ctor()
-{ return new LowmemQModule; }
-
-static void mod_dtor(Module* m)
-{ delete m; }
-
-static Mpse* lmq_ctor(SnortConfig*, class Module* mod, bool use_gc, const MpseAgent* agent)
-{
- LowmemQModule* lmqm = (LowmemQModule*)mod;
- s_var = lmqm->var;
- return new LowmemQMpse(use_gc, agent);
-}
-
-static void lmq_dtor(Mpse* p)
-{
- delete p;
-}
-
-static void lmq_init()
-{
- KTrie_init_xlatcase();
- KTrieInitMemUsed();
-}
-
-static void lmq_print()
-{
- if ( !KTrieMemUsed() )
- return;
-
- if ( !s_var.empty() )
- LogMessage("lowmemq.var = %s\n", s_var.c_str());
-
- double x = (double)KTrieMemUsed();
-
- LogMessage("[ LowMem Search-Method Memory Used : %g %s ]\n",
- (x > 1.e+6) ? x/1.e+6 : x/1.e+3,
- (x > 1.e+6) ? "MBytes" : "KBytes");
-}
-
-static const MpseApi lmq_api =
-{
- {
- PT_SEARCH_ENGINE,
- sizeof(MpseApi),
- SEAPI_VERSION,
- 0,
- API_RESERVED,
- API_OPTIONS,
- s_name,
- "Keyword Trie (low memory, moderate performance) MPSE with queued events",
- mod_ctor,
- mod_dtor
- },
- false,
- nullptr,
- nullptr,
- nullptr,
- nullptr,
- lmq_ctor,
- lmq_dtor,
- lmq_init,
- lmq_print,
-};
-
-const BaseApi* se_lowmem_q = &lmq_api.base;
-
#include "sfksearch.h"
#include "search_engines/pat_stats.h"
-#define SFKSEARCH_TRACK_Q
-
static void KTrieFree(KTRIENODE* n);
static unsigned int mtot = 0;
}
}
-static int KTrieBuildMatchStateNodeWithSnortConf(
+static int KTrieBuildMatchStateNode(
SnortConfig* sc, KTRIENODE* root, KTRIE_STRUCT* ts)
{
int cnt = 0;
/* for child of this root */
if (root->child)
{
- cnt += KTrieBuildMatchStateNodeWithSnortConf(sc, root->child, ts);
+ cnt += KTrieBuildMatchStateNode(sc, root->child, ts);
}
/* 1st sibling of this root -- other siblings will be processed from
* within the processing for root->sibling. */
if (root->sibling)
{
- cnt += KTrieBuildMatchStateNodeWithSnortConf(sc, root->sibling, ts);
+ cnt += KTrieBuildMatchStateNode(sc, root->sibling, ts);
}
return cnt;
}
-static int KTrieBuildMatchStateTreesWithSnortConf(SnortConfig* sc, KTRIE_STRUCT* ts)
+static int KTrieBuildMatchStateTrees(SnortConfig* sc, KTRIE_STRUCT* ts)
{
int i, cnt = 0;
KTRIENODE* root;
/* each and every prefix match at this root*/
if ( root and ts->agent )
{
- cnt += KTrieBuildMatchStateNodeWithSnortConf(sc, root, ts);
+ cnt += KTrieBuildMatchStateNode(sc, root, ts);
}
}
return 0;
}
-int KTrieCompileWithSnortConf(SnortConfig* sc, KTRIE_STRUCT* ts)
+int KTrieCompile(SnortConfig* sc, KTRIE_STRUCT* ts)
{
int rval;
return rval;
if ( ts->agent )
- KTrieBuildMatchStateTreesWithSnortConf(sc, ts);
+ KTrieBuildMatchStateTrees(sc, ts);
return 0;
}
void sfksearch_print_qinfo(void)
{
-#ifdef SFKSEARCH_TRACK_Q
- print_pat_stats("sfksearch", SFK_MAX_INQ);
-#endif
-}
-
-static inline void _init_queue(SFK_PMQ* b)
-{
- b->inq=0;
- b->inq_flush=0;
-}
-
-/* uniquely insert into q */
-static inline int _add_queue(SFK_PMQ* b, void* p)
-{
- int i;
-
-#ifdef SFKSEARCH_TRACK_Q
- pmqs.tot_inq_inserts++;
-#endif
-
- for (i=(int)(b->inq)-1; i>=0; i--)
- if ( p == b->q[i] )
- return 0;
-
-#ifdef SFKSEARCH_TRACK_Q
- pmqs.tot_inq_uinserts++;
-#endif
-
- if ( b->inq < SFK_MAX_INQ )
- {
- b->q[ b->inq++ ] = p;
- }
-
- if ( b->inq == SFK_MAX_INQ )
- {
-#ifdef SFKSEARCH_TRACK_Q
- b->inq_flush++;
-#endif
- return 1;
- }
- return 0;
-}
-
-static inline unsigned _process_queue(SFK_PMQ* q, MpseMatch match, void* context)
-{
- KTRIEPATTERN* pk;
- unsigned int i;
-
-#ifdef SFKSEARCH_TRACK_Q
- if ( q->inq > pmqs.max_inq )
- pmqs.max_inq = q->inq;
- pmqs.tot_inq_flush += q->inq_flush;
-#endif
-
- for ( i=0; i<q->inq; i++ )
- {
- pk = (KTRIEPATTERN*)q->q[i];
- if (pk)
- {
- if (match(pk->user, pk->rule_option_tree, 0, context, pk->neg_list) > 0)
- {
- q->inq=0;
- return 1;
- }
- }
- }
- q->inq=0;
- return 0;
-}
-
-static inline int KTriePrefixMatchQ(
- KTRIE_STRUCT* kt, const uint8_t* T, int n, MpseMatch match, void* context)
-{
- KTRIENODE* root;
- //KTRIEPATTERN * pk;
- //int index ;
-
- root = kt->root[ xlatcase[*T] ];
-
- if ( !root )
- return 0;
-
- while ( n )
- {
- if ( root->edge == xlatcase[*T] )
- {
- T++;
- n--;
-
- if ( root->pkeyword )
- {
- if ( _add_queue(&kt->q, root->pkeyword) )
- {
- if ( _process_queue(&kt->q,match,context) )
- {
- return 1;
- }
- }
- }
-
- if ( n && root->child )
- {
- root = root->child;
- }
- else /* cannot continue -- match is over */
- {
- break;
- }
- }
- else
- {
- if ( root->sibling )
- {
- root = root->sibling;
- }
- else /* cannot continue */
- {
- break;
- }
- }
- }
-
- return 0;
+ print_pat_stats("lowmem", 0);
}
/*
pk = root->pkeyword;
if (pk)
{
- index = (int)(T - bT - pk->n );
+ index = (int)(T - bT);
nfound++;
if (match (pk->user, pk->rule_option_tree, index, context, pk->neg_list) > 0)
{
return nfound;
}
-int KTrieSearchQ(
- KTRIE_STRUCT* ks, const uint8_t* T, int n, MpseMatch match, void* context)
-{
- _init_queue(&ks->q);
- while ( n > 0 )
- {
- if ( KTriePrefixMatchQ(ks, T++, n--, match, context) )
- return 0;
- }
- _process_queue(&ks->q,match,context);
-
- return 0;
-}
-
-static inline int KTrieSearchQBC(
- KTRIE_STRUCT* ks, const uint8_t* T, int n, MpseMatch match, void* context)
-{
- int tshift;
- const uint8_t* Tend;
- short* bcShift = (short*)ks->bcShift;
- int bcSize = ks->bcSize;
-
- _init_queue(&ks->q);
-
- Tend = T + n - bcSize;
-
- bcSize--;
-
- for (; T <= Tend; n--, T++ )
- {
- while ( (tshift = bcShift[ T[bcSize] ]) > 0 )
- {
- T += tshift;
- if ( T > Tend )
- return 0;
- }
-
- if ( KTriePrefixMatchQ(ks, T, n, match, context) )
- return 0;
- }
-
- _process_queue(&ks->q,match,context);
-
- return 0;
-}
-
/*
*
*/
printf("Patterns added \n");
- KTrieCompileWithSnortConf(nullptr, ts);
+ KTrieCompile(nullptr, ts);
printf("Patterns compiled \n");
printf("--> %d characters, %d patterns, %d bytes allocated\n",ts->nchars,ts->npats,ts->memory);
#include <stdint.h>
#include "search_engines/search_common.h"
-#define ALPHABET_SIZE 256
-
-#define KTRIEMETHOD_STD 0
-#define KTRIEMETHOD_QUEUE 1
-
struct KTRIEPATTERN
{
KTRIEPATTERN* next; /* global list of all patterns*/
#define KTRIE_ROOT_NODES 256
-#define SFK_MAX_INQ 32
-
-struct SFK_PMQ
-{
- unsigned inq;
- unsigned inq_flush;
- void* q[SFK_MAX_INQ];
-};
-
struct KTRIE_STRUCT
{
KTRIEPATTERN* patrn; /* List of patterns, built as they are added*/
int bcSize;
unsigned short bcShift[KTRIE_ROOT_NODES];
-
- SFK_PMQ q;
};
void KTrie_init_xlatcase();
KTRIE_STRUCT*, const uint8_t* P, unsigned n,
bool nocase, bool negative, void* id);
-int KTrieCompileWithSnortConf(struct SnortConfig*, KTRIE_STRUCT*);
+int KTrieCompile(struct SnortConfig*, KTRIE_STRUCT*);
int KTrieSearch(KTRIE_STRUCT*, const uint8_t* T, int n, MpseMatch, void* context);
-int KTrieSearchQ(KTRIE_STRUCT*, const uint8_t* T, int n, MpseMatch, void* context);
unsigned int KTrieMemUsed();
void KTrieInitMemUsed();
#include "framework/mpse.h"
extern const BaseApi* se_lowmem;
-extern const BaseApi* se_lowmem_q;
SO_PUBLIC const BaseApi* snort_plugins[] =
{
se_lowmem,
- se_lowmem_q,
nullptr
};
//--------------------------------------------------------------------------
// unit_test.h author Russ Combs <rucombs@cisco.com>
+#include "unit_test.h"
+
#include <stdlib.h>
#include <string.h>
#include <vector>
#include <string>
-#include "unit_test.h"
+#define CATCH_CONFIG_RUNNER
+#include "catch.hpp"
static bool s_catch = false;
static std::vector<std::string> test_tags;
-void unit_test_catch_test_filter(const char* s)
+void catch_set_filter(const char* s)
{
if ( s && strcmp(s, "all") )
test_tags.push_back( s );
return s_catch;
}
-// check defines fail, so we must squash that because
-// catch uses stream and that has a fail method
-#undef fail
-#define CATCH_CONFIG_RUNNER
-#include "catch.hpp"
-
static bool run_catch()
{
Catch::Session session;
// Unit test interface
-void unit_test_catch_test_filter(const char* s);
+void catch_set_filter(const char* s);
bool catch_enabled();
// Add the match for this otn to the queue.
{
OptTreeNode* otn = (OptTreeNode*)node->option_data;
- PatternMatchData* pmd = (PatternMatchData*)eval_data->pmd;
-
- int pattern_size = 0;
- if ( pmd )
- pattern_size = pmd->pattern_size;
-
int16_t app_proto = p->get_application_protocol();
int check_ports = 1;
otn->state[get_instance_id()].matches++;
if ( !eval_data->flowbit_noalert )
+ {
+ PatternMatchData* pmd = (PatternMatchData*)eval_data->pmd;
+ int pattern_size = pmd ? pmd->pattern_size : 0;
fpAddMatch((OTNX_MATCH_DATA*)pomd, pattern_size, otn);
-
+ }
result = rval = DETECTION_OPTION_MATCH;
}
}
max_queue_events = 5;
bleedover_port_limit = 1024;
- search_api = MpseManager::get_search_api("ac_bnfa_q");
+ search_api = MpseManager::get_search_api("ac_bnfa");
assert(search_api);
trim = MpseManager::search_engine_trim(search_api);
}
static void fpDeletePMX(void* data);
-static int fpGetFinalPattern(FastPatternConfig*, PatternMatchData* pmd,
- char** ret_pattern, int* ret_bytes);
+static int fpGetFinalPattern(
+ FastPatternConfig*, PatternMatchData*, char** ret_pattern, int* ret_bytes);
-static void PrintFastPatternInfo(OptTreeNode* otn, PatternMatchData* pmd,
- const char* pattern, int pattern_length);
+static void PrintFastPatternInfo(
+ OptTreeNode*, PatternMatchData*, const char* pattern, int pattern_length);
static const char* const pm_type_strings[PM_TYPE_MAX] =
{
static bool pmd_can_be_fp(PatternMatchData* pmd, CursorActionType cat)
{
- if ( !pmd->pattern_buf || !pmd->pattern_size )
- return false;
-
if ( cat <= CAT_SET_OTHER )
return false;
- if ( !pmd->negated )
- return true;
-
- /* Negative contents can only be considered if they are not relative
- * and don't have any offset or depth. This is because the pattern
- * matcher does not take these into consideration and may find the
- * content in a non-relevant section of the payload and thus disable
- * the rule when it shouldn't be.
- * Also case sensitive patterns cannot be considered since patterns
- * are inserted into the pattern matcher without case which may
- * lead to false negatives */
- if ( pmd->relative || !pmd->no_case ||
- pmd->offset || pmd->depth )
- return false;
-
- return true;
+ return pmd->can_be_fp();
}
struct FpFoo
#include "config.h"
#endif
+#include <strings.h>
+
#include "detect.h"
#include "fp_config.h"
#include "fp_create.h"
#include "protocols/tcp.h"
#include "protocols/udp.h"
#include "protocols/icmp4.h"
+#include "search_engines/pat_stats.h"
THREAD_LOCAL ProfileStats rulePerfStats;
THREAD_LOCAL ProfileStats ruleRTNEvalPerfStats;
{
PMX* pmx = (PMX*)user;
OTNX_MATCH_DATA* pomd = (OTNX_MATCH_DATA*)context;
+
+ unsigned sz = pmx->pmd->pattern_size;
+ assert(sz <= (unsigned)index and (unsigned)index <= pomd->size);
+ assert(!strncasecmp((char*)pmx->pmd->pattern_buf, (char*)pomd->data+index-sz, sz));
+
detection_option_tree_root_t* root = (detection_option_tree_root_t*)tree;
detection_option_eval_data_t eval_data;
NCListNode* ncl;
return 0;
}
+class MpseStash
+{
+public:
+ static const unsigned max = 32;
+
+ void init()
+ { count = flushed = 0; }
+
+ bool push(void* user, void* tree, int index, void* list);
+ bool process(MpseMatch, void*);
+
+private:
+ unsigned count;
+ unsigned flushed;
+
+ struct Node {
+ void* user;
+ void* tree;
+ void* list;
+ int index;
+ } queue[max];
+};
+
+static THREAD_LOCAL MpseStash stash;
+
+// uniquely insert into q, should splay elements for performance
+// return true if maxed out to trigger a flush
+bool MpseStash::push(void* user, void* tree, int index, void* list)
+{
+ pmqs.tot_inq_inserts++;
+
+ for ( int i = (int)(count) - 1; i >= 0; --i )
+ {
+ if ( tree == queue[i].tree )
+ return false;
+ }
+
+ if ( count < max )
+ {
+ Node& node = queue[count++];
+ node.user = user;
+ node.tree = tree;
+ node.index = index;
+ node.list = list;
+ pmqs.tot_inq_uinserts++;
+ }
+
+ if ( count == max )
+ {
+ flushed++;
+ return true;
+ }
+
+ return false;
+}
+
+bool MpseStash::process(MpseMatch match, void* context)
+{
+ if ( count > pmqs.max_inq )
+ pmqs.max_inq = count;
+
+ pmqs.tot_inq_flush += flushed;
+
+ for ( unsigned i = 0; i < count; ++i )
+ {
+ Node& node = queue[i];
+
+ // process a pattern - case is handled by otn processing
+ int res = match(node.user, node.tree, node.index, context, node.list);
+
+ if ( res > 0 )
+ {
+ /* terminate matching */
+ count = 0;
+ return true;
+ }
+ }
+ count = 0;
+ return false;
+}
+
+// rule_tree_match() could be used instead to bypass the queuing
+static int rule_tree_queue(
+ void* user, void* tree, int index, void* context, void* list)
+{
+ if ( stash.push(user, tree, index, list) )
+ {
+ if ( stash.process(rule_tree_match, context) )
+ {
+ return 1;
+ }
+ }
+ return 0;
+}
+
#ifdef PPM_MGR
#define CHECK_PPM() \
if (PPM_PACKET_ABORT_FLAG()) \
assert(so->get_pattern_count() > 0); \
int start_state = 0; \
cnt++; \
- so->search(buf, len, rule_tree_match, omd, &start_state); \
+ omd->data = buf; omd->size = len; \
+ stash.init(); \
+ so->search(buf, len, rule_tree_queue, omd, &start_state); \
+ stash.process(rule_tree_match, omd); \
CHECK_PPM() \
}
{
PortGroup* pg;
Packet* p;
+
+ const uint8_t* data;
+ unsigned size;
+
int check_ports;
MATCH_INFO* matchInfo;
bool unbounded()
{ return !depth; }
+
+ bool can_be_fp()
+ {
+ if ( !pattern_buf || !pattern_size )
+ return false;
+
+ if ( !negated )
+ return true;
+
+ // Negative contents can only be considered if they are not
+ // relative and don't have any offset or depth. This is because
+ // the pattern matcher does not take these into consideration and
+ // may find the content in a non-relevant section of the payload
+ // and thus disable the rule when it shouldn't be.
+
+ // Also case sensitive patterns cannot be considered since patterns
+ // are inserted into the pattern matcher without case which may
+ // lead to false negatives.
+
+ if ( relative || !no_case || offset || depth )
+ return false;
+
+ return true;
+ }
};
#endif
{
public:
HashOption(const char* s, HashPsIdx hpi, HashMatchData* c, HashFunc f, unsigned n) :
- IpsOption(s, RULE_OPTION_TYPE_OTHER)
+ IpsOption(s)
{ config = c; hashf = f; size = n; idx = hpi; assert(n <= MAX_HASH_SIZE); }
~HashOption() { delete config; }
{
public:
IpProtoOption(const IpProtoData& c) :
- IpsOption(s_name, RULE_OPTION_TYPE_OTHER)
+ IpsOption(s_name)
{ config = c; }
uint32_t hash() const override;
uint32_t hash() const override;
bool operator==(const IpsOption&) const override;
+ CursorActionType get_cursor_type() const override
+ { return CAT_ADJUST; }
+
bool is_relative() override
{ return (config->options & SNORT_PCRE_RELATIVE) != 0; }
uint32_t hash() const override;
bool operator==(const IpsOption&) const override;
+ CursorActionType get_cursor_type() const override
+ { return CAT_ADJUST; }
+
bool is_relative() override
{ return config.relative; }
{ "inspect_stream_inserts", Parameter::PT_BOOL, nullptr, "false",
"inspect reassembled payload - disabling is good for performance, bad for detection" },
- { "search_method", Parameter::PT_DYNAMIC, (void*)get_search_methods, "ac_bnfa_q",
+ { "search_method", Parameter::PT_DYNAMIC, (void*)get_search_methods, "ac_bnfa",
"set fast pattern algorithm - choose available search engine" },
{ "split_any_any", Parameter::PT_BOOL, nullptr, "false",
#ifdef UNIT_TEST
else if ( v.is("--catch-test") )
- unit_test_catch_test_filter(v.get_string());
+ catch_set_filter(v.get_string());
#endif
else if ( v.is("--version") )
help_version(sc, v.get_string());
void MpseManager::print_qinfo()
{
sfksearch_print_qinfo();
- bnfa_print_qinfo();
acsmx2_print_qinfo();
}
set (ACSMX2_SOURCES
ac_banded.cc
ac_full.cc
- ac_full_q.cc
ac_sparse.cc
ac_sparse_bands.cc
acsmx2.cc
set (BNFA_SOURCES
ac_bnfa.cc
- ac_bnfa_q.cc
bnfa_search.cc
bnfa_search.h
)
acsmx2_sources = \
ac_banded.cc \
ac_full.cc \
-ac_full_q.cc \
ac_sparse.cc \
ac_sparse_bands.cc \
acsmx2.cc \
bnfa_sources = \
ac_bnfa.cc \
-ac_bnfa_q.cc \
bnfa_search.cc \
bnfa_search.h
+++ /dev/null
-//--------------------------------------------------------------------------
-// Copyright (C) 2014-2015 Cisco and/or its affiliates. All rights reserved.
-// Copyright (C) 2002-2013 Sourcefire, Inc.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License Version 2 as published
-// by the Free Software Foundation. You may not use, modify or distribute
-// this program under any other version of the GNU General Public License.
-//
-// This program is distributed in the hope that it will be useful, but
-// WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, write to the Free Software Foundation, Inc.,
-// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-//--------------------------------------------------------------------------
-/*
-* An abstracted interface to the Multi-Pattern Matching routines,
-* thats why we're passing 'void *' objects around.
-*
-* Marc A Norton <mnorton@sourcefire.com>
-*
-* Updates:
-* 3/06 - Added AC_BNFA search
-*/
-
-#include "bnfa_search.h"
-
-#include "main/snort_debug.h"
-#include "main/snort_types.h"
-#include "main/snort_config.h"
-#include "utils/util.h"
-#include "profiler/profiler.h"
-#include "framework/mpse.h"
-
-//-------------------------------------------------------------------------
-// "ac_bnfa_q"
-//-------------------------------------------------------------------------
-
-class AcBnfaQMpse : public Mpse
-{
-private:
- bnfa_struct_t* obj;
-
-public:
- AcBnfaQMpse(SnortConfig*, bool use_gc, const MpseAgent* agent)
- : Mpse("ac_bnfa_q", use_gc)
- {
- obj = bnfaNew(agent);
-
- if (obj)
- obj->bnfaMethod = 0;
- }
-
- ~AcBnfaQMpse()
- {
- if (obj)
- bnfaFree(obj);
- }
-
- void set_opt(int flag) override
- {
- if (obj)
- bnfaSetOpt(obj, flag);
- }
-
- int add_pattern(
- SnortConfig*, const uint8_t* P, unsigned m,
- bool noCase, bool negative, void* user) override
- {
- return bnfaAddPattern(obj, P, m, noCase, negative, user);
- }
-
- int prep_patterns(SnortConfig* sc) override
- {
- return bnfaCompile(sc, obj);
- }
-
- int _search(
- const uint8_t* T, int n, MpseMatch match,
- void* context, int* current_state) override
- {
- /* return is actually the state */
- return _bnfa_search_csparse_nfa_q(
- obj, T, n, match, context, 0 /* start-state */, current_state);
- }
-
- int print_info() override
- {
- bnfaPrintInfo(obj);
- return 0;
- }
-
- int get_pattern_count() override
- {
- return bnfaPatternCount(obj);
- }
-};
-
-//-------------------------------------------------------------------------
-// api
-//-------------------------------------------------------------------------
-
-static Mpse* bnfaq_ctor(
- SnortConfig* sc, class Module*, bool use_gc, const MpseAgent* agent)
-{
- return new AcBnfaQMpse(sc, use_gc, agent);
-}
-
-static void bnfaq_dtor(Mpse* p)
-{
- delete p;
-}
-
-static void bnfaq_init()
-{
- bnfa_init_xlatcase();
- bnfaInitSummary();
-}
-
-static void bnfaq_print()
-{
- bnfaPrintSummary();
-}
-
-static const MpseApi bnfaq_api =
-{
- {
- PT_SEARCH_ENGINE,
- sizeof(MpseApi),
- SEAPI_VERSION,
- 0,
- API_RESERVED,
- API_OPTIONS,
- "ac_bnfa_q",
- "Aho-Corasick Binary NFA (low memory, high performance) with queued events",
- nullptr,
- nullptr
- },
- false,
- nullptr,
- nullptr,
- nullptr,
- nullptr,
- bnfaq_ctor,
- bnfaq_dtor,
- bnfaq_init,
- bnfaq_print,
-};
-
-const BaseApi* se_ac_bnfa_q = &bnfaq_api.base;
-
+++ /dev/null
-//--------------------------------------------------------------------------
-// Copyright (C) 2014-2015 Cisco and/or its affiliates. All rights reserved.
-// Copyright (C) 2013-2013 Sourcefire, Inc.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License Version 2 as published
-// by the Free Software Foundation. You may not use, modify or distribute
-// this program under any other version of the GNU General Public License.
-//
-// This program is distributed in the hope that it will be useful, but
-// WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, write to the Free Software Foundation, Inc.,
-// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-//--------------------------------------------------------------------------
-
-#include "acsmx.h"
-#include "acsmx2.h"
-
-#include "main/snort_debug.h"
-#include "main/snort_types.h"
-#include "main/snort_config.h"
-#include "utils/util.h"
-#include "profiler/profiler.h"
-#include "framework/mpse.h"
-
-//-------------------------------------------------------------------------
-// "ac_full_q"
-//-------------------------------------------------------------------------
-
-class AcfQMpse : public Mpse
-{
-private:
- ACSM_STRUCT2* obj;
-
-public:
- AcfQMpse(SnortConfig*, bool use_gc, const MpseAgent* agent)
- : Mpse("ac_full_q", use_gc)
- {
- obj = acsmNew2(agent);
- if (obj) acsmSelectFormat2(obj, ACF_FULLQ);
- }
-
- ~AcfQMpse()
- {
- if (obj)
- acsmFree2(obj);
- }
-
- void set_opt(int flag) override
- {
- if (obj)
- acsmCompressStates(obj, flag);
- }
-
- int add_pattern(
- SnortConfig*, const uint8_t* P, unsigned m,
- bool noCase, bool negative, void* user) override
- {
- return acsmAddPattern2(obj, P, m, noCase, negative, user);
- }
-
- int prep_patterns(SnortConfig* sc) override
- {
- return acsmCompile2(sc, obj);
- }
-
- int _search(
- const uint8_t* T, int n, MpseMatch match,
- void* context, int* current_state) override
- {
- return acsmSearchSparseDFA_Full_q(obj, T, n, match, context, current_state);
- }
-
- int search_all(
- const uint8_t* T, int n, MpseMatch match,
- void* context, int* current_state) override
- {
- return acsmSearchSparseDFA_Full_q_all(obj, T, n, match, context, current_state);
- }
-
- int print_info() override
- {
- return acsmPrintDetailInfo2(obj);
- }
-
- int get_pattern_count() override
- {
- return acsmPatternCount2(obj);
- }
-};
-
-//-------------------------------------------------------------------------
-// api
-//-------------------------------------------------------------------------
-
-static Mpse* acfq_ctor(
- SnortConfig* sc, class Module*, bool use_gc, const MpseAgent* agent)
-{
- return new AcfQMpse(sc, use_gc, agent);
-}
-
-static void acfq_dtor(Mpse* p)
-{
- delete p;
-}
-
-static void acfq_init()
-{
- acsmx2_init_xlatcase();
- acsm_init_summary();
-}
-
-static void acfq_print()
-{
- acsmPrintSummaryInfo2();
-}
-
-static const MpseApi acfq_api =
-{
- {
- PT_SEARCH_ENGINE,
- sizeof(MpseApi),
- SEAPI_VERSION,
- 0,
- API_RESERVED,
- API_OPTIONS,
- "ac_full_q",
- "Aho-Corasick Full (high memory, best performance) with queued events,"
- " implements search_all()",
- nullptr,
- nullptr
- },
- false,
- nullptr,
- nullptr,
- nullptr,
- nullptr,
- acfq_ctor,
- acfq_dtor,
- acfq_init,
- acfq_print,
-};
-
-const BaseApi* se_ac_full_q = &acfq_api.base;
-
if ( StateTable[state].MatchList != NULL )
{
mlist = StateTable[state].MatchList;
- index = T - mlist->n + 1 - Tc;
+ index = T + 1 - Tc;
nfound++;
if (match(mlist->udata->id, mlist->rule_option_tree, index, context,
mlist->neg_list) > 0)
** 8/28/06
** man - Sparse and SparseBands - fixed off by one in calculating matching index
** SparseBands changed ps increment to 2+n to increment between bands.
-**
-** 01/2008
-** man - added 2 phase pattern matcher using a pattern match queue.
-**
-** Matching states are queued, duplicate matches are dropped,
-** and after the complete buffer scan the queued matches are
-** processed. This improves cacheing performance, and reduces
-** duplicate rule processing. The queue is limited in size and
-** is flushed if it becomes full during the scan. This allows
-** simple insertions. Tracking queue ops is optional, as this can
-** impose a modest performance hit of a few percent.
-**
*/
#include "acsmx2.h"
mlist = MatchList[state];
if (mlist)
{
- index = T - mlist->n - Tc + 1;
+ index = T - Tc + 1;
nfound++;
if (match (mlist->udata, mlist->rule_option_tree, index, context,
mlist->neg_list) > 0)
void acsmx2_print_qinfo(void)
{
#ifdef ACSMX2_TRACK_Q
- print_pat_stats("acsmx2", AC_MAX_INQ);
+ print_pat_stats("acsmx2", 0);
#endif
}
-static inline void _init_queue(PMQ* b)
-{
- b->inq=0;
- b->inq_flush=0;
-}
-
-/* uniquely insert into q, should splay elements for performance */
-static inline int _add_queue(PMQ* b, void* p)
-{
- int i;
-
-#ifdef ACSMX2_TRACK_Q
- pmqs.tot_inq_inserts++;
-#endif
-
- for (i=(int)(b->inq)-1; i>=0; i--)
- if ( p == b->q[i] )
- return 0;
-
-#ifdef ACSMX2_TRACK_Q
- pmqs.tot_inq_uinserts++;
-#endif
-
- if ( b->inq < AC_MAX_INQ )
- {
- b->q[ b->inq++ ] = p;
- }
-
- if ( b->inq == AC_MAX_INQ )
- {
-#ifdef ACSMX2_TRACK_Q
- b->inq_flush++;
-#endif
- return 1;
- }
- return 0;
-}
-
-static inline unsigned _process_queue(PMQ* q, MpseMatch match, void* context)
-{
- ACSM_PATTERN2* mlist;
- unsigned int i;
-
-#ifdef ACSMX2_TRACK_Q
- if ( q->inq > pmqs.max_inq )
- pmqs.max_inq = q->inq;
- pmqs.tot_inq_flush += q->inq_flush;
-#endif
-
- for ( i=0; i<q->inq; i++ )
- {
- mlist = (ACSM_PATTERN2*)q->q[i];
- if (mlist)
- {
- if (match (mlist->udata, mlist->rule_option_tree, 0, context, mlist->neg_list) > 0)
- {
- q->inq = 0;
- return 1;
- }
- }
- }
- q->inq=0;
- return 0;
-}
-
-/*
- * Matching states are queued, duplicate matches are dropped,
- * and after the complete buffer scan, the queued matches are
- * processed. This improves cacheing performance, and reduces
- * duplicate rule processing. The queue is limited in size and
- * is flushed if it becomes full during the scan. This allows
- * simple insertions. Tracking queue ops is optional, as this can
- * impose a modest performance hit of a few percent.
- */
-#define AC_SEARCH_Q \
- for (; T < Tend; T++) \
- { \
- ps = NextState[state]; \
- sindex = xlatcase[T[0]]; \
- if (ps[1]) \
- { \
- if (MatchList[state]) \
- { \
- if (_add_queue(&acsm->q,MatchList[state])) \
- { \
- if (_process_queue(&acsm->q, match, context)) \
- { \
- *current_state = state; \
- return 1; \
- } \
- } \
- } \
- } \
- state = ps[2 + sindex]; \
- }
-
-int acsmSearchSparseDFA_Full_q(
- ACSM_STRUCT2* acsm, const uint8_t* T, int n, MpseMatch match,
- void* context, int* current_state)
-{
- const uint8_t* Tend;
- int sindex;
- acstate_t state;
- ACSM_PATTERN2** MatchList = acsm->acsmMatchList;
-
- Tend = T + n;
-
- if (current_state == NULL)
- return 0;
-
- _init_queue(&acsm->q);
-
- state = *current_state;
-
- switch (acsm->sizeofstate)
- {
- case 1:
- {
- uint8_t* ps;
- uint8_t** NextState = (uint8_t**)acsm->acsmNextState;
- AC_SEARCH_Q;
- }
- break;
- case 2:
- {
- uint16_t* ps;
- uint16_t** NextState = (uint16_t**)acsm->acsmNextState;
- AC_SEARCH_Q;
- }
- break;
- default:
- {
- acstate_t* ps;
- acstate_t** NextState = acsm->acsmNextState;
- AC_SEARCH_Q;
- }
- break;
- }
-
- *current_state = state;
-
- if (MatchList[state])
- _add_queue(&acsm->q,MatchList[state]);
-
- _process_queue(&acsm->q, match, context);
-
- return 0;
-}
-
-/*
- * Matching states are queued, duplicate matches are dropped,
- * and after the complete buffer scan, the queued matches are
- * processed. This improves cacheing performance, and reduces
- * duplicate rule processing. The queue is limited in size and
- * is flushed if it becomes full during the scan. This allows
- * simple insertions. Tracking queue ops is optional, as this can
- * impose a modest performance hit of a few percent.
- */
-#define AC_SEARCH_Q_ALL \
- for (; T < Tend; T++) \
- { \
- ps = NextState[state]; \
- sindex = xlatcase[T[0]]; \
- if (ps[1]) \
- { \
- for ( mlist = MatchList[state]; \
- mlist!= NULL; \
- mlist = mlist->next ) \
- { \
- if ( mlist->nocase || (memcmp (mlist->casepatrn, T - mlist->n, mlist->n) == 0)) \
- { \
- if (_add_queue(&acsm->q,mlist)) \
- { \
- if (_process_queue(&acsm->q, match, context)) \
- { \
- *current_state = state; \
- return 1; \
- } \
- } \
- } \
- } \
- } \
- state = ps[2 + sindex]; \
- }
-
-int acsmSearchSparseDFA_Full_q_all(
- ACSM_STRUCT2* acsm, const uint8_t* T, int n, MpseMatch match,
- void* context, int* current_state)
-{
- const uint8_t* Tend;
- int sindex;
- acstate_t state;
- ACSM_PATTERN2** MatchList = acsm->acsmMatchList;
- ACSM_PATTERN2* mlist;
-
- Tend = T + n;
-
- if (current_state == NULL)
- return 0;
-
- _init_queue(&acsm->q);
-
- state = *current_state;
-
- switch (acsm->sizeofstate)
- {
- case 1:
- {
- uint8_t* ps;
- uint8_t** NextState = (uint8_t**)acsm->acsmNextState;
- AC_SEARCH_Q_ALL;
- }
- break;
- case 2:
- {
- uint16_t* ps;
- uint16_t** NextState = (uint16_t**)acsm->acsmNextState;
- AC_SEARCH_Q_ALL;
- }
- break;
- default:
- {
- acstate_t* ps;
- acstate_t** NextState = acsm->acsmNextState;
- AC_SEARCH_Q_ALL;
- }
- break;
- }
-
- *current_state = state;
-
- for ( mlist = MatchList[state];
- mlist!= NULL;
- mlist = mlist->next )
- {
- if ( mlist->nocase || (memcmp (mlist->casepatrn, T - mlist->n, mlist->n) == 0))
- {
- if (_add_queue(&acsm->q,mlist))
- {
- if (_process_queue(&acsm->q, match, context))
- {
- *current_state = state;
- return 1;
- }
- }
- }
- }
-
- _process_queue(&acsm->q, match, context);
-
- return 0;
-}
-
/*
* Full format DFA search
* Do not change anything here without testing, caching and prefetching
mlist = MatchList[state]; \
if (mlist) \
{ \
- index = T - mlist->n - Tx; \
+ index = T - Tx; \
nfound++; \
if (match (mlist->udata, mlist->rule_option_tree, index, context, \
mlist->neg_list) > 0) \
mlist = MatchList[state];
if (mlist)
{
- index = T - mlist->n - Tx;
+ index = T - Tx;
nfound++;
if (match(mlist->udata, mlist->rule_option_tree, index, context, mlist->neg_list) > 0)
{
mlist!= NULL; \
mlist = mlist->next ) \
{ \
- index = T - mlist->n - Tx; \
+ index = T - Tx; \
if ( mlist->nocase || (memcmp (mlist->casepatrn, Tx + index, mlist->n) == 0)) \
{ \
nfound++; \
mlist!= NULL;
mlist = mlist->next )
{
- index = T - mlist->n - Tx;
+ index = T - Tx;
if ( mlist->nocase || (memcmp (mlist->casepatrn, Tx + index, mlist->n) == 0))
{
mlist = MatchList[state];
if (mlist)
{
- index = T - mlist->n - Tx;
+ index = T - Tx;
nfound++;
if (match (mlist->udata, mlist->rule_option_tree, index, context,
mlist->neg_list) > 0)
mlist = MatchList[state];
if (mlist)
{
- index = T - mlist->n - Tx;
+ index = T - Tx;
nfound++;
if (match (mlist->udata, mlist->rule_option_tree, index, context, mlist->neg_list) > 0)
{
mlist = MatchList[state];
if (mlist)
{
- index = T - mlist->n - Tx;
+ index = T - Tx;
nfound++;
if (match (mlist->udata, mlist->rule_option_tree, index, context, mlist->neg_list) > 0)
{
FSA_DFA
};
-#define AC_MAX_INQ 32
-struct PMQ
-{
- unsigned inq;
- unsigned inq_flush;
- void* q[AC_MAX_INQ];
-};
-
/*
* Aho-Corasick State Machine Struct - one per group of pattterns
*/
int sizeofstate;
int compress_states;
-
- PMQ q;
};
/*
int acsmSearchSparseDFA_Full(
ACSM_STRUCT2*, const uint8_t* T, int n, MpseMatch, void* context, int* current_state);
-int acsmSearchSparseDFA_Full_q(
- ACSM_STRUCT2*, const uint8_t* T, int n, MpseMatch, void* context, int* current_state);
-
int acsmSearchSparseDFA_Banded(
ACSM_STRUCT2*, const uint8_t* T, int n, MpseMatch, void* context, int* current_state);
int acsmSearchSparseDFA_Full_All(
ACSM_STRUCT2*, const uint8_t* Tx, int n, MpseMatch, void* context, int* current_state);
-int acsmSearchSparseDFA_Full_q_all(
- ACSM_STRUCT2*, const uint8_t* T, int n, MpseMatch, void* context, int* current_state);
-
void acsmFree2(ACSM_STRUCT2* acsm);
int acsmPatternCount2(ACSM_STRUCT2* acsm);
void acsmCompressStates(ACSM_STRUCT2*, int);
#include <list>
-#define BNFA_TRACK_Q
-
-#ifdef BNFA_TRACK_Q
-# include "main/snort_config.h"
-#endif
-
#include "search_common.h"
#include "pat_stats.h"
-
#include "main/snort_types.h"
#include "main/snort_debug.h"
#include "utils/stats.h"
BNFA_FREE(tmpMatchList,sizeof(void*) * bnfa->bnfaMaxStates,bnfa->matchlist_memory);
-#ifdef MATCH_LIST_CNT
- bnfa->bnfaMatchListCnt=(unsigned*)calloc(sizeof(unsigned) * bnfa->bnfaNumStates);
- if (!bnfa->bnfaMatchListCnt)
- {
- return -1;
- }
-#endif
/* Alloc a failure state table - only need NumStates */
bnfa->bnfaFailState =(bnfa_state_t*)BNFA_MALLOC(sizeof(bnfa_state_t) * bnfa->bnfaNumStates,
bnfa->failstate_memory);
continue;
}
patrn = (bnfa_pattern_t*)mlist->data;
- if ( ( T - Tx) < patrn->n )
- index = 0;
- else
- index = T - Tx - patrn->n + 1;
+ index = T - Tx + 1;
nfound++;
/* Don't do anything specific for case sensitive patterns and not,
* since that will be covered by the rule tree itself. Each tree
continue;
}
patrn = (bnfa_pattern_t*)mlist->data;
- if ( ( T - Tx) < patrn->n )
- index = 0;
- else
- index = T - Tx - patrn->n + 1;
+ index = T - Tx + 1;
nfound++;
/* Don't do anything specific for case (in)sensitive patterns
* since that will be covered by the rule tree itself. Each
continue;
}
patrn = (bnfa_pattern_t*)mlist->data;
- if ( ( T - Tx) < patrn->n )
- index = 0;
- else
- index = T - Tx - patrn->n + 1;
+ index = T - Tx + 1;
/* Don't do anything specific for case sensitive patterns and not,
* since that will be covered by the rule tree itself. Each tree
* might have both case sensitive & case insensitive patterns.
return -1;
}
-#ifdef BNFA_MAIN
-/*
-* Sparse format for state table using single array storage
-*
-* word 1: state
-* word 2: control-word = cb<<24| fs
-* cb : control-byte
-* : mb | fb | nt
-* mb : bit 8 set if match state, zero otherwise
-* fb : bit 7 set if using full format, zero otherwise
-* nt : number of transitions 0..63 (more than 63 requires full format)
-* fs: failure-transition-state
-* word 3+: byte-value(0-255) << 24 | transition-state
-*/
-static inline unsigned _bnfa_get_next_state_csparse_nfa_qx(
- bnfa_state_t* pcx, unsigned sindex, unsigned input)
-{
- int k;
- int nc;
- int index;
- bnfa_state_t* pcs;
-
- for (;; )
- {
- pcs = pcx + sindex + 1; /* skip state-id == 1st word */
-
- if ( pcs[0] & BNFA_SPARSE_FULL_BIT )
- {
- if ( sindex == 0 )
- {
- return pcs[1+input] & BNFA_SPARSE_MAX_STATE;
- }
- else
- {
- if ( pcs[1+input] & BNFA_SPARSE_MAX_STATE )
- return pcs[1+input] & BNFA_SPARSE_MAX_STATE;
- }
- }
- else
- {
- nc = (pcs[0]>>BNFA_SPARSE_COUNT_SHIFT) & BNFA_SPARSE_MAX_ROW_TRANSITIONS;
- if ( nc > BNFA_SPARSE_LINEAR_SEARCH_LIMIT )
- {
- /* binary search... */
- index = _bnfa_binearch(pcs+1, nc, input);
- if ( index >= 0 )
- {
- return pcs[index+1] & BNFA_SPARSE_MAX_STATE;
- }
- }
- else
- {
- /* linear search... */
- for ( k=0; k<nc; k++ )
- {
- if ( (pcs[k+1]>>BNFA_SPARSE_VALUE_SHIFT) == input )
- {
- return pcs[k+1] & BNFA_SPARSE_MAX_STATE;
- }
- }
- }
- }
-
- return 0; /* no transition keyword match failed */
- }
-}
-#endif
-
/*
* Sparse format for state table using single array storage
*
}
/*
- * Per Pattern case search, case is on per pattern basis
- * standard snort search
- * note: index is not used by snort, so it's commented
- * TRACK_Q can impose a modest couple % performance difference in the
- * pattern matching rate.
+ * Per Pattern case search, case is on per pattern basis standard snort
+ * search note: index is not used by snort, so it's commented
*/
-/* Queue whole pattern groups at end states in AC */
-void bnfa_print_qinfo(void)
-{
-#ifdef BNFA_TRACK_Q
- print_pat_stats("bnfa", MAX_INQ);
-#endif
-}
-
-static inline void _init_queue(bnfa_struct_t* b)
-{
- b->inq=0;
- b->inq_flush=0;
-}
-
-/* uniquely insert into q, should splay elements for performance */
-static inline int _add_queue(bnfa_struct_t* b, bnfa_match_node_t* p)
-{
- int i;
-
-#ifdef BNFA_TRACK_Q
- pmqs.tot_inq_inserts++;
-#endif
-
- for (i=(int)(b->inq)-1; i>=0; i--)
- if ( p == b->q[i] )
- return 0;
-
-#ifdef BNFA_TRACK_Q
- pmqs.tot_inq_uinserts++;
-#endif
-
- if ( b->inq < MAX_INQ )
- {
- b->q[ b->inq++ ] = p;
- }
-
- if ( b->inq == MAX_INQ )
- {
-#ifdef BNFA_TRACK_Q
- b->inq_flush++;
-#endif
- return 1;
- }
-
- return 0;
-}
-
-static inline unsigned _process_queue(
- bnfa_struct_t* bnfa, MpseMatch match, void* context)
-{
- bnfa_match_node_t* mlist;
- bnfa_pattern_t* patrn;
- int res;
- unsigned int i;
-
-#ifdef BNFA_TRACK_Q
- if ( bnfa->inq > pmqs.max_inq )
- pmqs.max_inq = bnfa->inq;
- pmqs.tot_inq_flush += bnfa->inq_flush;
-#endif
-
- for ( i=0; i<bnfa->inq; i++ )
- {
- mlist = (bnfa_match_node_t*)bnfa->q[i];
- if (mlist)
- {
- patrn = (bnfa_pattern_t*)mlist->data;
- /*process a pattern - case is handled by otn processing */
- res = match(patrn->userdata, mlist->rule_option_tree, 0, context,
- mlist->neg_list);
- if ( res > 0 )
- {
- /* terminate matching */
- bnfa->inq=0; /* clear the q */
- return 1;
- }
- }
- }
- bnfa->inq=0; /* clear the q */
- return 0;
-}
-
-#ifdef BNFA_MAIN
-static inline unsigned _bnfa_search_csparse_nfa_qx(
- bnfa_struct_t* bnfa, uint8_t* T, int n, MpseMatch match, void* context)
-{
- bnfa_match_node_t* mlist;
- uint8_t* Tend;
- bnfa_match_node_t** MatchList = bnfa->bnfaMatchList;
- bnfa_state_t* transList = bnfa->bnfaTransList;
- unsigned sindex=0;
-
- Tend = T + n;
-
- for (; T<Tend; T++)
- {
- /* Transition to next state index */
- sindex = _bnfa_get_next_state_csparse_nfa_qx(transList,sindex,xlatcase[*T]);
-
- /* Log matches in this state - if any */
- if ( sindex )
- {
- if ( transList[sindex+1] & BNFA_SPARSE_MATCH_BIT )
- {
- mlist = MatchList[ transList[sindex] ];
- if ( mlist )
- {
- if ( _add_queue(bnfa,mlist) )
- {
- if ( _process_queue(bnfa, match, context) )
- {
- return 1;
- }
- }
- }
- }
- }
- else
- {
- return 0;
- }
- }
- return 0;
-}
-#endif
-
-unsigned _bnfa_search_csparse_nfa_q(
- bnfa_struct_t* bnfa, const uint8_t* T, int n, MpseMatch match,
- void* context, unsigned sindex, int* current_state)
-{
- bnfa_match_node_t* mlist;
- const uint8_t* Tend;
- bnfa_match_node_t** MatchList = bnfa->bnfaMatchList;
- bnfa_state_t* transList = bnfa->bnfaTransList;
- unsigned last_sindex;
-
- Tend = T + n;
-
- _init_queue(bnfa);
-
- for (; T<Tend; T++)
- {
- last_sindex = sindex;
-
- /* Transition to next state index */
- sindex = _bnfa_get_next_state_csparse_nfa(transList,sindex,xlatcase[*T]);
-
- /* Log matches in this state - if any */
- if (sindex && (transList[sindex+1] & BNFA_SPARSE_MATCH_BIT) )
- {
- /* Test for same as last state */
- if ( sindex == last_sindex )
- continue;
-
- mlist = MatchList[ transList[sindex] ];
- if ( mlist )
- {
- if ( _add_queue(bnfa,mlist) )
- {
- if ( _process_queue(bnfa, match, context) )
- {
- *current_state = sindex;
- return 1;
- }
- }
- }
- }
- }
- *current_state = sindex;
-
- return _process_queue(bnfa, match, context);
-}
-
/*
* Per Pattern case search, case is on per pattern basis
* standard snort search
*
- * note: index is not used by snort, so it's commented
*/
unsigned _bnfa_search_csparse_nfa(
bnfa_struct_t* bnfa, const uint8_t* Tx, int n, MpseMatch match,
unsigned last_match=LAST_STATE_INIT;
unsigned last_match_saved=LAST_STATE_INIT;
int res;
-#ifdef MATCH_LIST_CNT
- unsigned* MatchTestCnt = bnfa->bnfaMatchTestCnt;
-#endif
+
T = Tx;
Tend = T + n;
last_match_saved = last_match;
last_match = sindex;
-#ifdef MATCH_LIST_CNT
- if ( MatchList[ transList[sindex] ] )
- MatchTestCnt[ transList[index] ]++;
-#endif
-
{
mlist = MatchList[ transList[sindex] ];
if ( !mlist )
return nfound;
patrn = (bnfa_pattern_t*)mlist->data;
- if ( ( T - Tx) < patrn->n )
- index = 0;
- else
- index = T - Tx - patrn->n + 1;
+ index = T - Tx + 1;
nfound++;
/* Don't do anything specific for case sensitive patterns and not,
* since that will be covered by the rule tree itself. Each tree
{
mlist = MatchList[ transList[sindex] ];
patrn = (bnfa_pattern_t*)mlist->data;
- if ( ( T - Tx) < patrn->n )
- index = 0;
- else
- index = T - Tx - patrn->n + 1;
+ index = T - Tx + 1;
nfound++;
/* Don't do anything specific for case sensitive patterns and not,
* since that will be covered by the rule tree itself. Each tree
{
mlist = MatchList[ transList[sindex] ];
patrn = (bnfa_pattern_t*)mlist->data;
- if ( ( T - Tx) < patrn->n )
- index = 0;
- else
- index = T - Tx - patrn->n + 1;
+ index = T - Tx + 1;
nfound++;
/* Don't do anything specific for case sensitive patterns and not,
* since that will be covered by the rule tree itself. Each tree
px->failstate_memory += p->failstate_memory;
}
-#ifdef MATCH_LIST_CNT
-void bnfaPrintMatchListCnt(bnfa_struct_t* p)
-{
- unsigned* cnt = p->bnfaMatchListCnt;
- int i;
- bnfa_match_node_t* mn;
- bnfa_pattern_t* patrn;
-
- printf("[ MatchListCnt for ac-bnfa state machine\n ]");
-
- for (i=0; i<bnfa->bnfaNumStates; i++)
- {
- if ( cnt[i] )
- {
- printf("state[%d] cnt=%d",i,cnt[i]);
- mn = bnfa->MatchList[i];
- if ( mn )
- {
- patrn =(bnfa_pattern_t*)mn->data;
- //xprintOTNSidGid(cnt,patrn->userdata);
- }
- printf("\n");
- fflush(stdout);
- }
- }
-}
-
-#endif
-
#ifdef BNFA_MAIN
#include <stdarg.h>
/*
* The state or sindex of the state machine. This can than be passed back
* in on the next search, if desired.
*/
-static unsigned bnfaSearchX(
- bnfa_struct_t* bnfa, uint8_t* T, int n, MpseMatch match,
- void* context, unsigned, int*)
-{
- int ret;
-
- _init_queue(bnfa);
- while ( n > 0)
- {
- ret = _bnfa_search_csparse_nfa_qx(bnfa, T++, n--, match, context);
-
- if ( ret )
- return 0;
- }
- return _process_queue(bnfa, match, context);
-}
static unsigned bnfaSearch(
bnfa_struct_t* bnfa, uint8_t* Tx, int n, MpseMatch match,
int nextstate_memory;
int failstate_memory;
int matchlist_memory;
-
-#define MAX_INQ 32
- unsigned inq;
- unsigned inq_flush;
- void* q[MAX_INQ];
};
/*
bnfa_struct_t * pstruct, const uint8_t* t, int tlen, MpseMatch,
void* context, unsigned sindex, int* current_state);
-unsigned _bnfa_search_csparse_nfa_q(
- bnfa_struct_t * pstruct, const uint8_t* t, int tlen, MpseMatch,
- void* context, unsigned sindex, int* current_state);
-
int bnfaPatternCount(bnfa_struct_t* p);
void bnfaPrint(bnfa_struct_t* pstruct); /* prints the nfa states-verbose!! */
* MPSE = multi-pattern search engine
* DFA = deterministic finite automaton
* NFA = non-DFA
+* HFA = hybrid FA
-This code has has evolved through 3 major versions:
+This code has has evolved through 4 major versions:
1. acsmx.cc: ac_std
-2. acsmx2.cc: ac_full, ac_full_q, ac_sparse, ac_banded, ac_sparse_bands
-3. bnfa_search.cc: ac_bnfa, ac_bnfa_q
+2. acsmx2.cc: ac_full, ac_sparse, ac_banded, ac_sparse_bands
+3. bnfa_search.cc: ac_bnfa
+ intel_cpm.cc: intel_cpm was added later based on ac_bnfa
+4. hyperscan.cc: support of regex fast patterns
Check the comments at the start of the above files for details on the
implementation.
transitions are not stored
* sparse bands - a list of bands
-The *_q flavors use a match queue to defer rule tree evaluation until after
-the full buffer is searched in order to keep the cache warm. This aspect
-should be orthogonal such that any method can be used with or w/o a match
-queue.
+Version 4 entails a number of refactoring changes to support regex fast
+patterns using hyperscan, an HFA. A key change is to return the offset of
+the end of match the way hyperscan does to support relative matches to fast
+pattern only contents and regexes.
+
+Version 4 also includes refactoring the match queues. The *_q flavors were
+deleted as this aspect is orthogonal to the algorithm. Instead a match
+queue is always used to defer rule tree evaluation until after the full
+buffer is searched in order to keep the cache warm. This is a development
+decision based on overall performance.
+
+Note that hyperscan essentially results in single branch detection option
+trees because from a client view each match state is unique - one per rule.
+This is a potential negative impact on performance but does not yet seem
+significant. Furthermore, regex based fast patterns may obviate the need
+for the tree. However, the tree remains as it is essential for other
+algorithms.
+
+intel_cpm will likely be deleted as it requires a license and does not
+perform as well as hyperscan. It remains pending further performance
+evaluations.
SearchTool makes it easy to use ac_bnfa. This is used by http, pop, imap,
and smtp.
return 0;
}
-// FIXIT-P first cut does not queue matches which will likley be required
-// to improve cache performance. for now each match results in an
-// immediate callback.
-
int HyperscanMpse::match(unsigned id, unsigned long long to)
{
assert(id < pvector.size());
IntelPmMatchState* mstate = (IntelPmMatchState*)q->q[i];
if (mstate != NULL)
{
+ // FIXIT-L delete soft-cpm or delete queue and get index
if (match(mstate->user_data, mstate->rule_option_tree, 0, context,
mstate->neg_list) > 0)
{
struct BaseApi;
extern const BaseApi* se_ac_bnfa;
-extern const BaseApi* se_ac_bnfa_q;
#ifdef INTEL_SOFT_CPM
extern const BaseApi* se_intel_cpm;
#ifdef STATIC_SEARCH_ENGINES
extern const BaseApi* se_ac_banded;
extern const BaseApi* se_ac_full;
-extern const BaseApi* se_ac_full_q;
extern const BaseApi* se_ac_sparse;
extern const BaseApi* se_ac_sparse_bands;
extern const BaseApi* se_ac_std;
const BaseApi* search_engines[] =
{
se_ac_bnfa,
- se_ac_bnfa_q,
#ifdef INTEL_SOFT_CPM
se_intel_cpm,
#ifdef STATIC_SEARCH_ENGINES
se_ac_banded,
se_ac_full,
- se_ac_full_q,
se_ac_sparse,
se_ac_sparse_bands,
se_ac_std,
projects / thirdparty / snort3.git / commitdiff
