#include "main/snort_debug.h"
#include "main/thread.h"
#include "managers/inspector_manager.h"
+#include "managers/mpse_manager.h"
#include "packet_io/active.h"
#include "parser/parser.h"
#include "profiler/profiler_defs.h"
//--------------------------------------------------------------------------
void DetectionEngine::thread_init()
-{ offloader = new RegexOffload(SnortConfig::get_conf()->offload_threads); }
+{
+ SnortConfig* sc = SnortConfig::get_conf();
+ FastPatternConfig* fp = sc->fast_pattern_config;
+ const MpseApi* offload_search_api = fp->get_offload_search_api();
+
+ // Note: offload_threads is really the maximum number of offload_requests
+ if (offload_search_api and MpseManager::is_async_capable(offload_search_api))
+ {
+ // If the search method is async capable then the searches will be performed directly
+ // by the search engine, without requiring a processing thread.
+ offloader = RegexOffload::get_offloader(sc->offload_threads, false);
+ }
+ else
+ {
+ // If the search method is not async capable then offloaded searches will be performed
+ // in a separate processing thread that the RegexOffload instance needs to create.
+ offloader = RegexOffload::get_offloader(sc->offload_threads, true);
+ }
+}
void DetectionEngine::thread_term()
{ delete offloader; }
p->context->conf = SnortConfig::get_conf();
p->set_offloaded();
+ // Build the searches list in the packet context
+ fp_partial(p);
+
offloader->put(p);
pc.offloads++;
}
if ( depends_on_suspended )
{
fp_partial(p);
+ p->context->searches.search_sync();
sw->suspend();
return true;
}
trace_logf(detection,
TRACE_DETECTION_ENGINE, "(wire) %" PRIu64 " de::sleep\n", get_packet_number());
- const struct timespec blip = { 0, 1 };
- nanosleep(&blip, nullptr);
onload();
}
trace_logf(detection, TRACE_DETECTION_ENGINE, "(wire) %" PRIu64 " de::idle (r=%d)\n",
return search_api->base.name;
}
+bool FastPatternConfig::set_offload_search_method(const char* method)
+{
+ const MpseApi* api = MpseManager::get_search_api(method);
+
+ if ( !api )
+ return false;
+
+ offload_search_api = api;
+ return true;
+}
+
+const char* FastPatternConfig::get_offload_search_method()
+{
+ if ( !offload_search_api )
+ return nullptr;
+
+ return offload_search_api->base.name;
+}
+
void FastPatternConfig::set_max_pattern_len(unsigned int max_len)
{
if (max_pattern_len != 0)
bool set_search_method(const char*);
const char* get_search_method();
+ bool set_offload_search_method(const char*);
+ const char* get_offload_search_method();
+
void set_max_pattern_len(unsigned);
const snort::MpseApi* get_search_api()
{ return search_api; }
+ const snort::MpseApi* get_offload_search_api()
+ { return offload_search_api; }
+
bool get_trim()
{ return trim; }
unsigned set_max(unsigned bytes);
private:
- const snort::MpseApi* search_api;
+ const snort::MpseApi* search_api = nullptr;
+ const snort::MpseApi* offload_search_api = nullptr;
bool inspect_stream_insert = true;
bool trim;
#include "fp_create.h"
#include "framework/mpse.h"
+#include "framework/mpse_batch.h"
#include "hash/ghash.h"
#include "log/messages.h"
#include "main/snort_config.h"
using namespace std;
static unsigned mpse_count = 0;
+static unsigned offload_mpse_count = 0;
static const char* s_group = "";
static void fpDeletePMX(void* data);
-static int fpGetFinalPattern(
- FastPatternConfig*, PatternMatchData*, const char*& ret_pattern, unsigned& ret_bytes);
+static int fpGetFinalPattern(FastPatternConfig*, PatternMatchData*, const char*& ret_pattern,
+ unsigned& ret_bytes, Mpse::MpseType mpse_type);
static void print_nfp_info(const char*, OptTreeNode*);
static void print_fp_info(const char*, const OptTreeNode*, const PatternMatchData*,
return true;
}
-static int otn_create_tree(OptTreeNode* otn, void** existing_tree)
+static int otn_create_tree(OptTreeNode* otn, void** existing_tree, Mpse::MpseType mpse_type)
{
detection_option_tree_node_t* node = nullptr, * child;
bool need_leaf = false;
/* Don't add contents that are only for use in the
* fast pattern matcher */
- if ( is_fast_pattern_only(opt_fp) )
+ if ( is_fast_pattern_only(opt_fp, mpse_type) )
{
opt_fp = opt_fp->next;
continue;
*list = nullptr;
}
-static int pmx_create_tree(SnortConfig* sc, void* id, void** existing_tree)
+static int pmx_create_tree(SnortConfig* sc, void* id, void** existing_tree, Mpse::MpseType mpse_type)
{
assert(existing_tree);
if (!*existing_tree)
*existing_tree = new_root(otn);
- return otn_create_tree(otn, existing_tree);
+ return otn_create_tree(otn, existing_tree, mpse_type);
}
-static int fpFinishPortGroupRule(
- SnortConfig* sc, PortGroup* pg,
- OptTreeNode* otn, PatternMatchData* pmd, FastPatternConfig* fp)
+static int pmx_create_tree_normal(SnortConfig* sc, void* id, void** existing_tree)
{
- if ( !pmd )
- {
- pg->add_nfp_rule(otn);
- print_nfp_info(s_group, otn);
- return 0;
- }
- if ( !pg->mpse[pmd->pm_type] )
- {
- static MpseAgent agent =
- {
- pmx_create_tree, add_patrn_to_neg_list,
- fpDeletePMX, free_detection_option_root, neg_list_free
- };
-
- pg->mpse[pmd->pm_type] = MpseManager::get_search_engine(
- sc, fp->get_search_api(), &agent);
-
- if ( !pg->mpse[pmd->pm_type] )
- {
- ParseError("Failed to create pattern matcher for %d", pmd->pm_type);
- return -1;
- }
- mpse_count++;
-
- if ( fp->get_search_opt() )
- pg->mpse[pmd->pm_type]->set_opt(1);
- }
- if (pmd->is_negated())
- pg->add_nfp_rule(otn);
+ return pmx_create_tree(sc, id, existing_tree, Mpse::MPSE_TYPE_NORMAL);
+}
- else
- pg->add_rule();
+static int pmx_create_tree_offload(SnortConfig* sc, void* id, void** existing_tree)
+{
+ return pmx_create_tree(sc, id, existing_tree, Mpse::MPSE_TYPE_OFFLOAD);
+}
+static int fpFinishPortGroupRule(
+ SnortConfig* sc, snort::Mpse* mpse, OptTreeNode* otn, PatternMatchData* pmd,
+ FastPatternConfig* fp, Mpse::MpseType mpse_type, bool get_final_pat)
+{
const char* pattern;
unsigned pattern_length;
- if (fpGetFinalPattern(fp, pmd, pattern, pattern_length) == -1)
- return -1;
+ if (get_final_pat)
+ {
+ if (fpGetFinalPattern(fp, pmd, pattern, pattern_length, mpse_type) == -1)
+ return -1;
+ }
+ else
+ {
+ pattern = pmd->pattern_buf;
+ pattern_length = pmd->pattern_size;
+ }
if ( fp->get_debug_print_fast_patterns() )
print_fp_info(s_group, otn, pmd, pattern, pattern_length);
Mpse::PatternDescriptor desc(
pmd->is_no_case(), pmd->is_negated(), pmd->is_literal(), pmd->mpse_flags);
- pg->mpse[pmd->pm_type]->add_pattern(sc, (const uint8_t*)pattern, pattern_length, desc, pmx);
+ mpse->add_pattern(sc, (const uint8_t*)pattern, pattern_length, desc, pmx);
return 0;
}
for (i = PM_TYPE_PKT; i < PM_TYPE_MAX; i++)
{
- if (pg->mpse[i] != nullptr)
+ if (pg->mpsegrp[i] != nullptr)
{
- if (pg->mpse[i]->get_pattern_count() != 0)
+ if (pg->mpsegrp[i]->normal_mpse != nullptr)
{
- if ( !sc->test_mode() or sc->mem_check() )
+ if (pg->mpsegrp[i]->normal_mpse->get_pattern_count() != 0)
+ {
+ if ( !sc->test_mode() or sc->mem_check() )
+ {
+ if ( pg->mpsegrp[i]->normal_mpse->prep_patterns(sc) != 0 )
+ FatalError("Failed to compile port group patterns for normal "
+ "search engine.\n");
+ }
+
+ if (fp->get_debug_mode())
+ pg->mpsegrp[i]->normal_mpse->print_info();
+ rules = 1;
+ }
+ else
{
- if ( pg->mpse[i]->prep_patterns(sc) != 0 )
- FatalError("Failed to compile port group patterns.\n");
+ MpseManager::delete_search_engine(pg->mpsegrp[i]->normal_mpse);
+ pg->mpsegrp[i]->normal_mpse = nullptr;
}
+ }
+ if (pg->mpsegrp[i]->offload_mpse != nullptr)
+ {
+ if (pg->mpsegrp[i]->offload_mpse->get_pattern_count() != 0)
+ {
+ if ( !sc->test_mode() or sc->mem_check() )
+ {
+ if ( pg->mpsegrp[i]->offload_mpse->prep_patterns(sc) != 0 )
+ FatalError("Failed to compile port group patterns for offload "
+ "search engine.\n");
+ }
- if (fp->get_debug_mode())
- pg->mpse[i]->print_info();
- rules = 1;
+ if (fp->get_debug_mode())
+ pg->mpsegrp[i]->offload_mpse->print_info();
+ rules = 1;
+ }
+ else
+ {
+ MpseManager::delete_search_engine(pg->mpsegrp[i]->offload_mpse);
+ pg->mpsegrp[i]->offload_mpse = nullptr;
+ }
}
- else
+
+ if ((pg->mpsegrp[i]->normal_mpse == nullptr) and
+ (pg->mpsegrp[i]->offload_mpse == nullptr))
{
- MpseManager::delete_search_engine(pg->mpse[i]);
- pg->mpse[i] = nullptr;
+ delete pg->mpsegrp[i];
+ pg->mpsegrp[i] = nullptr;
}
}
}
for (ruleNode = pg->nfp_head; ruleNode; ruleNode = ruleNode->rnNext)
{
OptTreeNode* otn = (OptTreeNode*)ruleNode->rnRuleData;
- otn_create_tree(otn, &pg->nfp_tree);
+ otn_create_tree(otn, &pg->nfp_tree, Mpse::MPSE_TYPE_NORMAL);
}
finalize_detection_option_tree(sc, (detection_option_tree_root_t*)pg->nfp_tree);
return 0;
}
-static void fpAddAlternatePatterns(SnortConfig* sc, PortGroup* pg,
- OptTreeNode* otn, PatternMatchData* pmd, FastPatternConfig* fp)
+static void fpAddAlternatePatterns(SnortConfig* sc, snort::Mpse* mpse,
+ OptTreeNode* otn, PatternMatchData* pmd, FastPatternConfig* fp, Mpse::MpseType mpse_type)
{
- if ( fp->get_debug_print_fast_patterns() )
- print_fp_info(s_group, otn, pmd, pmd->pattern_buf, pmd->pattern_size);
-
- PMX* pmx = (PMX*)snort_calloc(sizeof(PMX));
- pmx->rule_node.rnRuleData = otn;
- pmx->pmd = pmd;
-
- Mpse::PatternDescriptor desc(
- pmd->is_no_case(), pmd->is_negated(), pmd->is_literal(), pmd->mpse_flags);
-
- pg->mpse[pmd->pm_type]->add_pattern(
- sc, (const uint8_t*)pmd->pattern_buf, pmd->pattern_size, desc, pmx);
+ fpFinishPortGroupRule(sc, mpse, otn, pmd, fp, mpse_type, false);
}
static int fpAddPortGroupRule(
SnortConfig* sc, PortGroup* pg, OptTreeNode* otn, FastPatternConfig* fp, bool srvc)
{
+ const MpseApi* search_api = nullptr;
+ const MpseApi* offload_search_api = nullptr;
PatternMatchVector pmv;
+ OptFpList* next = nullptr;
+ bool only_literal;
+ bool exclude;
// skip builtin rules, continue for text and so rules
if ( otn->sigInfo.builtin )
if ( !otn->enabled )
return -1;
- OptFpList* next = nullptr;
- bool only_literal = !MpseManager::is_regex_capable(fp->get_search_api());
- bool exclude;
+ search_api = fp->get_search_api();
+ assert(search_api);
+
+ only_literal = !MpseManager::is_regex_capable(search_api);
pmv = get_fp_content(otn, next, srvc, only_literal, exclude);
if ( !pmv.empty() )
{
- PatternMatchData* main_pmd = pmv.back();
- pmv.pop_back();
+ PatternMatchVector pmv_ol;
+ OptFpList* next_ol = nullptr;
+ bool add_to_offload = false;
+ bool cont = true;
+ PatternMatchData* ol_pmd = nullptr;
- if ( !main_pmd->is_relative() && !main_pmd->is_negated() && main_pmd->fp_only >= 0 &&
- // FIXIT-L no_case consideration is mpse specific, delegate
- !main_pmd->offset && !main_pmd->depth && main_pmd->is_no_case() )
+ offload_search_api = fp->get_offload_search_api();
+
+ // Only add rule to the offload search engine if the offload search engine
+ // is different to the normal search engine.
+ if (offload_search_api and (offload_search_api != search_api))
{
- if ( !next || !next->ips_opt || !next->ips_opt->is_relative() )
- main_pmd->fp_only = 1;
+ bool exclude_ol;
+ bool only_literal_ol = !MpseManager::is_regex_capable(offload_search_api);
+ pmv_ol = get_fp_content(otn, next_ol, srvc, only_literal_ol, exclude_ol);
+
+ // If we can get a fast_pattern for the normal search engine but not for the
+ // offload search engine then add rule to the non fast pattern list
+ if (!pmv_ol.empty())
+ add_to_offload = true;
+ else
+ cont = false;
}
- if (fpFinishPortGroupRule(sc, pg, otn, main_pmd, fp) == 0)
+ // From here on we will create the mpses that are needed and add the patterns
+ if (cont)
{
- if (main_pmd->pattern_size > otn->longestPatternLen)
- otn->longestPatternLen = main_pmd->pattern_size;
- for (auto p : pmv)
- fpAddAlternatePatterns(sc, pg, otn, p, fp);
+ PatternMatchData* main_pmd = pmv.back();
+ pmv.pop_back();
+
+ if ( !main_pmd->is_relative() && !main_pmd->is_negated() && main_pmd->fp_only >= 0 &&
+ // FIXIT-L no_case consideration is mpse specific, delegate
+ !main_pmd->offset && !main_pmd->depth && main_pmd->is_no_case() )
+ {
+ if ( !next || !next->ips_opt || !next->ips_opt->is_relative() )
+ main_pmd->fp_only |= (1 << Mpse::MPSE_TYPE_NORMAL);
+ }
+
+ static MpseAgent agent =
+ {
+ pmx_create_tree_normal, add_patrn_to_neg_list,
+ fpDeletePMX, free_detection_option_root, neg_list_free
+ };
+
+ if ( pg->mpsegrp[main_pmd->pm_type] == nullptr )
+ {
+ pg->mpsegrp[main_pmd->pm_type] = new MpseGroup;
+ if ( pg->mpsegrp[main_pmd->pm_type] == nullptr )
+ {
+ ParseError("Failed to create pattern matcher for %d", main_pmd->pm_type);
+ return -1;
+ }
+ }
+
+ if (pg->mpsegrp[main_pmd->pm_type]->normal_mpse == nullptr)
+ {
+ if (!pg->mpsegrp[main_pmd->pm_type]->create_normal_mpse(sc, &agent))
+ {
+ ParseError("Failed to create normal pattern matcher for %d", main_pmd->pm_type);
+ return -1;
+ }
+
+ mpse_count++;
+ if ( fp->get_search_opt() )
+ pg->mpsegrp[main_pmd->pm_type]->normal_mpse->set_opt(1);
+ }
+
+ if (add_to_offload)
+ {
+ ol_pmd = pmv_ol.back();
+ pmv_ol.pop_back();
+
+ if ( !ol_pmd->is_relative() && !ol_pmd->is_negated() && ol_pmd->fp_only >= 0 &&
+ // FIXIT-L no_case consideration is mpse specific, delegate
+ !ol_pmd->offset && !ol_pmd->depth && ol_pmd->is_no_case() )
+ {
+ if ( !next_ol || !next_ol->ips_opt || !next_ol->ips_opt->is_relative() )
+ ol_pmd->fp_only |= (1 << Mpse::MPSE_TYPE_OFFLOAD);
+ }
+
+ static MpseAgent agent_offload =
+ {
+ pmx_create_tree_offload, add_patrn_to_neg_list,
+ fpDeletePMX, free_detection_option_root, neg_list_free
+ };
+
+ // Keep the created mpse alongside the same pm type as the main pmd
+ if (pg->mpsegrp[main_pmd->pm_type]->offload_mpse == nullptr)
+ {
+ if (!pg->mpsegrp[main_pmd->pm_type]->create_offload_mpse(sc, &agent_offload))
+ {
+ ParseError("Failed to create offload pattern matcher for %d",
+ main_pmd->pm_type);
+ return -1;
+ }
+
+ offload_mpse_count++;
+ if ( fp->get_search_opt() )
+ pg->mpsegrp[main_pmd->pm_type]->offload_mpse->set_opt(1);
+ }
+ }
+
+ bool add_rule = false;
+ bool add_nfp_rule = false;
+
+ if (pg->mpsegrp[main_pmd->pm_type]->normal_mpse)
+ {
+ add_rule = true;
+ if (main_pmd->is_negated())
+ add_nfp_rule = true;
+
+ // Now add patterns
+ if (fpFinishPortGroupRule(sc, pg->mpsegrp[main_pmd->pm_type]->normal_mpse,
+ otn, main_pmd, fp, Mpse::MPSE_TYPE_NORMAL, true) == 0)
+ {
+ if (main_pmd->pattern_size > otn->longestPatternLen)
+ otn->longestPatternLen = main_pmd->pattern_size;
+
+ // Add Alternative patterns
+ for (auto p : pmv)
+ fpAddAlternatePatterns(sc, pg->mpsegrp[main_pmd->pm_type]->normal_mpse,
+ otn, p, fp, Mpse::MPSE_TYPE_NORMAL);
+ }
+ }
+
+ if (ol_pmd and pg->mpsegrp[main_pmd->pm_type]->offload_mpse)
+ {
+ add_rule = true;
+ if (ol_pmd->is_negated())
+ add_nfp_rule = true;
+
+ // Now add patterns
+ if (fpFinishPortGroupRule(sc, pg->mpsegrp[main_pmd->pm_type]->offload_mpse,
+ otn, ol_pmd, fp, Mpse::MPSE_TYPE_OFFLOAD, true) == 0)
+ {
+ if (ol_pmd->pattern_size > otn->longestPatternLen)
+ otn->longestPatternLen = ol_pmd->pattern_size;
+
+ // Add Alternative patterns
+ for (auto p : pmv_ol)
+ fpAddAlternatePatterns(sc, pg->mpsegrp[main_pmd->pm_type]->offload_mpse,
+ otn, p, fp, Mpse::MPSE_TYPE_OFFLOAD);
+ }
+ }
+
+ if (add_rule)
+ {
+ if (add_nfp_rule)
+ pg->add_nfp_rule(otn);
+ else
+ pg->add_rule();
+ }
return 0;
}
return 0;
// no fast pattern added
- if (fpFinishPortGroupRule(sc, pg, otn, nullptr, fp) != 0)
- return -1;
+ pg->add_nfp_rule(otn);
+ print_nfp_info(s_group, otn);
return 0;
}
static int fpGetFinalPattern(
FastPatternConfig* fp, PatternMatchData* pmd,
- const char*& ret_pattern, unsigned& ret_bytes)
+ const char*& ret_pattern, unsigned& ret_bytes, Mpse::MpseType mpse_type)
{
if ( !fp or !pmd )
{
// 3. non-literals like regex - truncation could invalidate the
// expression.
- if ( pmd->fp_only > 0 or pmd->is_negated() or !pmd->is_literal() )
+ assert((mpse_type == Mpse::MPSE_TYPE_NORMAL) or (mpse_type == Mpse::MPSE_TYPE_OFFLOAD));
+
+ if ( (pmd->fp_only & (1 << mpse_type)) or pmd->is_negated() or !pmd->is_literal() )
{
ret_pattern = pattern;
ret_bytes = bytes;
for (type = PM_TYPE_PKT; type < PM_TYPE_MAX; type++)
{
- int count = pg->mpse[type] ? pg->mpse[type]->get_pattern_count() : 0;
+ if (pg->mpsegrp[type])
+ {
+ int count = pg->mpsegrp[type]->normal_mpse ?
+ pg->mpsegrp[type]->normal_mpse->get_pattern_count() : 0;
+ int count_ol = pg->mpsegrp[type]->offload_mpse ?
+ pg->mpsegrp[type]->offload_mpse->get_pattern_count() : 0;
+
+ if ( count )
+ LogMessage("\tNormal Pattern Matcher %s: %d\n", pm_type_strings[type], count);
- if ( count )
- LogMessage("\t%s: %d\n", pm_type_strings[type], count);
+ if ( count_ol )
+ LogMessage("\tOffload Pattern Matcher %s: %d\n", pm_type_strings[type], count_ol);
+ }
}
if ( pg->nfp_rule_count )
- LogMessage("\tNo content: %u\n", pg->nfp_rule_count);
+ LogMessage("\tNormal Pattern Matcher No content: %u\n", pg->nfp_rule_count);
}
static void fpDeletePMX(void* pv)
return;
for ( int i = PM_TYPE_PKT; i < PM_TYPE_MAX; ++i )
- if ( pg->mpse[i] and pg->mpse[i]->get_pattern_count() )
+ if ( pg->mpsegrp[i] and pg->mpsegrp[i]->normal_mpse and
+ pg->mpsegrp[i]->normal_mpse->get_pattern_count() )
c[i]++;
}
}
mpse_count = 0;
+ offload_mpse_count = 0;
MpseManager::start_search_engine(fp->get_search_api());
LogLabel("search engine");
MpseManager::print_mpse_summary(fp->get_search_api());
}
+ if ( offload_mpse_count and (fp->get_offload_search_api()))
+ {
+ LogLabel("offload search engine");
+ MpseManager::print_mpse_summary(fp->get_offload_search_api());
+ }
if ( fp->get_num_patterns_truncated() )
LogMessage("%25.25s: %-12u\n", "truncated patterns", fp->get_num_patterns_truncated());
}
static inline int batch_search(
- Mpse* so, OtnxMatchData* omd, const uint8_t* buf, unsigned len, PegCount& cnt)
+ MpseGroup* so, OtnxMatchData* omd, const uint8_t* buf, unsigned len, PegCount& cnt)
{
- assert(so->get_pattern_count() > 0);
+ assert(so->get_normal_mpse()->get_pattern_count() > 0);
cnt++;
//FIXIT-P Batch outer UDP payload searches for teredo set and the outer header
int start_state = 0;
MpseStash* stash = omd->p->context->stash;
stash->init();
- so->search(buf, len, rule_tree_queue, omd, &start_state);
+ so->get_normal_mpse()->search(buf, len, rule_tree_queue, omd, &start_state);
stash->process(rule_tree_match, omd);
}
else
{
if ( gadget->get_fp_buf(ibt, omd->p, buf) )
{
- if ( Mpse* so = omd->pg->mpse[pmt] )
+ // Depending on where we are searching we call the appropriate mpse
+ if ( MpseGroup* so = omd->pg->mpsegrp[pmt] )
{
// FIXIT-H DELETE ME done - get the context packet number
trace_logf(detection, TRACE_FP_SEARCH, "%" PRIu64 " fp %s.%s[%d]\n",
if ( (!user_mode or type < 2) and p->data and p->dsize )
{
// ports search raw packet only
- if ( Mpse* so = port_group->mpse[PM_TYPE_PKT] )
+ if ( MpseGroup* so = port_group->mpsegrp[PM_TYPE_PKT] )
{
if ( uint16_t pattern_match_size = p->get_detect_limit() )
{
if ( !user_mode or type > 0 )
{
// file searches file only
- if ( Mpse* so = port_group->mpse[PM_TYPE_FILE] )
+ if ( MpseGroup* so = port_group->mpsegrp[PM_TYPE_FILE] )
{
// FIXIT-M file data should be obtained from
// inspector gadget as is done with search_buffer
c->searches.context = c->otnx;
fpEvalPacket(p, FPTask::BOTH);
- if (c->searches.items.size() > 0) {
- c->searches.search(c->searches);
- while (c->searches.items.size() > 0)
- c->searches.receive_responses(c->searches);
+ if ( c->searches.search_sync() )
stash->process(rule_tree_match, c->otnx);
- }
fpFinalSelectEvent(c->otnx, p);
}
c->searches.mf = rule_tree_queue;
c->searches.context = c->otnx;
fpEvalPacket(p, FPTask::FP);
-
- if (c->searches.items.size() > 0) {
- Mpse* so = c->searches.items.begin()->second.so[0];
- so->search(c->searches);
- }
}
void fp_complete(Packet* p)
IpsContext* c = p->context;
MpseStash* stash = c->stash;
stash->enable_process();
- while (c->searches.items.size() > 0)
- c->searches.items.begin()->second.so[0]->receive_responses(c->searches);
stash->process(rule_tree_match, c->otnx);
fpEvalPacket(p, FPTask::NON_FP);
fpFinalSelectEvent(c->otnx, p);
return ofl->ips_opt->get_pattern(snort_protocol_id, direction);
}
+bool is_fast_pattern_only(OptFpList* ofl, Mpse::MpseType mpse_type)
+{
+ PatternMatchData* pmd = get_pmd(ofl, UNKNOWN_PROTOCOL_ID, RULE_WO_DIR);
+
+ if ( !pmd )
+ return false;
+
+ assert((mpse_type == Mpse::MPSE_TYPE_NORMAL) or (mpse_type == Mpse::MPSE_TYPE_OFFLOAD));
+
+ return (pmd->fp_only & (1 << mpse_type));
+}
+
bool is_fast_pattern_only(OptFpList* ofl)
{
PatternMatchData* pmd = get_pmd(ofl, UNKNOWN_PROTOCOL_ID, RULE_WO_DIR);
if ( pmd->is_fast_pattern() )
{
ParseWarning(WARN_RULES, "content ineligible for fast_pattern matcher - ignored");
- pmd->flags &= ~PatternMatchData::FAST_PAT;
+ // When we have a normal search engine we do not wish to invalidate the user
+ // indicated fast pattern as this may be a valid fast pattern for use in the offload
+ // search engine
+ // pmd->flags &= ~PatternMatchData::FAST_PAT;
}
return false;
}
// fast pattern utilities
#include <vector>
#include "framework/ips_option.h"
+#include "framework/mpse.h"
struct OptFpList;
struct OptTreeNode;
struct PatternMatchData* get_pmd(OptFpList*, SnortProtocolId, snort::RuleDirection);
+bool is_fast_pattern_only(OptFpList*, snort::Mpse::MpseType);
bool is_fast_pattern_only(OptFpList*);
void validate_fast_pattern(OptTreeNode*);
#include "main/snort_types.h"
#include "framework/codec.h"
#include "framework/mpse.h"
+#include "framework/mpse_batch.h"
// required to get a decent decl of pkth
#include "protocols/packet.h"
#include <vector>
#include <thread>
-#include "main/snort_config.h"
-#include "latency/packet_latency.h"
-#include "latency/rule_latency.h"
#include "fp_detect.h"
#include "ips_context.h"
+#include "latency/packet_latency.h"
+#include "latency/rule_latency.h"
+#include "main/snort_config.h"
+// FIXIT-L this could be offloader specific
struct RegexRequest
{
snort::Packet* packet = nullptr;
bool go = true;
};
+RegexOffload* RegexOffload::get_offloader(unsigned max, bool async)
+{
+ if ( async )
+ return new ThreadRegexOffload(max);
+
+ return new MpseRegexOffload(max);
+}
+
//--------------------------------------------------------------------------
-// regex offload implementation
+// base offload implementation
//--------------------------------------------------------------------------
RegexOffload::RegexOffload(unsigned max)
for ( unsigned i = 0; i < max; ++i )
{
RegexRequest* req = new RegexRequest;
- req->thread = new std::thread(worker, req, snort::SnortConfig::get_conf());
idle.emplace_back(req);
}
}
assert(busy.empty());
for ( auto* req : idle )
- {
- req->thread->join();
- delete req->thread;
delete req;
- }
}
void RegexOffload::stop()
{
assert(busy.empty());
+}
- for ( auto* req : idle )
+bool RegexOffload::on_hold(snort::Flow* f) const
+{
+ for ( auto* req : busy )
{
- std::unique_lock<std::mutex> lock(req->mutex);
- req->go = false;
- req->cond.notify_one();
+ if ( req->packet->flow == f )
+ return true;
}
+ return false;
}
-void RegexOffload::worker(RegexRequest* req, snort::SnortConfig* initial_config)
+//--------------------------------------------------------------------------
+// synchronous (ie non) offload implementation
+//--------------------------------------------------------------------------
+
+MpseRegexOffload::MpseRegexOffload(unsigned max) : RegexOffload(max) { }
+
+void MpseRegexOffload::put(snort::Packet* p)
{
- snort::SnortConfig::set_conf(initial_config);
+ assert(p);
+ assert(!idle.empty());
- while ( true )
- {
- {
- std::unique_lock<std::mutex> lock(req->mutex);
- req->cond.wait_for(lock, std::chrono::seconds(1));
+ RegexRequest* req = idle.front();
+ idle.pop_front(); // FIXIT-H use splice to move instead
+ busy.emplace_back(req);
- // setting conf is somewhat expensive, checking the conf is not
- // this occurs here to take advantage if idling
- if ( req->packet and req->packet->context->conf != snort::SnortConfig::get_conf() )
- snort::SnortConfig::set_conf(req->packet->context->conf);
+ req->packet = p;
- if ( !req->go )
- break;
+ if (p->context->searches.items.size() > 0)
+ p->context->searches.offload_search();
+}
- if ( !req->offload )
+bool MpseRegexOffload::get(snort::Packet*& p)
+{
+ assert(!busy.empty());
+
+ for ( auto i = busy.begin(); i != busy.end(); i++ )
+ {
+ RegexRequest* req = *i;
+ snort::IpsContext* c = req->packet->context;
+
+ if ( c->searches.items.size() > 0 )
+ {
+ snort::Mpse::MpseRespType resp_ret = c->searches.receive_offload_responses();
+
+ if (resp_ret == snort::Mpse::MPSE_RESP_NOT_COMPLETE)
continue;
+
+ else if (resp_ret == snort::Mpse::MPSE_RESP_COMPLETE_FAIL)
+ {
+ if (!c->searches.can_fallback())
+ {
+ // FIXIT-M Add peg counts to record offload search fallback attempts
+ c->searches.search_sync();
+ }
+ // FIXIT-M else Add peg counts to record offload search failures
+ }
+ c->searches.items.clear();
}
- assert(req->packet);
- assert(req->packet->is_offloaded());
- fp_partial(req->packet);
+ p = req->packet;
+ req->packet = nullptr;
- req->offload = false;
+ busy.erase(i);
+ idle.emplace_back(req);
+
+ return true;
}
- tterm();
+
+ p = nullptr;
+ return false;
}
-void RegexOffload::tterm()
+//--------------------------------------------------------------------------
+// async (threads) offload implementation
+//--------------------------------------------------------------------------
+
+ThreadRegexOffload::ThreadRegexOffload(unsigned max) : RegexOffload(max)
{
- // FIXIT-M break this over-coupling. In reality we shouldn't be evaluating latency in offload.
- PacketLatency::tterm();
- RuleLatency::tterm();
+ for ( auto* req : idle )
+ req->thread = new std::thread(worker, req, snort::SnortConfig::get_conf());
+}
+
+ThreadRegexOffload::~ThreadRegexOffload()
+{
+ for ( auto* req : idle )
+ {
+ req->thread->join();
+ delete req->thread;
+ }
}
-void RegexOffload::put(snort::Packet* p)
+void ThreadRegexOffload::stop()
+{
+ RegexOffload::stop();
+
+ for ( auto* req : idle )
+ {
+ std::unique_lock<std::mutex> lock(req->mutex);
+ req->go = false;
+ req->cond.notify_one();
+ }
+}
+
+void ThreadRegexOffload::put(snort::Packet* p)
{
assert(p);
assert(!idle.empty());
RegexRequest* req = idle.front();
-
idle.pop_front(); // FIXIT-H use splice to move instead
busy.emplace_back(req);
std::unique_lock<std::mutex> lock(req->mutex);
-
req->packet = p;
- req->offload = true;
- req->cond.notify_one();
+ if (p->context->searches.items.size() > 0)
+ {
+ req->offload = true;
+ req->cond.notify_one();
+ }
}
-bool RegexOffload::get(snort::Packet*& p)
+bool ThreadRegexOffload::get(snort::Packet*& p)
{
assert(!busy.empty());
return true;
}
-
+
p = nullptr;
return false;
}
-bool RegexOffload::on_hold(snort::Flow* f)
+void ThreadRegexOffload::worker(RegexRequest* req, snort::SnortConfig* initial_config)
{
- for ( auto* req : busy )
+ snort::SnortConfig::set_conf(initial_config);
+
+ while ( true )
{
- if ( req->packet->flow == f )
- return true;
+ {
+ std::unique_lock<std::mutex> lock(req->mutex);
+ req->cond.wait_for(lock, std::chrono::seconds(1));
+
+ // setting conf is somewhat expensive, checking the conf is not
+ // this occurs here to take advantage if idling
+ if ( req->packet and req->packet->context->conf != snort::SnortConfig::get_conf() )
+ snort::SnortConfig::set_conf(req->packet->context->conf);
+
+ if ( !req->go )
+ break;
+
+ if ( !req->offload )
+ continue;
+ }
+
+ assert(req->packet);
+ assert(req->packet->is_offloaded());
+ assert(req->packet->context->searches.items.size() > 0);
+
+ snort::MpseBatch& batch = req->packet->context->searches;
+ batch.offload_search();
+ snort::Mpse::MpseRespType resp_ret;
+
+ do
+ {
+ resp_ret = batch.receive_offload_responses();
+ }
+ while (resp_ret == snort::Mpse::MPSE_RESP_NOT_COMPLETE);
+
+ if (resp_ret == snort::Mpse::MPSE_RESP_COMPLETE_FAIL)
+ {
+ if (!batch.can_fallback())
+ {
+ // FIXIT-M Add peg counts to record offload search fallback attempts
+ batch.search_sync();
+ }
+ // FIXIT-M else Add peg counts to record offload search failures
+ }
+
+ batch.items.clear();
+ req->offload = false;
}
- return false;
+
+ // FIXIT-M break this over-coupling. In reality we shouldn't be evaluating latency in offload.
+ PacketLatency::tterm();
+ RuleLatency::tterm();
}
#define REGEX_OFFLOAD_H
// RegexOffload provides an interface to fast pattern search accelerators.
-// currently implemented as a simple thread offload, but will become an
-// abstract base class with true hardware offload subclasses. for starters
-// the thread offload will "cheat" and tightly interface with fp_detect but
-// eventually morph into such a proper subclass as the offload api emerges.
-// presently all offload is per packet thread; packet threads do not share
-// offload resources.
+// There are two flavors: MPSE and thread. The MpseRegexOffload interfaces to
+// an MPSE that is capable of regex offload such as the RXP whereas
+// ThreadRegexOffload implements the regex search in auxiliary threads w/o
+// requiring extra MPSE instances. presently all offload is per packet thread;
+// packet threads do not share offload resources.
#include <condition_variable>
#include <list>
class RegexOffload
{
public:
- RegexOffload(unsigned max);
- ~RegexOffload();
+ static RegexOffload* get_offloader(unsigned max, bool async);
+ virtual ~RegexOffload();
+
+ virtual void stop();
- void stop();
+ virtual void put(snort::Packet*) = 0;
+ virtual bool get(snort::Packet*&) = 0;
- unsigned available()
+ unsigned available() const
{ return idle.size(); }
- unsigned count()
+ unsigned count() const
{ return busy.size(); }
- void put(snort::Packet*);
- bool get(snort::Packet*&);
-
- bool on_hold(snort::Flow*);
+ bool on_hold(snort::Flow*) const;
-private:
- static void worker(RegexRequest*, snort::SnortConfig*);
- static void tterm();
+protected:
+ RegexOffload(unsigned max);
-private:
+protected:
std::list<RegexRequest*> busy;
std::list<RegexRequest*> idle;
};
+class MpseRegexOffload : public RegexOffload
+{
+public:
+ MpseRegexOffload(unsigned max);
+
+ void put(snort::Packet*) override;
+ bool get(snort::Packet*&) override;
+};
+
+class ThreadRegexOffload : public RegexOffload
+{
+public:
+ ThreadRegexOffload(unsigned max);
+ ~ThreadRegexOffload();
+
+ void stop() override;
+
+ void put(snort::Packet*) override;
+ bool get(snort::Packet*&) override;
+
+private:
+ static void worker(RegexRequest*, snort::SnortConfig*);
+};
+
#endif
{
std::string& name = file->get_file_name();
- if (name.length() <= 0)
+ if ( name.empty() )
return;
size_t fname_len = name.length();
lua_api.h
module.h
mpse.h
+ mpse_batch.h
parameter.h
range.h
so_rule.h
parameter.cc
module.cc
mpse.cc
+ mpse_batch.cc
range.cc
value.cc
)
#include "mpse.h"
+#include <cassert>
+
#include "profiler/profiler_defs.h"
#include "search_engines/pat_stats.h"
+#include "managers/mpse_manager.h"
+#include "managers/module_manager.h"
+#include "main/snort_config.h"
+#include "detection/fp_config.h"
+#include "mpse_batch.h"
using namespace std;
return _search(T, n, match, context, current_state);
}
-void Mpse::search(MpseBatch& batch)
+void Mpse::search(MpseBatch& batch, MpseType mpse_type)
{
int start_state;
for ( auto& item : batch.items )
{
+ if (item.second.done)
+ continue;
+
+ item.second.error = false;
+ item.second.matches = 0;
+
for ( auto& so : item.second.so )
{
start_state = 0;
- so->search(item.first.buf, item.first.len, batch.mf, batch.context, &start_state);
+ switch (mpse_type)
+ {
+ case MPSE_TYPE_NORMAL:
+ item.second.matches += so->get_normal_mpse()->search(item.first.buf,
+ item.first.len, batch.mf, batch.context, &start_state);
+ break;
+ case MPSE_TYPE_OFFLOAD:
+ item.second.matches += so->get_offload_mpse()->search(item.first.buf,
+ item.first.len, batch.mf, batch.context, &start_state);
+ break;
+ }
}
item.second.done = true;
}
- batch.items.clear();
}
}
// machine from the patterns, and search either a single buffer or a set
// of (related) buffers for patterns.
+#include <cassert>
#include <string>
#include <unordered_map>
#include <vector>
struct MpseBatch;
struct ProfileStats;
-template<typename BUF = const uint8_t*, typename LEN = unsigned>
-struct MpseBatchKey
+class SO_PUBLIC Mpse
{
- BUF buf;
- LEN len;
- MpseBatchKey(BUF b, LEN n)
- {
- this->buf = b;
- this->len = n;
- }
-
- bool operator==(const MpseBatchKey &k) const
+public:
+ enum MpseType
{
- return buf == k.buf && len == k.len;
- }
-};
+ MPSE_TYPE_NORMAL = 0,
+ MPSE_TYPE_OFFLOAD = 1
+ };
-struct MpseBatchKeyHash
-{
- template <class BUF, class LEN>
- std::size_t operator()(const MpseBatchKey<BUF, LEN> &k) const
+ enum MpseRespType
{
- std::size_t h1 = std::hash<BUF>()(k.buf);
- std::size_t h2 = std::hash<LEN>()(k.len);
-
- return h1 ^ h2;
- }
-};
-
-class MpseBatchItem
-{
-public:
- std::vector<Mpse*> so;
- bool done;
-
- MpseBatchItem(Mpse* s = nullptr)
- { if (s) so.push_back(s); done = false; }
-};
+ MPSE_RESP_COMPLETE_FAIL = -1,
+ MPSE_RESP_NOT_COMPLETE = 0,
+ MPSE_RESP_COMPLETE_SUCCESS = 1
+ };
-class SO_PUBLIC Mpse
-{
-public:
virtual ~Mpse() = default;
struct PatternDescriptor
virtual int search_all(
const uint8_t* T, int n, MpseMatch, void* context, int* current_state);
- virtual void search(MpseBatch& batch);
+ virtual void search(MpseBatch& batch, MpseType mpse_type);
- virtual bool receive_responses(MpseBatch&) { return true; }
+ virtual MpseRespType receive_responses(MpseBatch&)
+ { return MPSE_RESP_COMPLETE_SUCCESS; }
virtual void set_opt(int) { }
virtual int print_info() { return 0; }
const MpseApi* api;
};
-struct MpseBatch
-{
- MpseMatch mf;
- void* context;
- std::unordered_map<MpseBatchKey<>, MpseBatchItem, MpseBatchKeyHash> items;
-
- void search(MpseBatch& batch)
- { items.begin()->second.so[0]->search(batch); }
-
- bool receive_responses(MpseBatch& batch)
- { return items.begin()->second.so[0]->receive_responses(batch); }
-};
-
extern THREAD_LOCAL ProfileStats mpsePerfStats;
typedef void (* MpseOptFunc)(SnortConfig*);
#define MPSE_BASE 0x00
#define MPSE_TRIM 0x01
#define MPSE_REGEX 0x02
+#define MPSE_ASYNC 0x04
struct MpseApi
{
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2018-2019 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+// mpse_batch.cc author Titan IC Systems <support@titan-ic.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "mpse_batch.h"
+
+#include "profiler/profiler_defs.h"
+#include "search_engines/pat_stats.h"
+#include "managers/mpse_manager.h"
+#include "managers/module_manager.h"
+#include "main/snort_config.h"
+#include "detection/fp_config.h"
+
+#include <cassert>
+
+using namespace std;
+
+namespace snort
+{
+
+//-------------------------------------------------------------------------
+// batch stuff
+//-------------------------------------------------------------------------
+
+bool MpseBatch::search_sync()
+{
+ bool searches = false;
+
+ if (items.size() > 0)
+ {
+ Mpse::MpseRespType resp_ret;
+ searches = true;
+
+ search();
+ do
+ {
+ resp_ret = receive_responses();
+ }
+ while (resp_ret == Mpse::MPSE_RESP_NOT_COMPLETE);
+
+ // Assumption here is that a normal search engine will never fail
+ }
+ items.clear();
+
+ return searches;
+}
+
+//-------------------------------------------------------------------------
+// group stuff
+//-------------------------------------------------------------------------
+
+MpseGroup::~MpseGroup()
+{
+ if (normal_mpse)
+ {
+ MpseManager::delete_search_engine(normal_mpse);
+ normal_mpse = nullptr;
+ }
+ if (offload_mpse)
+ {
+ MpseManager::delete_search_engine(offload_mpse);
+ offload_mpse = nullptr;
+ }
+}
+
+bool MpseGroup::create_normal_mpse(SnortConfig* sc, const MpseAgent* agent)
+{
+ FastPatternConfig* fp = sc->fast_pattern_config;
+ const MpseApi* search_api = nullptr;
+
+ if (fp)
+ search_api = fp->get_search_api();
+
+ if (search_api != nullptr)
+ {
+ normal_mpse = MpseManager::get_search_engine(sc, search_api, agent);
+ return true;
+ }
+ else
+ {
+ normal_mpse = nullptr;
+ return false;
+ }
+}
+
+bool MpseGroup::create_normal_mpse(const char* type)
+{
+ if ( !type and SnortConfig::get_conf()->fast_pattern_config )
+ type = SnortConfig::get_conf()->fast_pattern_config->get_search_method();
+
+ if ( !type )
+ type = "ac_bnfa";
+
+ const MpseApi* search_api = MpseManager::get_search_api(type);
+
+ if (search_api)
+ {
+ Module* mod = ModuleManager::get_module(search_api->base.name);
+ normal_mpse = search_api->ctor(nullptr, mod, nullptr);
+ normal_mpse->set_api(search_api);
+ return true;
+ }
+ else
+ {
+ normal_mpse = nullptr;
+ return false;
+ }
+}
+
+bool MpseGroup::create_offload_mpse(SnortConfig* sc, const MpseAgent* agent)
+{
+ FastPatternConfig* fp = sc->fast_pattern_config;
+ const MpseApi* search_api = nullptr;
+ const MpseApi* offload_search_api = nullptr;
+
+ if (fp)
+ {
+ search_api = fp->get_search_api();
+ offload_search_api = fp->get_offload_search_api();
+ }
+
+ if (offload_search_api and (offload_search_api != search_api))
+ {
+ offload_mpse = MpseManager::get_search_engine(sc, offload_search_api, agent);
+ return true;
+ }
+ else
+ {
+ offload_mpse = nullptr;
+ return false;
+ }
+}
+
+bool MpseGroup::create_offload_mpse()
+{
+ const MpseApi* search_api = nullptr;
+ const MpseApi* offload_search_api = nullptr;
+
+ if (SnortConfig::get_conf()->fast_pattern_config )
+ {
+ search_api = SnortConfig::get_conf()->fast_pattern_config->get_search_api();
+ offload_search_api = SnortConfig::get_conf()->fast_pattern_config->get_offload_search_api();
+ }
+
+ if (offload_search_api and (offload_search_api != search_api))
+ {
+ Module* mod = ModuleManager::get_module(offload_search_api->base.name);
+ offload_mpse = offload_search_api->ctor(nullptr, mod, nullptr);
+ offload_mpse->set_api(offload_search_api);
+ return true;
+ }
+ else
+ {
+ offload_mpse = nullptr;
+ return false;
+ }
+}
+
+}
+
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2018-2019 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+// mpse_batch.h author Titan IC Systems <support@titan-ic.com>
+
+#ifndef MPSE_BATCH_H
+#define MPSE_BATCH_H
+
+#include "framework/mpse.h"
+#include "main/snort_types.h"
+
+namespace snort
+{
+class SO_PUBLIC MpseGroup
+{
+public:
+ MpseGroup()
+ { normal_mpse = nullptr; offload_mpse = nullptr; }
+
+ MpseGroup(Mpse* normal)
+ { normal_mpse = normal; offload_mpse = nullptr; }
+
+ ~MpseGroup();
+
+ Mpse* get_normal_mpse() const
+ { return normal_mpse; }
+
+ // Offload will only be actioned if the offload_search_api is configured.
+ // If the offload_search_api is the same as the normal_search_api then
+ // only the normal mpse will get created and thus if the offload mpse
+ // is requested the normal mpse will be returned otherwise the offload_mpse
+ // will get returned
+ Mpse* get_offload_mpse() const
+ { return offload_mpse ? offload_mpse : normal_mpse; }
+
+ bool create_normal_mpse(SnortConfig*, const MpseAgent* agent);
+ bool create_normal_mpse(const char*);
+
+ bool create_offload_mpse(SnortConfig*, const MpseAgent* agent);
+ bool create_offload_mpse();
+
+ inline bool can_fallback() const
+ { return get_offload_mpse() != normal_mpse; }
+
+public: // FIXIT-L privatize
+ Mpse* normal_mpse;
+ Mpse* offload_mpse;
+};
+
+template<typename BUF = const uint8_t*, typename LEN = unsigned>
+struct MpseBatchKey
+{
+ BUF buf;
+ LEN len;
+ MpseBatchKey(BUF b, LEN n)
+ {
+ this->buf = b;
+ this->len = n;
+ }
+
+ bool operator==(const MpseBatchKey &k) const
+ {
+ return buf == k.buf && len == k.len;
+ }
+};
+
+struct MpseBatchKeyHash
+{
+ template <class BUF, class LEN>
+ std::size_t operator()(const MpseBatchKey<BUF, LEN> &k) const
+ {
+ std::size_t h1 = std::hash<BUF>()(k.buf);
+ std::size_t h2 = std::hash<LEN>()(k.len);
+
+ return h1 ^ h2;
+ }
+};
+
+class MpseBatchItem
+{
+public:
+ std::vector<MpseGroup*> so;
+ bool done;
+ bool error;
+ int matches;
+
+ MpseBatchItem(MpseGroup* s = nullptr)
+ { if (s) so.push_back(s); done = false; error = false; matches = 0; }
+};
+
+struct MpseBatch
+{
+ MpseMatch mf;
+ void* context;
+ std::unordered_map<MpseBatchKey<>, MpseBatchItem, MpseBatchKeyHash> items;
+
+ void search();
+ Mpse::MpseRespType receive_responses();
+
+ void offload_search();
+ Mpse::MpseRespType receive_offload_responses();
+
+ bool search_sync();
+ bool can_fallback() const;
+};
+
+inline void MpseBatch::search()
+{
+ items.begin()->second.so[0]->get_normal_mpse()->
+ search(*this, Mpse::MPSE_TYPE_NORMAL);
+}
+
+inline Mpse::MpseRespType MpseBatch::receive_responses()
+{
+ return items.begin()->second.so[0]->get_normal_mpse()->receive_responses(*this);
+}
+
+inline void MpseBatch::offload_search()
+{
+ assert(items.begin()->second.so[0]->get_offload_mpse());
+
+ items.begin()->second.so[0]->get_offload_mpse()->
+ search(*this, Mpse::MPSE_TYPE_OFFLOAD);
+}
+
+inline Mpse::MpseRespType MpseBatch::receive_offload_responses()
+{
+ assert(items.begin()->second.so[0]->get_offload_mpse());
+
+ return items.begin()->second.so[0]->get_offload_mpse()->
+ receive_responses(*this);
+}
+
+inline bool MpseBatch::can_fallback() const
+{
+ return items.begin()->second.so[0]->can_fallback();
+}
+}
+
+#endif
+
{
LruListIter list_iter;
- if (newsize <= 0)
+ if (newsize == 0)
return false; // Not allowed to set size to zero.
std::lock_guard<std::mutex> cache_lock(cache_mutex);
#ifdef UNIT_TEST
TEST_CASE("b64 decode", "[Base64Encoder]")
{
- Base64Encoder b64;
+ Base64Encoder b64e;
const char* text = "The quick brown segment jumped over the lazy dogs.\n";
const char* code = "VGhlIHF1aWNrIGJyb3duIHNlZ21lbnQganVtcGVkIG92ZXIgdGhlIGxhenkgZG9ncy4K";
SECTION("no decode")
{
- CHECK(!b64.finish(buf));
+ CHECK(!b64e.finish(buf));
}
SECTION("null data")
{
- CHECK(!b64.encode(nullptr, 0, buf));
- CHECK(!b64.finish(buf));
+ CHECK(!b64e.encode(nullptr, 0, buf));
+ CHECK(!b64e.finish(buf));
}
SECTION("zero length data")
{
- CHECK(!b64.encode((const uint8_t*)"ignore", 0, buf));
- CHECK(!b64.finish(buf));
+ CHECK(!b64e.encode((const uint8_t*)"ignore", 0, buf));
+ CHECK(!b64e.finish(buf));
}
SECTION("finish states")
{
for ( unsigned i = 0; i < to_do; ++i )
{
- unsigned n = b64.encode((const uint8_t*)txt[i], strlen(txt[i]), buf);
- n += b64.finish(buf+n);
+ unsigned n = b64e.encode((const uint8_t*)txt[i], strlen(txt[i]), buf);
+ n += b64e.finish(buf+n);
REQUIRE(n < sizeof(buf));
buf[n] = 0;
CHECK(!strcmp(buf, exp[i]));
- b64.reset();
+ b64e.reset();
}
}
SECTION("one shot")
{
- unsigned n = b64.encode((const uint8_t*)text, strlen(text), buf);
- n += b64.finish(buf+n);
+ unsigned n = b64e.encode((const uint8_t*)text, strlen(text), buf);
+ n += b64e.finish(buf+n);
REQUIRE(n < sizeof(buf));
buf[n] = 0;
while ( offset < len )
{
unsigned k = (offset + chunk > len) ? len - offset : chunk;
- n += b64.encode((const uint8_t*)text+offset, k, buf+n);
+ n += b64e.encode((const uint8_t*)text+offset, k, buf+n);
offset += k;
}
- n += b64.finish(buf+n);
+ n += b64e.finish(buf+n);
REQUIRE(n < sizeof(buf));
buf[n] = 0;
CHECK(!strcmp(buf, code));
- b64.reset();
+ b64e.reset();
}
}
}
private:
SdPatternConfig config;
- static void scratch_setup(SnortConfig* sc);
- static void scratch_cleanup(SnortConfig* sc);
+ static void scratch_setup(SnortConfig*);
+ static void scratch_cleanup(SnortConfig*);
};
bool SdPatternModule::begin(const char*, int, SnortConfig*)
return true;
}
-bool SdPatternModule::set(const char*, Value& v, SnortConfig* sc)
+bool SdPatternModule::set(const char*, Value& v, SnortConfig*)
{
if ( v.is("~pattern") )
{
alertdata.sport_itype = htons(p->ptrs.icmph->type);
alertdata.dport_icode = htons(p->ptrs.icmph->code);
}
-
- alertdata.sport_itype = htons(p->ptrs.sp);
- alertdata.dport_icode = htons(p->ptrs.dp);
+ else
+ {
+ alertdata.sport_itype = htons(p->ptrs.sp);
+ alertdata.dport_icode = htons(p->ptrs.dp);
+ }
if ( p->proto_bits & PROTO_BIT__MPLS )
alertdata.mpls_label = htonl(p->ptrs.mplsHdr.label);
alertdata.sport_itype = htons(p->ptrs.icmph->type);
alertdata.dport_icode = htons(p->ptrs.icmph->code);
}
-
- alertdata.sport_itype = htons(p->ptrs.sp);
- alertdata.dport_icode = htons(p->ptrs.dp);
+ else
+ {
+ alertdata.sport_itype = htons(p->ptrs.sp);
+ alertdata.dport_icode = htons(p->ptrs.dp);
+ }
if ( p->proto_bits & PROTO_BIT__MPLS )
alertdata.mpls_label = htonl(p->ptrs.mplsHdr.label);
{ "search_method", Parameter::PT_DYNAMIC, (void*)&get_search_methods, "ac_bnfa",
"set fast pattern algorithm - choose available search engine" },
+ { "offload_search_method", Parameter::PT_DYNAMIC, (void*)&get_search_methods, nullptr,
+ "set fast pattern offload algorithm - choose available search engine" },
+
{ "search_optimize", Parameter::PT_BOOL, nullptr, "true",
"tweak state machine construction for better performance" },
if ( !fp->set_search_method(v.get_string()) )
return false;
}
+ else if ( v.is("offload_search_method") )
+ {
+ if ( !fp->set_offload_search_method(v.get_string()) )
+ return false;
+ }
else if ( v.is("search_optimize") )
fp->set_search_opt(v.get_bool());
SnortConfig::get_conf()->post_setup();
- MpseManager::activate_search_engine(
- SnortConfig::get_conf()->fast_pattern_config->get_search_api(), SnortConfig::get_conf());
+ const MpseApi* search_api = SnortConfig::get_conf()->fast_pattern_config->get_search_api();
+ const MpseApi* offload_search_api = SnortConfig::get_conf()->fast_pattern_config->
+ get_offload_search_api();
+
+ MpseManager::activate_search_engine(search_api, SnortConfig::get_conf());
+
+ if ((offload_search_api != nullptr) and (offload_search_api != search_api))
+ MpseManager::activate_search_engine(offload_search_api, SnortConfig::get_conf());
SFAT_Start();
return (api->flags & MPSE_TRIM) != 0;
}
+bool MpseManager::is_async_capable(const MpseApi* api)
+{
+ assert(api);
+ return (api->flags & MPSE_ASYNC) != 0;
+}
+
bool MpseManager::is_regex_capable(const MpseApi* api)
{
assert(api);
static void start_search_engine(const snort::MpseApi*);
static void stop_search_engine(const snort::MpseApi*);
static bool search_engine_trim(const snort::MpseApi*);
+ static bool is_async_capable(const snort::MpseApi*);
static bool is_regex_capable(const snort::MpseApi*);
static void print_mpse_summary(const snort::MpseApi*);
static void print_search_engine_stats();
static bool is_packet_ignored(AppIdSession* asd, Packet* p, AppidSessionDirection direction)
{
-// FIXIT-M - Need to convert this _dpd stream api call to the correct snort++ method
#ifdef REMOVED_WHILE_NOT_IN_USE
- bool is_http2 = false; // _dpd.streamAPI->is_session_http2(p->flow);
-#else
- bool is_http2 = false;
-#endif
+ bool is_http2 = false; // FIXIT-M _dpd.streamAPI->is_session_http2(p->flow);
+
if (is_http2)
{
if (asd)
return true;
}
}
- else if ( p->is_rebuilt() && !p->flow->is_proxied() )
+ else
+#endif
+ if ( p->is_rebuilt() && !p->flow->is_proxied() )
{
// FIXIT-M: In snort2x, a rebuilt packet was ignored whether it had a session or not.
// Here, we are ignoring rebuilt packet only if it has a session. Why?
}
network_set->pnetwork = (Network**)snort_calloc(count * sizeof(Network*));
SF_LNODE* iter = nullptr;
- int i = 0;
+ int k = 0;
for (network = (Network*)sflist_first(&network_set->networks, &iter);
- network && i < count;
+ network && k < count;
network = (Network*)sflist_next(&iter))
{
- network_set->pnetwork[i++] = network;
+ network_set->pnetwork[k++] = network;
}
/* bubble sort this array */
for (int i = (count - 1); i >= 0; i--)
}
network_set->pnetwork6 = (Network6**)snort_calloc(count * sizeof(Network6*));
SF_LNODE* iter = nullptr;
- int i = 0;
+ int k = 0;
for (network6 = (Network6*)sflist_first(&network_set->networks6, &iter);
- network6 && i < count;
+ network6 && k < count;
network6 = (Network6*)sflist_next(&iter))
{
- network_set->pnetwork6[i++] = network6;
+ network_set->pnetwork6[k++] = network6;
}
/* bubble sort this array */
for (int i = (count - 1); i >= 0; i--)
/* Check if this is a variable that stores an IP */
else if (*value == '$')
{
- sfip_var_t* var;
- if ((var = sfvt_lookup_var(ip_vartable, value)) != nullptr)
+ if ( sfvt_lookup_var(ip_vartable, value) )
{
sfvt_define(ip_vartable, name, value);
return nullptr;
#include "port_group.h"
#include "detection/detection_options.h"
-#include "managers/mpse_manager.h"
+#include "framework/mpse.h"
+#include "framework/mpse_batch.h"
#include "utils/util.h"
void PortGroup::add_rule()
for (int i = PM_TYPE_PKT; i < PM_TYPE_MAX; i++)
{
- if (pg->mpse[i] != nullptr)
+ if (pg->mpsegrp[i])
{
- MpseManager::delete_search_engine(pg->mpse[i]);
- pg->mpse[i] = nullptr;
+ delete pg->mpsegrp[i];
+ pg->mpsegrp[i] = nullptr;
}
}
namespace snort
{
- class Mpse;
+ class MpseGroup;
}
// PortGroup contains a set of fast patterns in the form of an MPSE and a
RULE_NODE* nfp_head, * nfp_tail;
// pattern matchers
- snort::Mpse* mpse[PM_TYPE_MAX];
+ snort::MpseGroup* mpsegrp[PM_TYPE_MAX];
// detection option tree
void* nfp_tree;
bool update_ip4_len = false;
uint8_t num_layers = p->num_layers;
- if ( num_layers <= 0 )
+ if ( num_layers == 0 )
return -1;
c->reset();
update_ip4_len = true;
}
- if (num_layers <= 0)
+ if (num_layers == 0)
return -1;
}
#include "config.h"
#endif
-#include "search_tool.h"
-
#include <cassert>
-#include "managers/mpse_manager.h"
+#include "detection/fp_config.h"
+#include "framework/mpse.h"
+#include "framework/mpse_batch.h"
+#include "main/snort_config.h"
+#include "search_tool.h"
namespace snort
{
SearchTool::SearchTool(const char* method, bool dfa)
{
- mpse = MpseManager::get_search_engine(method);
- assert(mpse);
+ mpsegrp = new MpseGroup;
+
+ // When no method is passed in, a normal search engine mpse will be created
+ // with the search method the same as that configured for the fast pattern
+ // normal search method, and also an offload search engine mpse will be
+ // created with the search method the same as that configured for the fast
+ // pattern offload search method. If a method is passed in then an offload
+ // search engine will not be created
+
+ if (mpsegrp->create_normal_mpse(method))
+ {
+ if( dfa )
+ mpsegrp->normal_mpse->set_opt(1);
+ }
+
+ if (method == nullptr)
+ {
+ if (mpsegrp->create_offload_mpse())
+ {
+ if ( dfa )
+ mpsegrp->offload_mpse->set_opt(1);
+ }
+ }
+
max_len = 0;
- if( dfa )
- mpse->set_opt(1);
}
SearchTool::~SearchTool()
{
- MpseManager::delete_search_engine(mpse);
+ delete mpsegrp;
}
void SearchTool::add(const char* pat, unsigned len, int id, bool no_case)
{
Mpse::PatternDescriptor desc(no_case, false, true);
- if ( mpse )
- mpse->add_pattern(nullptr, pat, len, desc, id);
+ if ( mpsegrp->normal_mpse )
+ mpsegrp->normal_mpse->add_pattern(nullptr, pat, len, desc, id);
+ if ( mpsegrp->offload_mpse )
+ mpsegrp->offload_mpse->add_pattern(nullptr, pat, len, desc, id);
if ( len > max_len )
max_len = len;
void SearchTool::prep()
{
- if ( mpse )
- mpse->prep_patterns(nullptr);
+ if ( mpsegrp->normal_mpse )
+ mpsegrp->normal_mpse->prep_patterns(nullptr);
+ if ( mpsegrp->offload_mpse )
+ mpsegrp->offload_mpse->prep_patterns(nullptr);
}
int SearchTool::find(
bool confine,
void* user_data)
{
+ int num = 0;
+ SnortConfig* sc = SnortConfig::get_conf();
+ FastPatternConfig* fp = sc->fast_pattern_config;
+
if ( confine && max_len > 0 )
{
if ( max_len < len )
if ( !user_data )
user_data = (void*)str;
- int num = mpse->search((const uint8_t*)str, len, mf, user_data, &state);
+ if ( fp and fp->get_offload_search_api() and (len >= sc->offload_limit) and
+ (mpsegrp->get_offload_mpse() != mpsegrp->get_normal_mpse()) )
+ {
+ num = mpsegrp->get_offload_mpse()->search((const uint8_t*)str, len, mf, user_data, &state);
+
+ if ( num < 0 )
+ num = mpsegrp->get_normal_mpse()->search((const uint8_t*)str, len, mf, user_data,
+ &state);
+ }
+ else
+ num = mpsegrp->get_normal_mpse()->search((const uint8_t*)str, len, mf, user_data, &state);
+
+ // SeachTool::find expects the number found to be returned so if we have a failure return 0
+ if ( num < 0 )
+ num = 0;
return num;
}
bool confine,
void* user_data)
{
+ int num = 0;
+ SnortConfig* sc = SnortConfig::get_conf();
+ FastPatternConfig* fp = sc->fast_pattern_config;
+
if ( confine && max_len > 0 )
{
if ( max_len < len )
int state = 0;
- int num = mpse->search_all((const uint8_t*)str, len, mf, user_data, &state);
+ if ( fp and fp->get_offload_search_api() and (len >= sc->offload_limit) and
+ (mpsegrp->get_offload_mpse() != mpsegrp->get_normal_mpse()) )
+ {
+ num = mpsegrp->get_offload_mpse()->search_all((const uint8_t*)str, len, mf, user_data,
+ &state);
+
+ if ( num < 0 )
+ num = mpsegrp->get_normal_mpse()->search_all((const uint8_t*)str, len, mf, user_data,
+ &state);
+ }
+ else
+ num = mpsegrp->get_normal_mpse()->search_all((const uint8_t*)str, len, mf, user_data,
+ &state);
+
+ // SeachTool::find expects the number found to be returned so if we have a failure return 0
+ if ( num < 0 )
+ num = 0;
return num;
}
bool confine = false, void* user_data = nullptr);
private:
- class Mpse* mpse;
+ class MpseGroup* mpsegrp;
unsigned max_len;
};
} // namespace snort
#include "framework/base_api.h"
#include "framework/mpse.h"
+#include "framework/mpse_batch.h"
#include "main/snort_config.h"
// must appear after snort_config.h to avoid broken c++ map include
return _search(T, n, match, context, current_state);
}
-void Mpse::search(MpseBatch& batch)
+void Mpse::search(MpseBatch& batch, MpseType mpse_type)
{
int start_state;
for ( auto& item : batch.items )
{
+ if (item.second.done)
+ continue;
+
+ item.second.error = false;
+ item.second.matches = 0;
+
for ( auto& so : item.second.so )
{
start_state = 0;
- so->search(item.first.buf, item.first.len, batch.mf, batch.context, &start_state);
+ switch (mpse_type)
+ {
+ case MPSE_TYPE_NORMAL:
+ item.second.matches += so->normal_mpse->search(item.first.buf, item.first.len,
+ batch.mf, batch.context, &start_state);
+ break;
+ case MPSE_TYPE_OFFLOAD:
+ item.second.matches += so->offload_mpse->search(item.first.buf, item.first.len,
+ batch.mf, batch.context, &start_state);
+ break;
+ }
}
item.second.done = true;
}
- batch.items.clear();
}
SnortConfig s_conf;
#include "framework/base_api.h"
#include "framework/mpse.h"
-#include "managers/mpse_manager.h"
+#include "framework/mpse_batch.h"
#include "main/snort_config.h"
+#include "managers/mpse_manager.h"
// must appear after snort_config.h to avoid broken c++ map include
#include <CppUTest/CommandLineTestRunner.h>
return _search(T, n, match, context, current_state);
}
-void Mpse::search(MpseBatch& batch)
+void Mpse::search(MpseBatch& batch, MpseType mpse_type)
{
int start_state;
for ( auto& item : batch.items )
{
+ if (item.second.done)
+ continue;
+
+ item.second.error = false;
+ item.second.matches = 0;
+
for ( auto& so : item.second.so )
{
start_state = 0;
- so->search(item.first.buf, item.first.len, batch.mf, batch.context, &start_state);
+ switch (mpse_type)
+ {
+ case MPSE_TYPE_NORMAL:
+ item.second.matches += so->normal_mpse->search(item.first.buf, item.first.len,
+ batch.mf, batch.context, &start_state);
+ break;
+ case MPSE_TYPE_OFFLOAD:
+ item.second.matches += so->offload_mpse->search(item.first.buf, item.first.len,
+ batch.mf, batch.context, &start_state);
+ break;
+ }
}
item.second.done = true;
}
- batch.items.clear();
}
+
}
extern const BaseApi* se_ac_bnfa;
api->dtor(eng);
}
+MpseGroup::~MpseGroup()
+{
+ if (normal_mpse)
+ {
+ MpseManager::delete_search_engine(normal_mpse);
+ normal_mpse = nullptr;
+ }
+ if (offload_mpse)
+ {
+ MpseManager::delete_search_engine(offload_mpse);
+ offload_mpse = nullptr;
+ }
+}
+
+bool MpseGroup::create_normal_mpse(const char* type)
+{
+ normal_mpse = MpseManager::get_search_engine(type);
+
+ return true;
+}
+
+bool MpseGroup::create_offload_mpse()
+{
+ offload_mpse = nullptr;
+ return false;
+}
+
struct ExpectedMatch
{
int id;
CHECK(se_ac_bnfa);
stool = new SearchTool("ac_bnfa");
- CHECK(stool->mpse);
+ CHECK(stool->mpsegrp->normal_mpse);
int pattern_id = 1;
stool->add("the", 3, pattern_id);
CHECK(se_ac_full);
stool = new SearchTool("ac_full", true);
- CHECK(stool->mpse);
+ CHECK(stool->mpsegrp->normal_mpse);
int pattern_id = 1;
stool->add("the", 3, pattern_id);
Packet* p = DetectionEngine::get_current_packet();
const uint8_t* data_ptr = p->data;
uint16_t data_len = p->dsize;
- uint32_t next_command_offset = 0;
/*Check header length*/
if (data_len < sizeof(NbssHdr) + SMB2_HEADER_LENGTH)
if (p->is_pdu_start())
{
const Smb2Hdr* smb_hdr = (const Smb2Hdr*)(data_ptr + sizeof(NbssHdr));
+ uint32_t next_command_offset;
/* SMB protocol allows multiple smb commands to be grouped in a single packet.
So loop through to parse all the smb commands.
Reference: https://msdn.microsoft.com/en-us/library/cc246614.aspx
}
trace_logf(stream_ip, "Frag Status: %s:%s\n",
- ft->frag_flags&FRAG_GOT_FIRST ? "FIRST" : "No FIRST",
- ft->frag_flags&FRAG_GOT_LAST ? "LAST" : "No LAST");
+ (ft->frag_flags&FRAG_GOT_FIRST) ? "FIRST" : "No FIRST",
+ (ft->frag_flags&FRAG_GOT_LAST) ? "LAST" : "No LAST");
return retVal;
}
bool Pcre::convert(std::istringstream& data_stream)
{
- std::string keyword;
bool sticky_buffer_set = false;
std::string buffer = "pkt_data";
projects / thirdparty / snort3.git / commitdiff
