From a8a93c26ae70fa5abca8daf547e41b229ad033fe Mon Sep 17 00:00:00 2001 From: "Michael Altizer (mialtize)" Date: Thu, 31 Jan 2019 15:11:09 -0500 Subject: [PATCH] Merge pull request #1483 in SNORT/snort3 from ~BBANTWAL/snort3:batching to master Squashed commit of the following: commit ecb607e1f70f760b545101a4dfa687f184aa2c36 Author: Jonathan McDowell Date: Wed Jan 9 14:36:35 2019 +0000 detection: Add search batching infrastructure Batch jobs for submission in fp_detect, allowing for a single submission of related buffers in the MPSE. Add a generic search_batch() which just calls search() for each MPSE instance, and a receive_responses() to provide initial infrastructure for asynchronous operation. --- src/detection/dev_notes.txt | 7 +++ src/detection/fp_detect.cc | 52 +++++++++++++---- src/detection/fp_detect.h | 3 - src/detection/ips_context.h | 2 + src/framework/mpse.cc | 16 ++++++ src/framework/mpse.h | 63 ++++++++++++++++++++- src/search_engines/test/hyperscan_test.cc | 16 ++++++ src/search_engines/test/search_tool_test.cc | 16 ++++++ 8 files changed, 161 insertions(+), 14 deletions(-) diff --git a/src/detection/dev_notes.txt b/src/detection/dev_notes.txt index e92b33979..1c6b7c505 100644 --- a/src/detection/dev_notes.txt +++ b/src/detection/dev_notes.txt @@ -46,6 +46,13 @@ patterns precisely based on all of these parameters can quickly become complicated by both algorithmic implications and implementation details. The procedure described herein is quick and simple. +Some implementations of MPSE have the capability to perform searches for +multiple pattern groups over the same piece of data efficiently, rather than +requiring a separate search for each group. A batch of searches can be built +up from a single piece of data and passed to the search_batch() MPSE method, +allowing MPSE specific optimisation of how to carry out the searches to be +performed. + The methodology presented here to solve this problem is based on the premise that we can use the source and destination ports to isolate pattern groups for pattern matching, and rely on an event validation procedure to diff --git a/src/detection/fp_detect.cc b/src/detection/fp_detect.cc index b10760b11..1f956faeb 100644 --- a/src/detection/fp_detect.cc +++ b/src/detection/fp_detect.cc @@ -873,18 +873,30 @@ static int rule_tree_queue( return 0; } -static inline int search_data( +static inline int batch_search( Mpse* so, OtnxMatchData* omd, const uint8_t* buf, unsigned len, PegCount& cnt) { assert(so->get_pattern_count() > 0); - int start_state = 0; cnt++; - omd->data = buf; omd->size = len; - MpseStash* stash = omd->p->context->stash; - stash->init(); + + //FIXIT-P Batch outer UDP payload searches for teredo set and the outer header + //during any signature evaluation + if ( omd->p->ptrs.udph && (omd->p->proto_bits & (PROTO_BIT__TEREDO | PROTO_BIT__GTP)) ) + { + int start_state = 0; + MpseStash* stash = omd->p->context->stash; + stash->init(); + so->search(buf, len, rule_tree_queue, omd, &start_state); + stash->process(rule_tree_match, omd); + } + else + { + MpseBatchKey<> key = MpseBatchKey<>(buf, len); + omd->p->context->searches.items[key].so.push_back(so); + } + dump_buffer(buf, len, omd->p); - so->search(buf, len, rule_tree_queue, omd, &start_state); - stash->process(rule_tree_match, omd); + if ( PacketLatency::fastpath() ) return 1; return 0; @@ -902,7 +914,7 @@ static inline int search_buffer( trace_logf(detection, TRACE_FP_SEARCH, "%" PRIu64 " fp %s.%s[%d]\n", omd->p->context->packet_number, gadget->get_name(), pm_type_strings[pmt], buf.len); - search_data(so, omd, buf.data, buf.len, cnt); + batch_search(so, omd, buf.data, buf.len, cnt); } } return 0; @@ -932,7 +944,7 @@ static int fp_search( trace_logf(detection, TRACE_FP_SEARCH, "%" PRIu64 " fp %s[%u]\n", p->context->packet_number, pm_type_strings[PM_TYPE_PKT], pattern_match_size); - search_data(so, omd, p->data, pattern_match_size, pc.pkt_searches); + batch_search(so, omd, p->data, pattern_match_size, pc.pkt_searches); p->is_cooked() ? pc.cooked_searches++ : pc.raw_searches++; } } @@ -971,7 +983,7 @@ static int fp_search( trace_logf(detection, TRACE_FP_SEARCH, "%" PRIu64 " fp search %s[%d]\n", p->context->packet_number, pm_type_strings[PM_TYPE_FILE], file_data.len); - search_data(so, omd, file_data.data, file_data.len, pc.file_searches); + batch_search(so, omd, file_data.data, file_data.len, pc.file_searches); } } } @@ -1298,7 +1310,18 @@ void fp_full(Packet* p) stash->enable_process(); stash->init(); init_match_info(c->otnx); + + c->searches.mf = rule_tree_queue; + c->searches.context = c->otnx; fpEvalPacket(p, FPTask::BOTH); + + if (c->searches.items.size() > 0) { + c->searches.search(c->searches); + while (c->searches.items.size() > 0) + c->searches.receive_responses(c->searches); + stash->process(rule_tree_match, c->otnx); + } + fpFinalSelectEvent(c->otnx, p); } @@ -1310,7 +1333,14 @@ void fp_partial(Packet* p) stash->init(); stash->disable_process(); init_match_info(c->otnx); + c->searches.mf = rule_tree_queue; + c->searches.context = c->otnx; fpEvalPacket(p, FPTask::FP); + + if (c->searches.items.size() > 0) { + Mpse* so = c->searches.items.begin()->second.so[0]; + so->search(c->searches); + } } void fp_complete(Packet* p) @@ -1318,6 +1348,8 @@ void fp_complete(Packet* p) IpsContext* c = p->context; MpseStash* stash = c->stash; stash->enable_process(); + while (c->searches.items.size() > 0) + c->searches.items.begin()->second.so[0]->receive_responses(c->searches); stash->process(rule_tree_match, c->otnx); fpEvalPacket(p, FPTask::NON_FP); fpFinalSelectEvent(c->otnx, p); diff --git a/src/detection/fp_detect.h b/src/detection/fp_detect.h index e0af16168..71994d958 100644 --- a/src/detection/fp_detect.h +++ b/src/detection/fp_detect.h @@ -86,9 +86,6 @@ struct OtnxMatchData PortGroup* pg; snort::Packet* p; - const uint8_t* data; - unsigned size; - int check_ports; bool have_match; bool do_fp; diff --git a/src/detection/ips_context.h b/src/detection/ips_context.h index 2136f92c4..d226ea2ce 100644 --- a/src/detection/ips_context.h +++ b/src/detection/ips_context.h @@ -27,6 +27,7 @@ #include "main/snort_types.h" #include "framework/codec.h" +#include "framework/mpse.h" // required to get a decent decl of pkth #include "protocols/packet.h" @@ -138,6 +139,7 @@ public: uint8_t* buf; SnortConfig* conf; + MpseBatch searches; MpseStash* stash; OtnxMatchData* otnx; SF_EVENTQ* equeue; diff --git a/src/framework/mpse.cc b/src/framework/mpse.cc index 6e72bd24c..fa97b88bb 100644 --- a/src/framework/mpse.cc +++ b/src/framework/mpse.cc @@ -62,5 +62,21 @@ int Mpse::search_all( return _search(T, n, match, context, current_state); } +void Mpse::search(MpseBatch& batch) +{ + int start_state; + + for ( auto& item : batch.items ) + { + for ( auto& so : item.second.so ) + { + start_state = 0; + so->search(item.first.buf, item.first.len, batch.mf, batch.context, &start_state); + } + item.second.done = true; + } + batch.items.clear(); +} + } diff --git a/src/framework/mpse.h b/src/framework/mpse.h index f1e7d150f..1201478bb 100644 --- a/src/framework/mpse.h +++ b/src/framework/mpse.h @@ -22,9 +22,12 @@ // MPSE = Multi-Pattern Search Engine - ie fast pattern matching. The key // methods of an MPSE are the ability to add patterns, compile a state -// machine from the patterns, and search a buffer for patterns. +// machine from the patterns, and search either a single buffer or a set +// of (related) buffers for patterns. #include +#include +#include #include "framework/base_api.h" #include "main/snort_types.h" @@ -37,9 +40,50 @@ namespace snort #define SEAPI_VERSION ((BASE_API_VERSION << 16) | 0) struct SnortConfig; +class Mpse; struct MpseApi; +struct MpseBatch; struct ProfileStats; +template +struct MpseBatchKey +{ + BUF buf; + LEN len; + MpseBatchKey(BUF b, LEN n) + { + this->buf = b; + this->len = n; + } + + bool operator==(const MpseBatchKey &k) const + { + return buf == k.buf && len == k.len; + } +}; + +struct MpseBatchKeyHash +{ + template + std::size_t operator()(const MpseBatchKey &k) const + { + std::size_t h1 = std::hash()(k.buf); + std::size_t h2 = std::hash()(k.len); + + return h1 ^ h2; + } +}; + +class MpseBatchItem +{ +public: + std::vector so; + bool done; + + MpseBatchItem(Mpse* s = nullptr) + { if (s) so.push_back(s); done = false; } +}; + class SO_PUBLIC Mpse { public: @@ -69,6 +113,10 @@ public: virtual int search_all( const uint8_t* T, int n, MpseMatch, void* context, int* current_state); + virtual void search(MpseBatch& batch); + + virtual bool receive_responses(MpseBatch&) { return true; } + virtual void set_opt(int) { } virtual int print_info() { return 0; } virtual int get_pattern_count() const { return 0; } @@ -91,6 +139,19 @@ private: const MpseApi* api; }; +struct MpseBatch +{ + MpseMatch mf; + void* context; + std::unordered_map, MpseBatchItem, MpseBatchKeyHash> items; + + void search(MpseBatch& batch) + { items.begin()->second.so[0]->search(batch); } + + bool receive_responses(MpseBatch& batch) + { return items.begin()->second.so[0]->receive_responses(batch); } +}; + extern THREAD_LOCAL ProfileStats mpsePerfStats; typedef void (* MpseOptFunc)(SnortConfig*); diff --git a/src/search_engines/test/hyperscan_test.cc b/src/search_engines/test/hyperscan_test.cc index c2cb7e2a4..bfd4ccdb1 100644 --- a/src/search_engines/test/hyperscan_test.cc +++ b/src/search_engines/test/hyperscan_test.cc @@ -55,6 +55,22 @@ int Mpse::search_all( return _search(T, n, match, context, current_state); } +void Mpse::search(MpseBatch& batch) +{ + int start_state; + + for ( auto& item : batch.items ) + { + for ( auto& so : item.second.so ) + { + start_state = 0; + so->search(item.first.buf, item.first.len, batch.mf, batch.context, &start_state); + } + item.second.done = true; + } + batch.items.clear(); +} + SnortConfig s_conf; THREAD_LOCAL SnortConfig* snort_conf = &s_conf; diff --git a/src/search_engines/test/search_tool_test.cc b/src/search_engines/test/search_tool_test.cc index cf159ba2a..2ccc4a61f 100644 --- a/src/search_engines/test/search_tool_test.cc +++ b/src/search_engines/test/search_tool_test.cc @@ -110,6 +110,22 @@ int Mpse::search_all( { return _search(T, n, match, context, current_state); } + +void Mpse::search(MpseBatch& batch) +{ + int start_state; + + for ( auto& item : batch.items ) + { + for ( auto& so : item.second.so ) + { + start_state = 0; + so->search(item.first.buf, item.first.len, batch.mf, batch.context, &start_state); + } + item.second.done = true; + } + batch.items.clear(); +} } extern const BaseApi* se_ac_bnfa; -- 2.47.3