Merge pull request #1483 in SNORT/snort3 from ~BBANTWAL/snort3:batching to master

Squashed commit of the following:

commit ecb607e1f70f760b545101a4dfa687f184aa2c36
Author: Jonathan McDowell <j.mcdowell@titan-ic.com>
Date:   Wed Jan 9 14:36:35 2019 +0000

    detection: Add search batching infrastructure

    Batch jobs for submission in fp_detect, allowing for a single
    submission of related buffers in the MPSE. Add a generic search_batch()
    which just calls search() for each MPSE instance, and a
    receive_responses() to provide initial infrastructure for asynchronous
    operation.

diff --git a/src/detection/dev_notes.txt b/src/detection/dev_notes.txt
index e92b339790a2d0cc29eac8fbaa5503d3e94b11e6..1c6b7c505e14a6bf455fd74961e6341407db2f7e 100644 (file)
--- a/src/detection/dev_notes.txt
+++ b/src/detection/dev_notes.txt
@@ -46,6 +46,13 @@ patterns precisely based on all of these parameters can quickly become
 complicated by both algorithmic implications and implementation details.
 The procedure described herein is quick and simple.
 
+Some implementations of MPSE have the capability to perform searches for
+multiple pattern groups over the same piece of data efficiently, rather than
+requiring a separate search for each group. A batch of searches can be built
+up from a single piece of data and passed to the search_batch() MPSE method,
+allowing MPSE specific optimisation of how to carry out the searches to be
+performed.
+
 The methodology presented here to solve this problem is based on the
 premise that we can use the source and destination ports to isolate pattern
 groups for pattern matching, and rely on an event validation procedure to

diff --git a/src/detection/fp_detect.cc b/src/detection/fp_detect.cc
index b10760b11ec235b271998aa130d5aa0a1cedf21e..1f956faeb053b3b70644e88ce71a0bad309012c1 100644 (file)
--- a/src/detection/fp_detect.cc
+++ b/src/detection/fp_detect.cc
@@ -873,18 +873,30 @@ static int rule_tree_queue(
     return 0;
 }
 
-static inline int search_data(
+static inline int batch_search(
     Mpse* so, OtnxMatchData* omd, const uint8_t* buf, unsigned len, PegCount& cnt)
 {
     assert(so->get_pattern_count() > 0);
-    int start_state = 0;
     cnt++;
-    omd->data = buf; omd->size = len;
-    MpseStash* stash = omd->p->context->stash;
-    stash->init();
+
+    //FIXIT-P Batch outer UDP payload searches for teredo set and the outer header
+    //during any signature evaluation
+    if ( omd->p->ptrs.udph && (omd->p->proto_bits & (PROTO_BIT__TEREDO | PROTO_BIT__GTP)) )
+    {
+        int start_state = 0;
+        MpseStash* stash = omd->p->context->stash;
+        stash->init();
+        so->search(buf, len, rule_tree_queue, omd, &start_state);
+        stash->process(rule_tree_match, omd);
+    }
+    else
+    {
+        MpseBatchKey<> key = MpseBatchKey<>(buf, len);
+        omd->p->context->searches.items[key].so.push_back(so);
+    }
+
     dump_buffer(buf, len, omd->p);
-    so->search(buf, len, rule_tree_queue, omd, &start_state);
-    stash->process(rule_tree_match, omd);
+
     if ( PacketLatency::fastpath() )
         return 1;
     return 0;
@@ -902,7 +914,7 @@ static inline int search_buffer(
             trace_logf(detection, TRACE_FP_SEARCH, "%" PRIu64 " fp %s.%s[%d]\n",
                 omd->p->context->packet_number, gadget->get_name(), pm_type_strings[pmt], buf.len);
 
-            search_data(so, omd, buf.data, buf.len, cnt);
+            batch_search(so, omd, buf.data, buf.len, cnt);
         }
     }
     return 0;
@@ -932,7 +944,7 @@ static int fp_search(
                 trace_logf(detection, TRACE_FP_SEARCH, "%" PRIu64 " fp %s[%u]\n",
                     p->context->packet_number, pm_type_strings[PM_TYPE_PKT], pattern_match_size);
 
-                search_data(so, omd, p->data, pattern_match_size, pc.pkt_searches);
+                batch_search(so, omd, p->data, pattern_match_size, pc.pkt_searches);
                 p->is_cooked() ?  pc.cooked_searches++ : pc.raw_searches++;
             }
         }
@@ -971,7 +983,7 @@ static int fp_search(
                 trace_logf(detection, TRACE_FP_SEARCH, "%" PRIu64 " fp search %s[%d]\n",
                     p->context->packet_number, pm_type_strings[PM_TYPE_FILE], file_data.len);
 
-                search_data(so, omd, file_data.data, file_data.len, pc.file_searches);
+                batch_search(so, omd, file_data.data, file_data.len, pc.file_searches);
             }
         }
     }
@@ -1298,7 +1310,18 @@ void fp_full(Packet* p)
     stash->enable_process();
     stash->init();
     init_match_info(c->otnx);
+
+    c->searches.mf = rule_tree_queue;
+    c->searches.context = c->otnx;
     fpEvalPacket(p, FPTask::BOTH);
+
+    if (c->searches.items.size() > 0) {
+        c->searches.search(c->searches);
+        while (c->searches.items.size() > 0)
+            c->searches.receive_responses(c->searches);
+        stash->process(rule_tree_match, c->otnx);
+    }
+
     fpFinalSelectEvent(c->otnx, p);
 }
 
@@ -1310,7 +1333,14 @@ void fp_partial(Packet* p)
     stash->init();
     stash->disable_process();
     init_match_info(c->otnx);
+    c->searches.mf = rule_tree_queue;
+    c->searches.context = c->otnx;
     fpEvalPacket(p, FPTask::FP);
+
+    if (c->searches.items.size() > 0) {
+        Mpse* so = c->searches.items.begin()->second.so[0];
+        so->search(c->searches);
+    }
 }
 
 void fp_complete(Packet* p)
@@ -1318,6 +1348,8 @@ void fp_complete(Packet* p)
     IpsContext* c = p->context;
     MpseStash* stash = c->stash;
     stash->enable_process();
+    while (c->searches.items.size() > 0)
+        c->searches.items.begin()->second.so[0]->receive_responses(c->searches);
     stash->process(rule_tree_match, c->otnx);
     fpEvalPacket(p, FPTask::NON_FP);
     fpFinalSelectEvent(c->otnx, p);

diff --git a/src/detection/fp_detect.h b/src/detection/fp_detect.h
index e0af16168d2e8a7f418cb4e73a94c42a8c03c2b0..71994d958d000161cb1f9c9e43e2bb625867cb39 100644 (file)
--- a/src/detection/fp_detect.h
+++ b/src/detection/fp_detect.h
@@ -86,9 +86,6 @@ struct OtnxMatchData
     PortGroup* pg;
     snort::Packet* p;
 
-    const uint8_t* data;
-    unsigned size;
-
     int check_ports;
     bool have_match;
     bool do_fp;

diff --git a/src/detection/ips_context.h b/src/detection/ips_context.h
index 2136f92c47f4b144b5d18b4d197764806dc2717d..d226ea2ce027800fc75c8ab0ac8bd2c312bf2844 100644 (file)
--- a/src/detection/ips_context.h
+++ b/src/detection/ips_context.h
@@ -27,6 +27,7 @@
 
 #include "main/snort_types.h"
 #include "framework/codec.h"
+#include "framework/mpse.h"
 
 // required to get a decent decl of pkth
 #include "protocols/packet.h"
@@ -138,6 +139,7 @@ public:
     uint8_t* buf;
 
     SnortConfig* conf;
+    MpseBatch searches;
     MpseStash* stash;
     OtnxMatchData* otnx;
     SF_EVENTQ* equeue;

diff --git a/src/framework/mpse.cc b/src/framework/mpse.cc
index 6e72bd24c2591e26e9e5c840b1e76d426179de3c..fa97b88bb7e739ce5f46c3db82d3e970d0b1cc76 100644 (file)
--- a/src/framework/mpse.cc
+++ b/src/framework/mpse.cc
@@ -62,5 +62,21 @@ int Mpse::search_all(
     return _search(T, n, match, context, current_state);
 }
 
+void Mpse::search(MpseBatch& batch)
+{
+    int start_state;
+
+    for ( auto& item : batch.items )
+    {
+        for ( auto& so : item.second.so )
+        {
+            start_state = 0;
+            so->search(item.first.buf, item.first.len, batch.mf, batch.context, &start_state);
+        }
+        item.second.done = true;
+    }
+    batch.items.clear();
+}
+
 }
 

diff --git a/src/framework/mpse.h b/src/framework/mpse.h
index f1e7d150f2206fc6c5859247d3a066beef36e45a..1201478bb493a1de8fc36f887f9f23657371769a 100644 (file)
--- a/src/framework/mpse.h
+++ b/src/framework/mpse.h
@@ -22,9 +22,12 @@
 
 // MPSE = Multi-Pattern Search Engine - ie fast pattern matching. The key
 // methods of an MPSE are the ability to add patterns, compile a state
-// machine from the patterns, and search a buffer for patterns.
+// machine from the patterns, and search either a single buffer or a set
+// of (related) buffers for patterns.
 
 #include <string>
+#include <unordered_map>
+#include <vector>
 
 #include "framework/base_api.h"
 #include "main/snort_types.h"
@@ -37,9 +40,50 @@ namespace snort
 #define SEAPI_VERSION ((BASE_API_VERSION << 16) | 0)
 
 struct SnortConfig;
+class Mpse;
 struct MpseApi;
+struct MpseBatch;
 struct ProfileStats;
 
+template<typename BUF = const uint8_t*, typename LEN = unsigned>
+struct MpseBatchKey
+{
+    BUF buf;
+    LEN len;
+    MpseBatchKey(BUF b, LEN n)
+    {
+        this->buf = b;
+        this->len = n;
+    }
+
+    bool operator==(const MpseBatchKey &k) const
+    {
+        return buf == k.buf && len == k.len;
+    }
+};
+
+struct MpseBatchKeyHash
+{
+    template <class BUF, class LEN>
+    std::size_t operator()(const MpseBatchKey<BUF, LEN> &k) const
+    {
+        std::size_t h1 = std::hash<BUF>()(k.buf);
+        std::size_t h2 = std::hash<LEN>()(k.len);
+
+        return h1 ^ h2;
+    }
+};
+
+class MpseBatchItem
+{
+public:
+    std::vector<Mpse*> so;
+    bool done;
+
+    MpseBatchItem(Mpse* s = nullptr)
+    { if (s) so.push_back(s); done = false; }
+};
+
 class SO_PUBLIC Mpse
 {
 public:
@@ -69,6 +113,10 @@ public:
     virtual int search_all(
         const uint8_t* T, int n, MpseMatch, void* context, int* current_state);
 
+    virtual void search(MpseBatch& batch);
+
+    virtual bool receive_responses(MpseBatch&) { return true; }
+
     virtual void set_opt(int) { }
     virtual int print_info() { return 0; }
     virtual int get_pattern_count() const { return 0; }
@@ -91,6 +139,19 @@ private:
     const MpseApi* api;
 };
 
+struct MpseBatch
+{
+    MpseMatch mf;
+    void* context;
+    std::unordered_map<MpseBatchKey<>, MpseBatchItem, MpseBatchKeyHash> items;
+
+    void search(MpseBatch& batch)
+    { items.begin()->second.so[0]->search(batch); }
+
+    bool receive_responses(MpseBatch& batch)
+    { return items.begin()->second.so[0]->receive_responses(batch); }
+};
+
 extern THREAD_LOCAL ProfileStats mpsePerfStats;
 
 typedef void (* MpseOptFunc)(SnortConfig*);

diff --git a/src/search_engines/test/hyperscan_test.cc b/src/search_engines/test/hyperscan_test.cc
index c2cb7e2a45d7c1924c8a3812e6d9594666e32106..bfd4ccdb124d522428a69403416ceee6da4df547 100644 (file)
--- a/src/search_engines/test/hyperscan_test.cc
+++ b/src/search_engines/test/hyperscan_test.cc
@@ -55,6 +55,22 @@ int Mpse::search_all(
     return _search(T, n, match, context, current_state);
 }
 
+void Mpse::search(MpseBatch& batch)
+{
+    int start_state;
+
+    for ( auto& item : batch.items )
+    {
+        for ( auto& so : item.second.so )
+        {
+            start_state = 0;
+            so->search(item.first.buf, item.first.len, batch.mf, batch.context, &start_state);
+        }
+        item.second.done = true;
+    }
+    batch.items.clear();
+}
+
 SnortConfig s_conf;
 THREAD_LOCAL SnortConfig* snort_conf = &s_conf;
 

diff --git a/src/search_engines/test/search_tool_test.cc b/src/search_engines/test/search_tool_test.cc
index cf159ba2a6eb7b6aba86dc08a62c0e1026293e40..2ccc4a61f0b0e48b4588ba0aac791e29a9d296f0 100644 (file)
--- a/src/search_engines/test/search_tool_test.cc
+++ b/src/search_engines/test/search_tool_test.cc
@@ -110,6 +110,22 @@ int Mpse::search_all(
 {
     return _search(T, n, match, context, current_state);
 }
+
+void Mpse::search(MpseBatch& batch)
+{
+    int start_state;
+
+    for ( auto& item : batch.items )
+    {
+        for ( auto& so : item.second.so )
+        {
+            start_state = 0;
+            so->search(item.first.buf, item.first.len, batch.mf, batch.context, &start_state);
+        }
+        item.second.done = true;
+    }
+    batch.items.clear();
+}
 }
 
 extern const BaseApi* se_ac_bnfa;