Merge pull request #3079 in SNORT/snort3 from ~YVELYKOZ/snort3:glob_several_packets...

Squashed commit of the following:

commit b768e09bc0b09ea3aac32b88eaf3b53c2e035e39
Author: Yehor Velykozhon <yvelykoz@cisco.com>
Date:   Mon Sep 13 20:34:15 2021 +0300

    wizard: update globbing and max_pattern

    In order to support globbing over several packets, was added state-variable that contain middle state of pattern.
    Max_pattern now applying per flow instead of pre segment.
    Max_pattern was renamed to max_search_depth.
    Fixed bug with reentering wizard after tcp_hits.

diff --git a/src/service_inspectors/wizard/hexes.cc b/src/service_inspectors/wizard/hexes.cc
index a8344827813c4a6eb93b019ced07dc000141dd9b..1a03ddce0dc0ccc966d5f4928458ed407b1e0b1f 100644 (file)
--- a/src/service_inspectors/wizard/hexes.cc
+++ b/src/service_inspectors/wizard/hexes.cc
@@ -162,18 +162,3 @@ const MagicPage* HexBook::find_spell(
     }
     return p;
 }
-
-const char* HexBook::find_spell(
-    const uint8_t* data, unsigned len, unsigned max, const MagicPage*& p) const
-{
-    if ( max and len > max )
-        len = max;
-
-    p = find_spell(data, len, p, 0);
-
-    if ( p and !p->value.empty() )
-        return p->value.c_str();
-
-    return nullptr;
-}
-

diff --git a/src/service_inspectors/wizard/magic.cc b/src/service_inspectors/wizard/magic.cc
index ffd2567d50889217c80f01d1eeff77a9adc3e967..ba530fa12ba14c60bf066b5d0e380213300b32c0 100644 (file)
--- a/src/service_inspectors/wizard/magic.cc
+++ b/src/service_inspectors/wizard/magic.cc
@@ -23,6 +23,8 @@
 
 #include "magic.h"
 
+#include <cassert>
+
 MagicPage::MagicPage(const MagicBook& b) : book(b)
 {
     for ( int i = 0; i < 256; ++i )
@@ -40,6 +42,18 @@ MagicPage::~MagicPage()
     delete any;
 }
 
+const char* MagicBook::find_spell(const uint8_t* data, unsigned len, const MagicPage*& p) const
+{
+    assert(p);
+
+    p = find_spell(data, len, p, 0);
+
+    if ( p && !p->value.empty() )
+        return p->value.c_str();
+
+    return nullptr;
+}
+
 MagicBook::MagicBook()
 { root = new MagicPage(*this); }
 

diff --git a/src/service_inspectors/wizard/magic.h b/src/service_inspectors/wizard/magic.h
index 8b7db0cc8c42e0984d343d6ef09e8da0eafd6e4d..a34e5d3e5dc785e223e2a3d826bc6a4ddf4645d2 100644 (file)
--- a/src/service_inspectors/wizard/magic.h
+++ b/src/service_inspectors/wizard/magic.h
@@ -52,15 +52,22 @@ public:
     MagicBook& operator=(const MagicBook&) = delete;
 
     virtual bool add_spell(const char* key, const char*& val) = 0;
-    virtual const char* find_spell(const uint8_t*, unsigned len, unsigned max,
-        const MagicPage*&) const = 0;
+    virtual const char* find_spell(const uint8_t*, unsigned len, const MagicPage*&) const;
 
-    const MagicPage* page1()
+    const MagicPage* page1() const
     { return root; }
 
+    virtual void set_bookmark(const MagicPage* page = nullptr) const
+    { (void)page; }
+    virtual const MagicPage* get_bookmark() const
+    { return nullptr; }
+
 protected:
     MagicBook();
     MagicPage* root;
+
+    virtual const MagicPage* find_spell(const uint8_t*, unsigned,
+        const MagicPage*, unsigned) const = 0;
 };
 
 //-------------------------------------------------------------------------
@@ -74,13 +81,19 @@ public:
     SpellBook();
 
     bool add_spell(const char*, const char*&) override;
-    const char* find_spell(const uint8_t*, unsigned len, unsigned max,
-        const MagicPage*&) const override;
+
+    void set_bookmark(const MagicPage* page = nullptr) const override
+    { glob = page; }
+
+    const MagicPage* get_bookmark() const override
+    { return glob; }
 
 private:
     bool translate(const char*, HexVector&);
     void add_spell(const char*, const char*, HexVector&, unsigned, MagicPage*);
-    const MagicPage* find_spell(const uint8_t*, unsigned, const MagicPage*, unsigned) const;
+    const MagicPage* find_spell(const uint8_t*, unsigned, const MagicPage*, unsigned) const override;
+
+    mutable const MagicPage* glob;
 };
 
 //-------------------------------------------------------------------------
@@ -94,13 +107,11 @@ public:
     HexBook() = default;
 
     bool add_spell(const char*, const char*&) override;
-    const char* find_spell(const uint8_t*, unsigned len, unsigned,
-        const MagicPage*&) const override;
 
 private:
     bool translate(const char*, HexVector&);
     void add_spell(const char*, const char*, HexVector&, unsigned, MagicPage*);
-    const MagicPage* find_spell(const uint8_t*, unsigned, const MagicPage*, unsigned) const;
+    const MagicPage* find_spell(const uint8_t*, unsigned, const MagicPage*, unsigned) const override;
 };
 
 #endif

diff --git a/src/service_inspectors/wizard/spells.cc b/src/service_inspectors/wizard/spells.cc
index 51719f79ff133c34d6e76afc02b9433eab8a5f35..227ca4db7bbd30eed3c84e6fa9246cc8780502b5 100644 (file)
--- a/src/service_inspectors/wizard/spells.cc
+++ b/src/service_inspectors/wizard/spells.cc
@@ -29,7 +29,7 @@ using namespace std;
 
 #define WILD 0x100
 
-SpellBook::SpellBook()
+SpellBook::SpellBook() : glob(nullptr)
 {
     // allows skipping leading whitespace only
     root->next[(int)' '] = root;
@@ -152,29 +152,25 @@ const MagicPage* SpellBook::find_spell(
             while ( i < n )
             {
                 if ( const MagicPage* q = find_spell(s, n, p->any, i) )
+                {
+                    glob = q->any ? q : p;
                     return q;
+                }
                 ++i;
             }
-            return p->any;
+            return p;
         }
+
+        // If no match but has glob, continue lookup from glob
+        if ( p->value.empty() && glob )
+        {   
+            p = glob;
+            glob = nullptr;
+            
+            return find_spell(s, n, p, i);
+        }
+
         return p->value.empty() ? nullptr : p;
     }
     return p;
 }
-
-const char* SpellBook::find_spell(
-    const uint8_t* data, unsigned len, unsigned max, const MagicPage*& p) const
-{
-    assert(p);
-
-    if ( max and len > max )
-        len = max;
-
-    p = find_spell(data, len, p, 0);
-
-    if ( p and !p->value.empty() )
-        return p->value.c_str();
-
-    return nullptr;
-}
-

diff --git a/src/service_inspectors/wizard/wiz_module.cc b/src/service_inspectors/wizard/wiz_module.cc
index 5e9cff76962e94f105967fddb63c323c38991cbf..c85a98c9fb09757f2ddb9448d3988ec76c3100b4 100644 (file)
--- a/src/service_inspectors/wizard/wiz_module.cc
+++ b/src/service_inspectors/wizard/wiz_module.cc
@@ -106,8 +106,8 @@ static const Parameter s_params[] =
     { "curses", Parameter::PT_MULTI, "dce_smb | dce_udp | dce_tcp | sslv2", nullptr,
       "enable service identification based on internal algorithm" },
 
-    { "max_pattern", Parameter::PT_INT, "0:65535", "64",
-      "maximum scan depth per segment (0 is unlimited)" },
+    { "max_search_depth", Parameter::PT_INT, "0:65535", "64",
+      "maximum scan depth per flow" },
 
     { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
 };
@@ -159,8 +159,8 @@ bool WizardModule::set(const char*, Value& v, SnortConfig*)
     else if ( v.is("curses") )
         curses->add_curse(v.get_string());
 
-    else if ( v.is("max_pattern") )
-        max_pattern = v.get_uint16();
+    else if ( v.is("max_search_depth") )
+        max_search_depth = v.get_uint16();
 
     return true;
 }

diff --git a/src/service_inspectors/wizard/wiz_module.h b/src/service_inspectors/wizard/wiz_module.h
index 5d3dc4d0df921751647299885b58810b0991d589..485db1fc4cc6cad582118a33d54f5992d8526a82 100644 (file)
--- a/src/service_inspectors/wizard/wiz_module.h
+++ b/src/service_inspectors/wizard/wiz_module.h
@@ -58,8 +58,8 @@ public:
     MagicBook* get_book(bool c2s, bool hex);
     CurseBook* get_curse_book();
 
-    uint16_t get_max_pattern()
-    { return max_pattern; }
+    uint16_t get_max_search_depth()
+    { return max_search_depth; }
 
     Usage get_usage() const override
     { return INSPECT; }
@@ -83,7 +83,7 @@ private:
     MagicBook* s2c_spells = nullptr;
 
     CurseBook* curses = nullptr;
-    uint16_t max_pattern = 0;
+    uint16_t max_search_depth = 0;
 };
 
 #endif

diff --git a/src/service_inspectors/wizard/wizard.cc b/src/service_inspectors/wizard/wizard.cc
index 273af7b50d1ea550867adc5e246bf974b8a449df..16f02ea8e294aee2da51bad219b84aeb863bd2cf 100644 (file)
--- a/src/service_inspectors/wizard/wizard.cc
+++ b/src/service_inspectors/wizard/wizard.cc
@@ -124,6 +124,8 @@ private:
 private:
     Wizard* wizard;
     Wand wand;
+    uint16_t wizard_processed_bytes;
+    const MagicPage* bookmark;         // pointer to last glob
 };
 
 class Wizard : public Inspector
@@ -138,7 +140,7 @@ public:
 
     void reset(Wand&, bool tcp, bool c2s);
     bool finished(Wand&);
-    bool cast_spell(Wand&, Flow*, const uint8_t*, unsigned);
+    bool cast_spell(Wand&, Flow*, const uint8_t*, unsigned, uint16_t&);
     bool spellbind(const MagicPage*&, Flow*, const uint8_t*, unsigned);
     bool cursebind(const vector<CurseServiceTracker>&, Flow*, const uint8_t*, unsigned);
 
@@ -150,7 +152,8 @@ public:
     MagicBook* s2c_spells;
 
     CurseBook* curses;
-    uint16_t max_pattern;
+
+    uint16_t max_search_depth;
 };
 
 //-------------------------------------------------------------------------
@@ -160,7 +163,7 @@ public:
 //-------------------------------------------------------------------------
 
 MagicSplitter::MagicSplitter(bool c2s, class Wizard* w) :
-    StreamSplitter(c2s)
+    StreamSplitter(c2s), wizard_processed_bytes(0), bookmark(nullptr)
 {
     wizard = w;
     w->add_ref();
@@ -176,7 +179,6 @@ MagicSplitter::~MagicSplitter()
         delete wand.curse_tracker[i].tracker;
 }
 
-// FIXIT-M stop search on hit and failure (no possible match)
 StreamSplitter::Status MagicSplitter::scan(
     Packet* pkt, const uint8_t* data, uint32_t len,
     uint32_t, uint32_t*)
@@ -184,21 +186,32 @@ StreamSplitter::Status MagicSplitter::scan(
     Profile profile(wizPerfStats);
     count_scan(pkt->flow);
 
+    // setting last glob from current flow
+    if ( wand.spell )
+        wand.spell->book.set_bookmark(bookmark);
     bytes_scanned += len;
-    if ( wizard->cast_spell(wand, pkt->flow, data, len) )
+
+    if ( wizard->cast_spell(wand, pkt->flow, data, len, wizard_processed_bytes) )
     {
         trace_logf(wizard_trace, pkt, "%s streaming search found service %s\n",
             to_server() ? "c2s" : "s2c", pkt->flow->service);
         count_hit(pkt->flow);
+        wizard_processed_bytes = 0;
+        return STOP;
     }
 
     else if ( wizard->finished(wand) || bytes_scanned >= max(pkt->flow) )
     {
         count_miss(pkt->flow);
         trace_logf(wizard_trace, pkt, "%s streaming search abandoned\n", to_server() ? "c2s" : "s2c");
+        wizard_processed_bytes = 0;
         return ABORT;
     }
 
+    // saving new last glob from current flow
+    if ( wand.spell )
+        bookmark = wand.spell->book.get_bookmark();
+
     // ostensibly continue but splitter will be swapped out upon hit
     return SEARCH;
 }
@@ -216,7 +229,7 @@ Wizard::Wizard(WizardModule* m)
     s2c_spells = m->get_book(false, false);
 
     curses = m->get_curse_book();
-    max_pattern = m->get_max_pattern();
+    max_search_depth = m->get_max_search_depth();
 }
 
 Wizard::~Wizard()
@@ -236,11 +249,13 @@ void Wizard::reset(Wand& w, bool tcp, bool c2s)
     {
         w.hex = c2s_hexes->page1();
         w.spell = c2s_spells->page1();
+        c2s_spells->set_bookmark();
     }
     else
     {
         w.hex = s2c_hexes->page1();
         w.spell = s2c_spells->page1();
+        s2c_spells->set_bookmark();
     }
 
     if (w.curse_tracker.empty())
@@ -269,9 +284,10 @@ void Wizard::eval(Packet* p)
     bool c2s = p->is_from_client();
     Wand wand;
     reset(wand, false, c2s);
+    uint16_t udp_processed_bytes = 0;
 
     ++tstats.udp_scans;
-    if ( cast_spell(wand, p->flow, p->data, p->dsize) )
+    if ( cast_spell(wand, p->flow, p->data, p->dsize, udp_processed_bytes) )
     {
         trace_logf(wizard_trace, p, "%s datagram search found service %s\n",
             c2s ? "c2s" : "s2c", p->flow->service);
@@ -293,7 +309,7 @@ StreamSplitter* Wizard::get_splitter(bool c2s)
 bool Wizard::spellbind(
     const MagicPage*& m, Flow* f, const uint8_t* data, unsigned len)
 {
-    f->service = m->book.find_spell(data, len, max_pattern, m);
+    f->service = m->book.find_spell(data, len, m);
     return ( f->service != nullptr );
 }
 
@@ -314,17 +330,31 @@ bool Wizard::cursebind(const vector<CurseServiceTracker>& curse_tracker, Flow* f
 }
 
 bool Wizard::cast_spell(
-    Wand& w, Flow* f, const uint8_t* data, unsigned len)
+    Wand& w, Flow* f, const uint8_t* data, unsigned len, uint16_t& wizard_processed_bytes)
 {
+    auto curse_len = len;
+
+    len = std::min(len, static_cast<unsigned>(max_search_depth - wizard_processed_bytes));
+
+    wizard_processed_bytes += len;
+
     if ( w.hex && spellbind(w.hex, f, data, len) )
         return true;
 
     if ( w.spell && spellbind(w.spell, f, data, len) )
         return true;
 
-    if (cursebind(w.curse_tracker, f, data, len))
+    if (cursebind(w.curse_tracker, f, data, curse_len))
         return true;
 
+    // If we reach max value of wizard_processed_bytes, 
+    // but not assign any inspector - raise tcp_miss and stop
+    if ( !f->service && wizard_processed_bytes >= max_search_depth )
+    {
+        w.spell = nullptr;
+        w.hex = nullptr;
+    }
+
     return false;
 }
 

diff --git a/src/stream/paf.cc b/src/stream/paf.cc
index 0b0b9cd3dc98a3e30c738e17a8dde7b733a8ee42..7ecf63aad23014789ab9afa1a0f44a5631b6a29b 100644 (file)
--- a/src/stream/paf.cc
+++ b/src/stream/paf.cc
@@ -136,7 +136,7 @@ static bool paf_callback (
     ps->fpt = 0;
     ps->paf = ss->scan(pkt, data, len, flags, &ps->fpt);
 
-    if ( ps->paf == StreamSplitter::ABORT )
+    if ( ps->paf == StreamSplitter::ABORT || ps->paf == StreamSplitter::STOP )
         return false;
 
     if ( ps->paf != StreamSplitter::SEARCH )

diff --git a/src/stream/stream_splitter.h b/src/stream/stream_splitter.h
index 9d7bdb0505963c0aa5196ab30d44204465f5d932..89b934c2a194e2453a7f350c9e5ec8c7916cc008 100644 (file)
--- a/src/stream/stream_splitter.h
+++ b/src/stream/stream_splitter.h
@@ -42,13 +42,14 @@ public:
 
     enum Status
     {
-        ABORT,  // non-paf operation
-        START,  // internal use only
-        SEARCH, // searching for next flush point
-        FLUSH,  // flush at given offset
-        LIMIT,  // flush to given offset upon reaching paf_max
-        SKIP,   // skip ahead to given offset
-        LIMITED // previously did limit flush
+        ABORT,   // non-paf operation
+        START,   // internal use only
+        SEARCH,  // searching for next flush point
+        FLUSH,   // flush at given offset
+        LIMIT,   // flush to given offset upon reaching paf_max
+        SKIP,    // skip ahead to given offset
+        LIMITED, // previously did limit flush
+        STOP     // stop paf scan loop
     };
 
     // scan(), finish(), reassemble() are called in this order: