From: Mike Stepanek (mstepane) Date: Mon, 25 Oct 2021 16:11:11 +0000 (+0000) Subject: Merge pull request #3079 in SNORT/snort3 from ~YVELYKOZ/snort3:glob_several_packets... X-Git-Tag: 3.1.16.0~19 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f78f91690d114318dd96aaef4736839bb0ff5dca;p=thirdparty%2Fsnort3.git Merge pull request #3079 in SNORT/snort3 from ~YVELYKOZ/snort3:glob_several_packets to master Squashed commit of the following: commit b768e09bc0b09ea3aac32b88eaf3b53c2e035e39 Author: Yehor Velykozhon Date: Mon Sep 13 20:34:15 2021 +0300 wizard: update globbing and max_pattern In order to support globbing over several packets, was added state-variable that contain middle state of pattern. Max_pattern now applying per flow instead of pre segment. Max_pattern was renamed to max_search_depth. Fixed bug with reentering wizard after tcp_hits. --- diff --git a/src/service_inspectors/wizard/hexes.cc b/src/service_inspectors/wizard/hexes.cc index a83448278..1a03ddce0 100644 --- a/src/service_inspectors/wizard/hexes.cc +++ b/src/service_inspectors/wizard/hexes.cc @@ -162,18 +162,3 @@ const MagicPage* HexBook::find_spell( } return p; } - -const char* HexBook::find_spell( - const uint8_t* data, unsigned len, unsigned max, const MagicPage*& p) const -{ - if ( max and len > max ) - len = max; - - p = find_spell(data, len, p, 0); - - if ( p and !p->value.empty() ) - return p->value.c_str(); - - return nullptr; -} - diff --git a/src/service_inspectors/wizard/magic.cc b/src/service_inspectors/wizard/magic.cc index ffd2567d5..ba530fa12 100644 --- a/src/service_inspectors/wizard/magic.cc +++ b/src/service_inspectors/wizard/magic.cc @@ -23,6 +23,8 @@ #include "magic.h" +#include + MagicPage::MagicPage(const MagicBook& b) : book(b) { for ( int i = 0; i < 256; ++i ) @@ -40,6 +42,18 @@ MagicPage::~MagicPage() delete any; } +const char* MagicBook::find_spell(const uint8_t* data, unsigned len, const MagicPage*& p) const +{ + assert(p); + + p = find_spell(data, len, p, 0); + + if ( p && !p->value.empty() ) + return p->value.c_str(); + + return nullptr; +} + MagicBook::MagicBook() { root = new MagicPage(*this); } diff --git a/src/service_inspectors/wizard/magic.h b/src/service_inspectors/wizard/magic.h index 8b7db0cc8..a34e5d3e5 100644 --- a/src/service_inspectors/wizard/magic.h +++ b/src/service_inspectors/wizard/magic.h @@ -52,15 +52,22 @@ public: MagicBook& operator=(const MagicBook&) = delete; virtual bool add_spell(const char* key, const char*& val) = 0; - virtual const char* find_spell(const uint8_t*, unsigned len, unsigned max, - const MagicPage*&) const = 0; + virtual const char* find_spell(const uint8_t*, unsigned len, const MagicPage*&) const; - const MagicPage* page1() + const MagicPage* page1() const { return root; } + virtual void set_bookmark(const MagicPage* page = nullptr) const + { (void)page; } + virtual const MagicPage* get_bookmark() const + { return nullptr; } + protected: MagicBook(); MagicPage* root; + + virtual const MagicPage* find_spell(const uint8_t*, unsigned, + const MagicPage*, unsigned) const = 0; }; //------------------------------------------------------------------------- @@ -74,13 +81,19 @@ public: SpellBook(); bool add_spell(const char*, const char*&) override; - const char* find_spell(const uint8_t*, unsigned len, unsigned max, - const MagicPage*&) const override; + + void set_bookmark(const MagicPage* page = nullptr) const override + { glob = page; } + + const MagicPage* get_bookmark() const override + { return glob; } private: bool translate(const char*, HexVector&); void add_spell(const char*, const char*, HexVector&, unsigned, MagicPage*); - const MagicPage* find_spell(const uint8_t*, unsigned, const MagicPage*, unsigned) const; + const MagicPage* find_spell(const uint8_t*, unsigned, const MagicPage*, unsigned) const override; + + mutable const MagicPage* glob; }; //------------------------------------------------------------------------- @@ -94,13 +107,11 @@ public: HexBook() = default; bool add_spell(const char*, const char*&) override; - const char* find_spell(const uint8_t*, unsigned len, unsigned, - const MagicPage*&) const override; private: bool translate(const char*, HexVector&); void add_spell(const char*, const char*, HexVector&, unsigned, MagicPage*); - const MagicPage* find_spell(const uint8_t*, unsigned, const MagicPage*, unsigned) const; + const MagicPage* find_spell(const uint8_t*, unsigned, const MagicPage*, unsigned) const override; }; #endif diff --git a/src/service_inspectors/wizard/spells.cc b/src/service_inspectors/wizard/spells.cc index 51719f79f..227ca4db7 100644 --- a/src/service_inspectors/wizard/spells.cc +++ b/src/service_inspectors/wizard/spells.cc @@ -29,7 +29,7 @@ using namespace std; #define WILD 0x100 -SpellBook::SpellBook() +SpellBook::SpellBook() : glob(nullptr) { // allows skipping leading whitespace only root->next[(int)' '] = root; @@ -152,29 +152,25 @@ const MagicPage* SpellBook::find_spell( while ( i < n ) { if ( const MagicPage* q = find_spell(s, n, p->any, i) ) + { + glob = q->any ? q : p; return q; + } ++i; } - return p->any; + return p; } + + // If no match but has glob, continue lookup from glob + if ( p->value.empty() && glob ) + { + p = glob; + glob = nullptr; + + return find_spell(s, n, p, i); + } + return p->value.empty() ? nullptr : p; } return p; } - -const char* SpellBook::find_spell( - const uint8_t* data, unsigned len, unsigned max, const MagicPage*& p) const -{ - assert(p); - - if ( max and len > max ) - len = max; - - p = find_spell(data, len, p, 0); - - if ( p and !p->value.empty() ) - return p->value.c_str(); - - return nullptr; -} - diff --git a/src/service_inspectors/wizard/wiz_module.cc b/src/service_inspectors/wizard/wiz_module.cc index 5e9cff769..c85a98c9f 100644 --- a/src/service_inspectors/wizard/wiz_module.cc +++ b/src/service_inspectors/wizard/wiz_module.cc @@ -106,8 +106,8 @@ static const Parameter s_params[] = { "curses", Parameter::PT_MULTI, "dce_smb | dce_udp | dce_tcp | sslv2", nullptr, "enable service identification based on internal algorithm" }, - { "max_pattern", Parameter::PT_INT, "0:65535", "64", - "maximum scan depth per segment (0 is unlimited)" }, + { "max_search_depth", Parameter::PT_INT, "0:65535", "64", + "maximum scan depth per flow" }, { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; @@ -159,8 +159,8 @@ bool WizardModule::set(const char*, Value& v, SnortConfig*) else if ( v.is("curses") ) curses->add_curse(v.get_string()); - else if ( v.is("max_pattern") ) - max_pattern = v.get_uint16(); + else if ( v.is("max_search_depth") ) + max_search_depth = v.get_uint16(); return true; } diff --git a/src/service_inspectors/wizard/wiz_module.h b/src/service_inspectors/wizard/wiz_module.h index 5d3dc4d0d..485db1fc4 100644 --- a/src/service_inspectors/wizard/wiz_module.h +++ b/src/service_inspectors/wizard/wiz_module.h @@ -58,8 +58,8 @@ public: MagicBook* get_book(bool c2s, bool hex); CurseBook* get_curse_book(); - uint16_t get_max_pattern() - { return max_pattern; } + uint16_t get_max_search_depth() + { return max_search_depth; } Usage get_usage() const override { return INSPECT; } @@ -83,7 +83,7 @@ private: MagicBook* s2c_spells = nullptr; CurseBook* curses = nullptr; - uint16_t max_pattern = 0; + uint16_t max_search_depth = 0; }; #endif diff --git a/src/service_inspectors/wizard/wizard.cc b/src/service_inspectors/wizard/wizard.cc index 273af7b50..16f02ea8e 100644 --- a/src/service_inspectors/wizard/wizard.cc +++ b/src/service_inspectors/wizard/wizard.cc @@ -124,6 +124,8 @@ private: private: Wizard* wizard; Wand wand; + uint16_t wizard_processed_bytes; + const MagicPage* bookmark; // pointer to last glob }; class Wizard : public Inspector @@ -138,7 +140,7 @@ public: void reset(Wand&, bool tcp, bool c2s); bool finished(Wand&); - bool cast_spell(Wand&, Flow*, const uint8_t*, unsigned); + bool cast_spell(Wand&, Flow*, const uint8_t*, unsigned, uint16_t&); bool spellbind(const MagicPage*&, Flow*, const uint8_t*, unsigned); bool cursebind(const vector&, Flow*, const uint8_t*, unsigned); @@ -150,7 +152,8 @@ public: MagicBook* s2c_spells; CurseBook* curses; - uint16_t max_pattern; + + uint16_t max_search_depth; }; //------------------------------------------------------------------------- @@ -160,7 +163,7 @@ public: //------------------------------------------------------------------------- MagicSplitter::MagicSplitter(bool c2s, class Wizard* w) : - StreamSplitter(c2s) + StreamSplitter(c2s), wizard_processed_bytes(0), bookmark(nullptr) { wizard = w; w->add_ref(); @@ -176,7 +179,6 @@ MagicSplitter::~MagicSplitter() delete wand.curse_tracker[i].tracker; } -// FIXIT-M stop search on hit and failure (no possible match) StreamSplitter::Status MagicSplitter::scan( Packet* pkt, const uint8_t* data, uint32_t len, uint32_t, uint32_t*) @@ -184,21 +186,32 @@ StreamSplitter::Status MagicSplitter::scan( Profile profile(wizPerfStats); count_scan(pkt->flow); + // setting last glob from current flow + if ( wand.spell ) + wand.spell->book.set_bookmark(bookmark); bytes_scanned += len; - if ( wizard->cast_spell(wand, pkt->flow, data, len) ) + + if ( wizard->cast_spell(wand, pkt->flow, data, len, wizard_processed_bytes) ) { trace_logf(wizard_trace, pkt, "%s streaming search found service %s\n", to_server() ? "c2s" : "s2c", pkt->flow->service); count_hit(pkt->flow); + wizard_processed_bytes = 0; + return STOP; } else if ( wizard->finished(wand) || bytes_scanned >= max(pkt->flow) ) { count_miss(pkt->flow); trace_logf(wizard_trace, pkt, "%s streaming search abandoned\n", to_server() ? "c2s" : "s2c"); + wizard_processed_bytes = 0; return ABORT; } + // saving new last glob from current flow + if ( wand.spell ) + bookmark = wand.spell->book.get_bookmark(); + // ostensibly continue but splitter will be swapped out upon hit return SEARCH; } @@ -216,7 +229,7 @@ Wizard::Wizard(WizardModule* m) s2c_spells = m->get_book(false, false); curses = m->get_curse_book(); - max_pattern = m->get_max_pattern(); + max_search_depth = m->get_max_search_depth(); } Wizard::~Wizard() @@ -236,11 +249,13 @@ void Wizard::reset(Wand& w, bool tcp, bool c2s) { w.hex = c2s_hexes->page1(); w.spell = c2s_spells->page1(); + c2s_spells->set_bookmark(); } else { w.hex = s2c_hexes->page1(); w.spell = s2c_spells->page1(); + s2c_spells->set_bookmark(); } if (w.curse_tracker.empty()) @@ -269,9 +284,10 @@ void Wizard::eval(Packet* p) bool c2s = p->is_from_client(); Wand wand; reset(wand, false, c2s); + uint16_t udp_processed_bytes = 0; ++tstats.udp_scans; - if ( cast_spell(wand, p->flow, p->data, p->dsize) ) + if ( cast_spell(wand, p->flow, p->data, p->dsize, udp_processed_bytes) ) { trace_logf(wizard_trace, p, "%s datagram search found service %s\n", c2s ? "c2s" : "s2c", p->flow->service); @@ -293,7 +309,7 @@ StreamSplitter* Wizard::get_splitter(bool c2s) bool Wizard::spellbind( const MagicPage*& m, Flow* f, const uint8_t* data, unsigned len) { - f->service = m->book.find_spell(data, len, max_pattern, m); + f->service = m->book.find_spell(data, len, m); return ( f->service != nullptr ); } @@ -314,17 +330,31 @@ bool Wizard::cursebind(const vector& curse_tracker, Flow* f } bool Wizard::cast_spell( - Wand& w, Flow* f, const uint8_t* data, unsigned len) + Wand& w, Flow* f, const uint8_t* data, unsigned len, uint16_t& wizard_processed_bytes) { + auto curse_len = len; + + len = std::min(len, static_cast(max_search_depth - wizard_processed_bytes)); + + wizard_processed_bytes += len; + if ( w.hex && spellbind(w.hex, f, data, len) ) return true; if ( w.spell && spellbind(w.spell, f, data, len) ) return true; - if (cursebind(w.curse_tracker, f, data, len)) + if (cursebind(w.curse_tracker, f, data, curse_len)) return true; + // If we reach max value of wizard_processed_bytes, + // but not assign any inspector - raise tcp_miss and stop + if ( !f->service && wizard_processed_bytes >= max_search_depth ) + { + w.spell = nullptr; + w.hex = nullptr; + } + return false; } diff --git a/src/stream/paf.cc b/src/stream/paf.cc index 0b0b9cd3d..7ecf63aad 100644 --- a/src/stream/paf.cc +++ b/src/stream/paf.cc @@ -136,7 +136,7 @@ static bool paf_callback ( ps->fpt = 0; ps->paf = ss->scan(pkt, data, len, flags, &ps->fpt); - if ( ps->paf == StreamSplitter::ABORT ) + if ( ps->paf == StreamSplitter::ABORT || ps->paf == StreamSplitter::STOP ) return false; if ( ps->paf != StreamSplitter::SEARCH ) diff --git a/src/stream/stream_splitter.h b/src/stream/stream_splitter.h index 9d7bdb050..89b934c2a 100644 --- a/src/stream/stream_splitter.h +++ b/src/stream/stream_splitter.h @@ -42,13 +42,14 @@ public: enum Status { - ABORT, // non-paf operation - START, // internal use only - SEARCH, // searching for next flush point - FLUSH, // flush at given offset - LIMIT, // flush to given offset upon reaching paf_max - SKIP, // skip ahead to given offset - LIMITED // previously did limit flush + ABORT, // non-paf operation + START, // internal use only + SEARCH, // searching for next flush point + FLUSH, // flush at given offset + LIMIT, // flush to given offset upon reaching paf_max + SKIP, // skip ahead to given offset + LIMITED, // previously did limit flush + STOP // stop paf scan loop }; // scan(), finish(), reassemble() are called in this order: