From: Masud Hasan (mashasan) Date: Thu, 26 May 2022 18:37:05 +0000 (+0000) Subject: Pull request #3434: hyperscan: reallocate hyperscan scratch space when patterns are... X-Git-Tag: 3.1.31.0~10 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=26a8a30af80520368feff7e9814e5b12fb70da61;p=thirdparty%2Fsnort3.git Pull request #3434: hyperscan: reallocate hyperscan scratch space when patterns are reloaded during appid detector reload Merge in SNORT/snort3 from ~SATHIRKA/snort3:scratch_update to master Squashed commit of the following: commit d320d2fa108197eb0a5c78776a79b695dfe46ab7 Author: Sreeja Athirkandathil Narayanan Date: Wed May 11 14:38:03 2022 -0400 hyperscan: reallocate hyperscan scratch space when patterns are reloaded during appid detector reload --- diff --git a/src/helpers/hyper_scratch_allocator.h b/src/helpers/hyper_scratch_allocator.h index 6c1eb7f67..bd3227b46 100644 --- a/src/helpers/hyper_scratch_allocator.h +++ b/src/helpers/hyper_scratch_allocator.h @@ -44,7 +44,8 @@ public: bool setup(SnortConfig*) override; void cleanup(SnortConfig*) override; - + void update(SnortConfig*) override + { } bool allocate(hs_database_t*); hs_scratch_t* get() diff --git a/src/helpers/scratch_allocator.h b/src/helpers/scratch_allocator.h index d75fd1c69..e92b8d286 100644 --- a/src/helpers/scratch_allocator.h +++ b/src/helpers/scratch_allocator.h @@ -48,6 +48,7 @@ public: virtual bool setup(SnortConfig*) = 0; virtual void cleanup(SnortConfig*) = 0; + virtual void update(SnortConfig*) = 0; int get_id() { return id; } @@ -60,11 +61,13 @@ private: typedef bool (* ScratchSetup)(SnortConfig*); typedef void (* ScratchCleanup)(SnortConfig*); +typedef void (* ScratchUpdate)(SnortConfig*); class SO_PUBLIC SimpleScratchAllocator : public ScratchAllocator { public: - SimpleScratchAllocator(ScratchSetup fs, ScratchCleanup fc) : fsetup(fs), fcleanup(fc) { } + SimpleScratchAllocator(ScratchSetup fs, ScratchCleanup fc, ScratchUpdate fu = nullptr) + : fsetup(fs), fcleanup(fc), fupdate(fu) { } bool setup(SnortConfig* sc) override { return fsetup(sc); } @@ -72,9 +75,16 @@ public: void cleanup(SnortConfig* sc) override { fcleanup(sc); } + void update(SnortConfig* sc) override + { + if (fupdate) + fupdate(sc); + } + private: ScratchSetup fsetup; ScratchCleanup fcleanup; + ScratchUpdate fupdate; }; } diff --git a/src/main/analyzer_command.cc b/src/main/analyzer_command.cc index 2f192e5dd..a386a559d 100644 --- a/src/main/analyzer_command.cc +++ b/src/main/analyzer_command.cc @@ -233,6 +233,21 @@ ACDAQSwap::~ACDAQSwap() LogMessage("== daq module reload complete\n"); } +bool ACScratchUpdate::execute(Analyzer&, void**) +{ + for ( auto* s : handlers ) + { + s->update(sc); + } + return true; +} + +ACScratchUpdate::~ACScratchUpdate() +{ + LogMessage("== scratch update complete\n"); + ReloadTracker::end(ctrlcon); +} + SFDAQInstance* AnalyzerCommand::get_daq_instance(Analyzer& analyzer) { return analyzer.get_daq_instance(); diff --git a/src/main/analyzer_command.h b/src/main/analyzer_command.h index 41abd5dcf..6632a25ea 100644 --- a/src/main/analyzer_command.h +++ b/src/main/analyzer_command.h @@ -31,6 +31,9 @@ class Swapper; namespace snort { +class ScratchAllocator; +struct SnortConfig; + class SFDAQInstance; class AnalyzerCommand @@ -185,6 +188,20 @@ public: ~ACDAQSwap() override; }; +class ACScratchUpdate : public snort::AnalyzerCommand +{ +public: + ACScratchUpdate(snort::SnortConfig* sc, std::vector& handlers, + ControlConn* conn) : AnalyzerCommand(conn), sc(sc), handlers(handlers) + { } + bool execute(Analyzer&, void**) override; + const char* stringify() override { return "SCRATCH_UPDATE"; } + ~ACScratchUpdate() override; +private: + snort::SnortConfig* sc; + std::vector& handlers; +}; + namespace snort { // from main.cc diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc index 88f6aa0d4..595adc037 100644 --- a/src/main/snort_config.cc +++ b/src/main/snort_config.cc @@ -341,6 +341,11 @@ void SnortConfig::post_setup() } } +void SnortConfig::update_scratch(ControlConn* ctrlcon) +{ + main_broadcast_command(new ACScratchUpdate(this, scratch_handlers, ctrlcon)); +} + void SnortConfig::clone(const SnortConfig* const conf) { *this = *conf; diff --git a/src/main/snort_config.h b/src/main/snort_config.h index 3983440cb..e0e77f2d7 100644 --- a/src/main/snort_config.h +++ b/src/main/snort_config.h @@ -128,6 +128,7 @@ enum DumpConfigType }; class ConfigOutput; +class ControlConn; class FastPatternConfig; class RuleStateMap; class TraceConfig; @@ -176,6 +177,7 @@ public: void setup(); void post_setup(); + void update_scratch(ControlConn*); bool verify() const; void merge(const SnortConfig*); diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index 3e74e34f3..aba51255d 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -267,7 +267,7 @@ ACOdpContextSwap::~ACOdpContextSwap() file_path = std::string(ctxt.config.app_detector_dir) + "/../userappid.conf"; ctxt.get_odp_ctxt().get_app_info_mgr().dump_appid_configurations(file_path); } - ReloadTracker::end(ctrlcon); + SnortConfig::get_main_conf()->update_scratch(ctrlcon); log_message("== reload detectors complete\n"); } diff --git a/src/search_engines/hyperscan.cc b/src/search_engines/hyperscan.cc index 0cc9f8d58..e01e5d560 100644 --- a/src/search_engines/hyperscan.cc +++ b/src/search_engines/hyperscan.cc @@ -411,7 +411,7 @@ static bool scratch_setup(SnortConfig* sc) hs_scratch_t** ss = (hs_scratch_t**) &sc->state[i][scratch_index]; hs_clone_scratch(max, ss); } - hs_free_scratch(max); + s_scratch[get_instance_id()] = max; return true; } @@ -425,17 +425,55 @@ static void scratch_cleanup(SnortConfig* sc) } } +static bool need_update(SnortConfig* sc) +{ + if ( s_scratch.size() ) + { + size_t max_sz, instance_sz; + hs_scratch_size(s_scratch[0], &max_sz); + hs_scratch_size((hs_scratch_t*)sc->state[get_instance_id()][scratch_index], &instance_sz); + if ( max_sz > instance_sz ) + return true; + else + return false; + } + else + return false; +} + +void static scratch_update(SnortConfig* sc) +{ + if ( !need_update(sc) ) + return; + + hs_scratch_t** ss = (hs_scratch_t**) &sc->state[get_instance_id()][scratch_index]; + hs_free_scratch(*ss); + *ss = nullptr; + hs_clone_scratch(s_scratch[0], ss); +} + class HyperscanModule : public Module { public: HyperscanModule() : Module(s_name, s_help) { - scratcher = new SimpleScratchAllocator(scratch_setup, scratch_cleanup); + scratcher = new SimpleScratchAllocator(scratch_setup, scratch_cleanup, scratch_update); scratch_index = scratcher->get_id(); } ~HyperscanModule() override - { delete scratcher; } + { + delete scratcher; + + for ( auto& ss : s_scratch ) + { + if ( ss ) + { + hs_free_scratch(ss); + ss = nullptr; + } + } + } }; //------------------------------------------------------------------------- diff --git a/src/search_engines/test/hyperscan_test.cc b/src/search_engines/test/hyperscan_test.cc index b3d3df47b..707a4b166 100644 --- a/src/search_engines/test/hyperscan_test.cc +++ b/src/search_engines/test/hyperscan_test.cc @@ -23,6 +23,7 @@ #endif #include +#include #include "framework/base_api.h" #include "framework/counts.h" @@ -297,6 +298,59 @@ TEST(mpse_hs_multi, single) CHECK(hits == 1); } +//------------------------------------------------------------------------- +// scratch update test +//------------------------------------------------------------------------- + +TEST_GROUP(mpse_hs_scratch) +{ + Module* mod = nullptr; + Mpse* hs1 = nullptr; + Mpse* hs2 = nullptr; + const MpseApi* mpse_api = (const MpseApi*)se_hyperscan; + + void setup() override + { + CHECK(se_hyperscan); + mod = mpse_api->base.mod_ctor(); + hs1 = mpse_api->ctor(snort_conf, nullptr, nullptr); + hs2 = mpse_api->ctor(snort_conf, nullptr, nullptr); + CHECK(hs1); + CHECK(hs2); + } + void teardown() override + { + mpse_api->dtor(hs1); + mpse_api->dtor(hs2); + scratcher->cleanup(snort_conf); + mpse_api->base.mod_dtor(mod); + } +}; + +TEST(mpse_hs_scratch, scratch_update) +{ + Mpse::PatternDescriptor desc; + CHECK(hs1->add_pattern((const uint8_t*)"foo", 3, desc, s_user) == 0); + CHECK(hs1->prep_patterns(snort_conf) == 0); + CHECK(hs1->get_pattern_count() == 1); + + scratcher->setup(snort_conf); + + size_t instance_sz1; + hs_scratch_size((hs_scratch_t*)snort_conf->state[0][0], &instance_sz1); + + for (unsigned i = 0; i < 30; i++) + CHECK(hs2->add_pattern((const uint8_t*)"bar", 3, desc, s_user) == 0); + CHECK(hs2->prep_patterns(snort_conf) == 0); + CHECK(hs2->get_pattern_count() == 30); + + scratcher->update(snort_conf); + + size_t instance_sz2; + hs_scratch_size((hs_scratch_t*)snort_conf->state[0][0], &instance_sz2); + CHECK(instance_sz2 > instance_sz1); +} + //------------------------------------------------------------------------- // main //-------------------------------------------------------------------------