Pull request #3434: hyperscan: reallocate hyperscan scratch space when patterns are...

Merge in SNORT/snort3 from ~SATHIRKA/snort3:scratch_update to master

Squashed commit of the following:

commit d320d2fa108197eb0a5c78776a79b695dfe46ab7
Author: Sreeja Athirkandathil Narayanan <sathirka@cisco.com>
Date:   Wed May 11 14:38:03 2022 -0400

    hyperscan: reallocate hyperscan scratch space when patterns are reloaded during appid detector reload

diff --git a/src/helpers/hyper_scratch_allocator.h b/src/helpers/hyper_scratch_allocator.h
index 6c1eb7f67590a486b8e3af99739bb661995072ca..bd3227b46b5c81d105f17e8c07f4c57bb1ac8a7b 100644 (file)
--- a/src/helpers/hyper_scratch_allocator.h
+++ b/src/helpers/hyper_scratch_allocator.h
@@ -44,7 +44,8 @@ public:
 
     bool setup(SnortConfig*) override;
     void cleanup(SnortConfig*) override;
-
+    void update(SnortConfig*) override
+    { }
     bool allocate(hs_database_t*);
 
     hs_scratch_t* get()

diff --git a/src/helpers/scratch_allocator.h b/src/helpers/scratch_allocator.h
index d75fd1c699a4bd04ab0cfab7ae52a3ff868ebc76..e92b8d286109fe5f2365ca33819fe64dc78d171c 100644 (file)
--- a/src/helpers/scratch_allocator.h
+++ b/src/helpers/scratch_allocator.h
@@ -48,6 +48,7 @@ public:
 
     virtual bool setup(SnortConfig*) = 0;
     virtual void cleanup(SnortConfig*) = 0;
+    virtual void update(SnortConfig*) = 0;
 
     int get_id() { return id; }
 
@@ -60,11 +61,13 @@ private:
 
 typedef bool (* ScratchSetup)(SnortConfig*);
 typedef void (* ScratchCleanup)(SnortConfig*);
+typedef void (* ScratchUpdate)(SnortConfig*);
 
 class SO_PUBLIC SimpleScratchAllocator : public ScratchAllocator
 {
 public:
-    SimpleScratchAllocator(ScratchSetup fs, ScratchCleanup fc) : fsetup(fs), fcleanup(fc) { }
+    SimpleScratchAllocator(ScratchSetup fs, ScratchCleanup fc, ScratchUpdate fu = nullptr)
+        : fsetup(fs), fcleanup(fc), fupdate(fu) { }
 
     bool setup(SnortConfig* sc) override
     { return fsetup(sc); }
@@ -72,9 +75,16 @@ public:
     void cleanup(SnortConfig* sc) override
     { fcleanup(sc); }
 
+    void update(SnortConfig* sc) override
+    {
+        if (fupdate)
+            fupdate(sc);
+    }
+
 private:
     ScratchSetup fsetup;
     ScratchCleanup fcleanup;
+    ScratchUpdate fupdate;
 };
 
 }

diff --git a/src/main/analyzer_command.cc b/src/main/analyzer_command.cc
index 2f192e5dd6091483fce0fc7bb0dbaf46ad359c2a..a386a559d53b99929baefb9fa21011b029ea528f 100644 (file)
--- a/src/main/analyzer_command.cc
+++ b/src/main/analyzer_command.cc
@@ -233,6 +233,21 @@ ACDAQSwap::~ACDAQSwap()
     LogMessage("== daq module reload complete\n");
 }
 
+bool ACScratchUpdate::execute(Analyzer&, void**)
+{
+    for ( auto* s : handlers )
+    {
+        s->update(sc);
+    }
+    return true;
+}
+
+ACScratchUpdate::~ACScratchUpdate()
+{
+    LogMessage("== scratch update complete\n");
+    ReloadTracker::end(ctrlcon);
+}
+
 SFDAQInstance* AnalyzerCommand::get_daq_instance(Analyzer& analyzer)
 {
     return analyzer.get_daq_instance();

diff --git a/src/main/analyzer_command.h b/src/main/analyzer_command.h
index 41abd5dcf2e7d5aaff85f9ef6c8f4d61abe9fea2..6632a25eacd9bf6d8cb817804a631f4360e16233 100644 (file)
--- a/src/main/analyzer_command.h
+++ b/src/main/analyzer_command.h
@@ -31,6 +31,9 @@ class Swapper;
 
 namespace snort
 {
+class ScratchAllocator;
+struct SnortConfig;
+
 class SFDAQInstance;
 
 class AnalyzerCommand
@@ -185,6 +188,20 @@ public:
     ~ACDAQSwap() override;
 };
 
+class ACScratchUpdate : public snort::AnalyzerCommand
+{
+public:
+    ACScratchUpdate(snort::SnortConfig* sc, std::vector<snort::ScratchAllocator*>& handlers,
+        ControlConn* conn) : AnalyzerCommand(conn), sc(sc), handlers(handlers)
+    { }
+    bool execute(Analyzer&, void**) override;
+    const char* stringify() override { return "SCRATCH_UPDATE"; }
+    ~ACScratchUpdate() override;
+private:
+    snort::SnortConfig* sc;
+    std::vector<snort::ScratchAllocator*>& handlers;
+};
+
 namespace snort
 {
 // from main.cc

diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc
index 88f6aa0d47303c4661282467330938ec451e9a6a..595adc0374ceabbe7ca54aab8720b1e0fcaabe71 100644 (file)
--- a/src/main/snort_config.cc
+++ b/src/main/snort_config.cc
@@ -341,6 +341,11 @@ void SnortConfig::post_setup()
     }
 }
 
+void SnortConfig::update_scratch(ControlConn* ctrlcon)
+{
+    main_broadcast_command(new ACScratchUpdate(this, scratch_handlers, ctrlcon));
+}
+
 void SnortConfig::clone(const SnortConfig* const conf)
 {
     *this = *conf;

diff --git a/src/main/snort_config.h b/src/main/snort_config.h
index 3983440cb4dd75f44ce07eaab7bf02b6ce5119fc..e0e77f2d7936b6836576992560eec839b87da9b4 100644 (file)
--- a/src/main/snort_config.h
+++ b/src/main/snort_config.h
@@ -128,6 +128,7 @@ enum DumpConfigType
 };
 
 class ConfigOutput;
+class ControlConn;
 class FastPatternConfig;
 class RuleStateMap;
 class TraceConfig;
@@ -176,6 +177,7 @@ public:
 
     void setup();
     void post_setup();
+    void update_scratch(ControlConn*);
     bool verify() const;
 
     void merge(const SnortConfig*);

diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc
index 3e74e34f30ab410a27a62e67a1461473b5d6ad51..aba51255dd83c4ff8975fbb6fe62986ef0a671bb 100644 (file)
--- a/src/network_inspectors/appid/appid_module.cc
+++ b/src/network_inspectors/appid/appid_module.cc
@@ -267,7 +267,7 @@ ACOdpContextSwap::~ACOdpContextSwap()
             file_path = std::string(ctxt.config.app_detector_dir) + "/../userappid.conf";
         ctxt.get_odp_ctxt().get_app_info_mgr().dump_appid_configurations(file_path);
     }
-    ReloadTracker::end(ctrlcon);
+    SnortConfig::get_main_conf()->update_scratch(ctrlcon);
     log_message("== reload detectors complete\n");
 }
 

diff --git a/src/search_engines/hyperscan.cc b/src/search_engines/hyperscan.cc
index 0cc9f8d58eac8e802190d65cdf478d5d253d0ae5..e01e5d560cd7acc8a54e22de2b07d3365d70e1ee 100644 (file)
--- a/src/search_engines/hyperscan.cc
+++ b/src/search_engines/hyperscan.cc
@@ -411,7 +411,7 @@ static bool scratch_setup(SnortConfig* sc)
         hs_scratch_t** ss = (hs_scratch_t**) &sc->state[i][scratch_index];
         hs_clone_scratch(max, ss);
     }
-    hs_free_scratch(max);
+    s_scratch[get_instance_id()] = max;
     return true;
 }
 
@@ -425,17 +425,55 @@ static void scratch_cleanup(SnortConfig* sc)
     }
 }
 
+static bool need_update(SnortConfig* sc)
+{
+    if ( s_scratch.size() )
+    {
+        size_t max_sz, instance_sz;
+        hs_scratch_size(s_scratch[0], &max_sz);
+        hs_scratch_size((hs_scratch_t*)sc->state[get_instance_id()][scratch_index], &instance_sz);
+        if ( max_sz > instance_sz )
+            return true;
+        else
+            return false;
+    }
+    else
+        return false;
+}
+
+void static scratch_update(SnortConfig* sc)
+{
+    if ( !need_update(sc) )
+        return;
+
+    hs_scratch_t** ss = (hs_scratch_t**) &sc->state[get_instance_id()][scratch_index];
+    hs_free_scratch(*ss);
+    *ss = nullptr;
+    hs_clone_scratch(s_scratch[0], ss);
+}
+
 class HyperscanModule : public Module
 {
 public:
     HyperscanModule() : Module(s_name, s_help)
     {
-        scratcher = new SimpleScratchAllocator(scratch_setup, scratch_cleanup);
+        scratcher = new SimpleScratchAllocator(scratch_setup, scratch_cleanup, scratch_update);
         scratch_index = scratcher->get_id();
     }
 
     ~HyperscanModule() override
-    { delete scratcher; }
+    {
+        delete scratcher;
+
+        for ( auto& ss : s_scratch )
+        {
+             if ( ss )
+             {
+                 hs_free_scratch(ss);
+                 ss = nullptr;
+             }
+        }
+    }
 };
 
 //-------------------------------------------------------------------------

diff --git a/src/search_engines/test/hyperscan_test.cc b/src/search_engines/test/hyperscan_test.cc
index b3d3df47b06f66308df62f15cc5882abfd59f584..707a4b166abc6fd19f3fdc742b2082897ddfca4e 100644 (file)
--- a/src/search_engines/test/hyperscan_test.cc
+++ b/src/search_engines/test/hyperscan_test.cc
@@ -23,6 +23,7 @@
 #endif
 
 #include <cstring>
+#include <hs_runtime.h>
 
 #include "framework/base_api.h"
 #include "framework/counts.h"
@@ -297,6 +298,59 @@ TEST(mpse_hs_multi, single)
     CHECK(hits == 1);
 }
 
+//-------------------------------------------------------------------------
+// scratch update test
+//-------------------------------------------------------------------------
+
+TEST_GROUP(mpse_hs_scratch)
+{
+    Module* mod = nullptr;
+    Mpse* hs1 = nullptr;
+    Mpse* hs2 = nullptr;
+    const MpseApi* mpse_api = (const MpseApi*)se_hyperscan;
+
+    void setup() override
+    {
+        CHECK(se_hyperscan);
+        mod = mpse_api->base.mod_ctor();
+        hs1 = mpse_api->ctor(snort_conf, nullptr, nullptr);
+        hs2 = mpse_api->ctor(snort_conf, nullptr, nullptr);
+        CHECK(hs1);
+        CHECK(hs2);
+    }
+    void teardown() override
+    {
+        mpse_api->dtor(hs1);
+        mpse_api->dtor(hs2);
+        scratcher->cleanup(snort_conf);
+        mpse_api->base.mod_dtor(mod);
+    }
+};
+
+TEST(mpse_hs_scratch, scratch_update)
+{
+    Mpse::PatternDescriptor desc;
+    CHECK(hs1->add_pattern((const uint8_t*)"foo", 3, desc, s_user) == 0);
+    CHECK(hs1->prep_patterns(snort_conf) == 0);
+    CHECK(hs1->get_pattern_count() == 1);
+
+    scratcher->setup(snort_conf);
+
+    size_t instance_sz1;
+    hs_scratch_size((hs_scratch_t*)snort_conf->state[0][0], &instance_sz1);
+
+    for (unsigned i = 0; i < 30; i++)
+        CHECK(hs2->add_pattern((const uint8_t*)"bar", 3, desc, s_user) == 0);
+    CHECK(hs2->prep_patterns(snort_conf) == 0);
+    CHECK(hs2->get_pattern_count() == 30);
+
+    scratcher->update(snort_conf);
+
+    size_t instance_sz2;
+    hs_scratch_size((hs_scratch_t*)snort_conf->state[0][0], &instance_sz2);
+    CHECK(instance_sz2 > instance_sz1);
+}
+
 //-------------------------------------------------------------------------
 // main
 //-------------------------------------------------------------------------