Pull request #4095: Perf accumulate each thread data

Merge in SNORT/snort3 from ~AKAYAMBU/snort3:perf_accumulate_each_thread_data to master

Squashed commit of the following:

commit bc4dfd6566df0b9276141b33529c9c2e6d619231
Author: Arunkumar Kayambu <akayambu@cisco.com>
Date:   Fri Oct 27 11:19:09 2023 -0400

    stats: perf improvement changes

diff --git a/src/codecs/misc/test/geneve_codec_test.cc b/src/codecs/misc/test/geneve_codec_test.cc
index 0c865c91255d4a17e262f05cf6e97facd690f903..7082dfe73702b5349d81ce14d94b43c0003c1c39 100644 (file)
--- a/src/codecs/misc/test/geneve_codec_test.cc
+++ b/src/codecs/misc/test/geneve_codec_test.cc
@@ -20,6 +20,7 @@
 #include "../cd_geneve.cc"
 
 #include "utils/endian.h"
+#include "main/thread_config.h"
 
 #include <CppUTest/CommandLineTestRunner.h>
 #include <CppUTest/TestHarness.h>
@@ -33,6 +34,9 @@ namespace snort
     bool TextLog_Print(TextLog* const, const char*, ...) { return false; }
     void Codec::codec_event(const CodecData&, CodecSid) { }
     bool SnortConfig::tunnel_bypass_enabled(unsigned short) const { return false; }
+    unsigned get_instance_id()
+    { return 0; }
+    unsigned ThreadConfig::get_instance_max() { return 1; }
 }
 
 // Geneve data with 2 variable options.

diff --git a/src/connectors/file_connector/test/file_connector_module_test.cc b/src/connectors/file_connector/test/file_connector_module_test.cc
index 6c7bd84d443dc6b46b224cead0346e93deffff43..63c3de64b046ea050fa57e78284fc84af8377d79 100644 (file)
--- a/src/connectors/file_connector/test/file_connector_module_test.cc
+++ b/src/connectors/file_connector/test/file_connector_module_test.cc
@@ -24,6 +24,7 @@
 #endif
 
 #include "connectors/file_connector/file_connector_module.h"
+#include "main/thread_config.h"
 #include "profiler/profiler.h"
 
 #include <CppUTest/CommandLineTestRunner.h>
@@ -31,6 +32,7 @@
 
 using namespace snort;
 
+
 THREAD_LOCAL SimpleStats file_connector_stats;
 THREAD_LOCAL ProfileStats file_connector_perfstats;
 
@@ -41,6 +43,9 @@ namespace snort
 {
 char* snort_strdup(const char* s)
 { return strdup(s); }
+unsigned get_instance_id()
+{ return 0; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 }
 
 TEST_GROUP(file_connector_module)

diff --git a/src/connectors/file_connector/test/file_connector_test.cc b/src/connectors/file_connector/test/file_connector_test.cc
index a5a690605a1393b124ed52bcb6d077f5064aaf46..5c4a3ce5ef04bcaf2cee978ce0a0cdc8ca34cf69 100644 (file)
--- a/src/connectors/file_connector/test/file_connector_test.cc
+++ b/src/connectors/file_connector/test/file_connector_test.cc
@@ -25,12 +25,14 @@
 
 #include "connectors/file_connector/file_connector.h"
 #include "connectors/file_connector/file_connector_module.h"
+#include "main/thread_config.h"
 
 #include <CppUTest/CommandLineTestRunner.h>
 #include <CppUTest/TestHarness.h>
 
 using namespace snort;
 
+
 extern const BaseApi* file_connector;
 const ConnectorApi* fc_api = nullptr;
 
@@ -55,6 +57,9 @@ namespace snort
 {
 const char* get_instance_file(std::string& file, const char* name)
 { file += name; return nullptr; }
+unsigned get_instance_id()
+{ return 0; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 }
 
 FileConnectorModule::FileConnectorModule() :

diff --git a/src/connectors/tcp_connector/test/tcp_connector_module_test.cc b/src/connectors/tcp_connector/test/tcp_connector_module_test.cc
index 1c6cf684c5ad112757d779085d327149376e5706..e944cbd4793039f21237a73040862647f56c42db 100644 (file)
--- a/src/connectors/tcp_connector/test/tcp_connector_module_test.cc
+++ b/src/connectors/tcp_connector/test/tcp_connector_module_test.cc
@@ -24,6 +24,7 @@
 #endif
 
 #include "connectors/tcp_connector/tcp_connector_module.h"
+#include "main/thread_config.h"
 #include "profiler/profiler.h"
 
 #include <CppUTest/CommandLineTestRunner.h>
@@ -31,6 +32,7 @@
 
 using namespace snort;
 
+
 THREAD_LOCAL SimpleStats tcp_connector_stats;
 THREAD_LOCAL ProfileStats tcp_connector_perfstats;
 
@@ -41,6 +43,10 @@ namespace snort
 {
 char* snort_strdup(const char* s)
 { return strdup(s); }
+
+unsigned get_instance_id()
+{ return 0; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 }
 
 TEST_GROUP(tcp_connector_module)

diff --git a/src/connectors/tcp_connector/test/tcp_connector_test.cc b/src/connectors/tcp_connector/test/tcp_connector_test.cc
index edfc627c0e89478372c8b0917243ffdd13fe53b7..0a1798e9d85f84544f6ffdbf30ba26e12551c361 100644 (file)
--- a/src/connectors/tcp_connector/test/tcp_connector_test.cc
+++ b/src/connectors/tcp_connector/test/tcp_connector_test.cc
@@ -31,6 +31,8 @@
 #include <sys/socket.h>
 #include <unistd.h>
 
+#include "main/thread_config.h"
+
 #include <CppUTest/CommandLineTestRunner.h>
 #include <CppUTest/TestHarness.h>
 
@@ -72,6 +74,7 @@ namespace snort
 {
 unsigned get_instance_id()
 { return s_instance; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 
 void ErrorMessage(const char*, ...) { }
 void LogMessage(const char*, ...) { }

diff --git a/src/detection/regex_offload.cc b/src/detection/regex_offload.cc
index 69497b079b2dbe04c7c45094cb0a57329990ce1c..9ec335928e075ade1f5c7bcba1e4b34a8a2ae22e 100644 (file)
--- a/src/detection/regex_offload.cc
+++ b/src/detection/regex_offload.cc
@@ -182,7 +182,11 @@ ThreadRegexOffload::ThreadRegexOffload(unsigned max) : RegexOffload(max)
     const SnortConfig* sc = SnortConfig::get_conf();
 
     for ( auto* req : idle )
+    {
+        ModuleManager::add_thread_stats_entry("search_engine");
+        ModuleManager::add_thread_stats_entry("detection");
         req->thread = new std::thread(worker, req, sc, i++);
+    }
 }
 
 ThreadRegexOffload::~ThreadRegexOffload()

diff --git a/src/flow/test/flow_cache_test.cc b/src/flow/test/flow_cache_test.cc
index 3d39419eb2727add506011fa37459736c6150005..85cdf800d0b74b26bd9538ce3c3ac7eb26d06699 100644 (file)
--- a/src/flow/test/flow_cache_test.cc
+++ b/src/flow/test/flow_cache_test.cc
@@ -32,6 +32,7 @@
 #include "flow/ha.h"
 #include "flow/session.h"
 #include "main/analyzer.h"
+#include "main/thread_config.h"
 #include "managers/inspector_manager.h"
 #include "packet_io/active.h"
 #include "protocols/icmp4.h"

diff --git a/src/framework/module.cc b/src/framework/module.cc
index f95f5de61d34fe15dee83b01baed5cc295255c04..ed0af5b53747a2986e0276828cfad71b79bb67d4 100644 (file)
--- a/src/framework/module.cc
+++ b/src/framework/module.cc
@@ -22,6 +22,7 @@
 #endif
 
 #include "module.h"
+#include "main/thread_config.h"
 
 #include "trace/trace.h"
 
@@ -77,18 +78,55 @@ void Module::clear_global_active_counters()
     const PegInfo* q = get_pegs();
 
     assert(q);
+    for ( unsigned thread_index = 0; thread_index < dump_stats_counts.size(); thread_index++)
+    {
+        for ( int i = 0; i < num_counts; i++ )
+        {
+            if ( q[i].type == CountType::NOW )
+            {
+                dump_stats_counts[thread_index][i] = 0;
+                counts[thread_index][i] = 0;
+                dump_stats_results[i] = 0;
+            }
+        }
+    }
+}
 
-    for ( int i = 0; i < num_counts; i++ )
+void Module::main_accumulate_stats()
+{
+    const PegInfo* q = get_pegs();
+
+    if ( !q )
+        return;
+
+    if (!global_stats())
     {
-        if ( q[i].type == CountType::NOW )
-            counts[i] = 0;
+        for ( int i = 0; i < num_counts; i++ )
+        {
+            for ( unsigned thread_index = 0; thread_index < dump_stats_counts.size(); thread_index++)
+            {
+                if ( q[i].type == CountType::SUM || q[i].type == CountType::NOW)
+                {
+                    dump_stats_results[i] += dump_stats_counts[thread_index][i];
+                }
+                else if ( q[i].type == CountType::MAX)
+                {
+                    if (dump_stats_counts[thread_index][i] > dump_stats_results[i])
+                        dump_stats_results[i] = dump_stats_counts[thread_index][i];
+                }    
+            }
+        }
     }
 }
 
+
 void Module::sum_stats(bool dump_stats)
 {
     if ( num_counts < 0 )
+    {
+        init_stats();
         reset_stats();
+    }
 
     PegCount* p = get_counts();
     const PegInfo* q = get_pegs();
@@ -97,17 +135,25 @@ void Module::sum_stats(bool dump_stats)
         return;
 
     assert(q);
-    if(dump_stats && !dump_stats_initialized)
+
+    unsigned instance_id = get_instance_id();
+    if (dump_stats and !dump_stats_initialized[instance_id])
     {
-        for (unsigned long i=0; i<counts.size(); i++)
-            dump_stats_counts[i] = counts[i];
-        dump_stats_initialized = true;
+        dump_stats_counts[instance_id] = counts[instance_id];
+        dump_stats_initialized[instance_id] = 1;
     }
 
     if ( global_stats() )
     {
         for ( int i = 0; i < num_counts; i++ )
-            set_peg_count(i, p[i], dump_stats);
+        {
+            // we need each thread to update the same variable since it is global stats.
+            // we need the latest info updated by the last thread
+            if (dump_stats)
+                dump_stats_results[i] = p[i];
+            else
+                counts[0][i] = p[i];
+        }
     }
     else
     {
@@ -120,12 +166,12 @@ void Module::sum_stats(bool dump_stats)
 
             case CountType::SUM:
                 add_peg_count(i, p[i], dump_stats);
-                if(!dump_stats)
+                if (!dump_stats)
                     p[i] = 0;
                 break;
 
             case CountType::NOW:
-                if ( dump_stats )
+                if (dump_stats)
                     add_peg_count(i, p[i], dump_stats);
                 break;
 
@@ -147,18 +193,14 @@ void Module::show_stats()
 {
     if ( num_counts > 0 )
     {
-        ::show_stats(&dump_stats_counts[0], get_pegs(), num_counts, get_name());
-        dump_stats_initialized = false;
+        ::show_stats(&dump_stats_results[0], get_pegs(), num_counts, get_name());
+        dump_stats_initialized.assign(dump_stats_initialized.size(), 0);
+        std::fill(dump_stats_results.begin(), dump_stats_results.end(), 0);
     }
 }
 
-void Module::reset_stats()
+void Module::init_stats(bool new_thread)
 {
-    PegCount* p = get_counts();
-
-    if ( !p )
-        return;
-
     const PegInfo* pegs = get_pegs();
 
     if ( !pegs )
@@ -170,15 +212,39 @@ void Module::reset_stats()
 
         while ( pegs[num_counts].name )
             ++num_counts;
+    }
+
+    unsigned number_of_threads = new_thread ? 1 : ThreadConfig::get_instance_max();
 
-        counts.resize(num_counts);
-        dump_stats_counts.resize(num_counts);
+    for ( unsigned thread_index = 0; thread_index < number_of_threads; thread_index++)
+    {
+        std::vector<PegCount> stats(num_counts);
+        std::vector<PegCount> dump_stats(num_counts);
+        counts.push_back(stats);
+        dump_stats_counts.push_back(dump_stats);
+        dump_stats_initialized.push_back(0);
     }
+    dump_stats_results.resize(num_counts);
+}
+
+
+void Module::reset_stats()
+{
+    PegCount* p = get_counts();
+
+    if ( !p )
+        return;
+
+    const PegInfo* pegs = get_pegs();
+
+    if ( !pegs )
+        return;
+
 
     for ( int i = 0; i < num_counts; i++ )
     {
-        counts[i] = 0;
-        dump_stats_counts[i] = 0;
+        counts[get_instance_id()][i] = 0;
+        dump_stats_counts[get_instance_id()][i] = 0;
 
         if ( pegs[i].type != CountType::NOW )
             p[i] = 0;
@@ -193,7 +259,7 @@ PegCount Module::get_global_count(const char* name) const
     for ( unsigned i = 0; infos[i].name; i++ )
     {
         if ( strcmp(name, infos[i].name) == 0 )
-            return counts[i];
+            return dump_stats_results[i];
     }
     assert(false); // wrong name = programmer error
     return 0;

diff --git a/src/framework/module.h b/src/framework/module.h
index 390b5f8b69809a37663698fd1cfe46f57961cfd8..24b87b537fa80dac9b4280c952469fc0bced83be 100644 (file)
--- a/src/framework/module.h
+++ b/src/framework/module.h
@@ -175,6 +175,8 @@ public:
     virtual void show_interval_stats(IndexVec&, FILE*);
     virtual void show_stats();
     virtual void reset_stats();
+    virtual void init_stats(bool new_thread=false);
+    virtual void main_accumulate_stats();
     virtual void show_dynamic_stats() {}
     void clear_global_active_counters();
 
@@ -203,35 +205,50 @@ protected:
 
     void set_params(const Parameter* p)
     { params = p; }
-
-    bool dump_stats_initialized = false;
-
-    std::vector<PegCount> counts;
-    std::vector<PegCount> dump_stats_counts;
+    std::vector<unsigned> dump_stats_initialized;
+    std::vector<std::vector<PegCount>> counts;
+    std::vector<std::vector<PegCount>> dump_stats_counts;
+    std::vector<PegCount> dump_stats_results;
     int num_counts = -1;
 
-    void set_peg_count(int index, PegCount value, bool dump_stats = false)
+    void set_peg_count(int index, PegCount value, bool dump_stats)
+    {
+        assert(index < num_counts);
+        if (dump_stats)
+            dump_stats_counts[get_instance_id()][index] = value;
+        else
+            counts[get_instance_id()][index] = value;
+    }
+
+    void set_max_peg_count(int index, PegCount value, bool dump_stats)
     {
         assert(index < num_counts);
-        if(dump_stats)
-            dump_stats_counts[index] = value;
+        if (dump_stats)
+        {
+            if(value > dump_stats_counts[get_instance_id()][index])
+                dump_stats_counts[get_instance_id()][index] = value;
+        }
         else
-            counts[index] = value;
+        {
+            if(value > counts[get_instance_id()][index])
+                counts[get_instance_id()][index] = value;
+        }
     }
 
-    void add_peg_count(int index, PegCount value, bool dump_stats = false)
+    void add_peg_count(int index, PegCount value, bool dump_stats)
     {
         assert(index < num_counts);
-        if(dump_stats)
-            dump_stats_counts[index] += value;
+        if (dump_stats)
+            dump_stats_counts[get_instance_id()][index] += value;
         else
-            counts[index] += value;
+            counts[get_instance_id()][index] += value;
     }
 
 private:
     friend ModuleManager;
     void init(const char*, const char* = nullptr);
 
+
     const char* name;
     const char* help;
 
@@ -239,20 +256,6 @@ private:
     bool list;
     int table_level = 0;
 
-    void set_max_peg_count(int index, PegCount value, bool dump_stats = false)
-    {
-        assert(index < num_counts);
-        if(dump_stats)
-        {
-            if(value > dump_stats_counts[index])
-                dump_stats_counts[index] = value;
-        }
-        else
-        {
-            if(value > counts[index])
-                counts[index] = value;
-        }
-    }
 };
 }
 #endif

diff --git a/src/host_tracker/host_cache_module.cc b/src/host_tracker/host_cache_module.cc
index 8ed5adb82662a2b2e693319a19fc1653c94b4bd5..9612d28dfb6f3721b02a382cad62900ebffb69dc 100644 (file)
--- a/src/host_tracker/host_cache_module.cc
+++ b/src/host_tracker/host_cache_module.cc
@@ -580,7 +580,6 @@ void HostCacheModule::prep_counts(bool)
 void HostCacheModule::sum_stats(bool dump_stats)
 {
     Module::sum_stats(dump_stats);
-    host_cache.sum_stats(dump_stats);
 }
 
 void HostCacheModule::reset_stats()

diff --git a/src/host_tracker/host_cache_segmented.h b/src/host_tracker/host_cache_segmented.h
index 3f0d9fdb54fd41574c37870405901857550ac34e..01e6601098ae0a75ae217b9dfa94a94d84b3781b 100644 (file)
--- a/src/host_tracker/host_cache_segmented.h
+++ b/src/host_tracker/host_cache_segmented.h
@@ -65,7 +65,6 @@ public:
     void invalidate();
     void update_counts();
     void reset_counts();
-    void sum_stats(bool dump_stats);
 
     std::shared_ptr<Value> operator[](const Key& key);
 
@@ -278,27 +277,6 @@ void HostCacheSegmented<Key, Value>::reset_counts()
     }
 }
 
-template<typename Key, typename Value>
-void HostCacheSegmented<Key, Value>::sum_stats(bool dump_stats)
-{
-    std::lock_guard<std::mutex> guard(stats_lock);
-    if ( !dump_stats )
-    {
-        const PegInfo* pegs = get_pegs();
-
-        for (auto cache : seg_list)
-        {
-            PegCount* cache_counts = reinterpret_cast<PegCount*> (&cache->stats);
-            cache->lock();
-            for (int i = 0; pegs[i].type != CountType::END; i++)
-            {
-                if ( pegs[i].type == CountType::SUM )
-                    cache_counts[i] = 0;
-            }
-            cache->unlock();
-        }
-    }
-}
 
 template<typename Key, typename Value>
 std::shared_ptr<Value> HostCacheSegmented<Key, Value>:: find_else_create(const Key& key, bool* new_data)

diff --git a/src/host_tracker/test/host_cache_module_test.cc b/src/host_tracker/test/host_cache_module_test.cc
index 8b8f629441b3f40a8b55c3d71d20b9dcf690db28..bfa4c4955587a0b34206dc52b3378ec2f5294a7f 100644 (file)
--- a/src/host_tracker/test/host_cache_module_test.cc
+++ b/src/host_tracker/test/host_cache_module_test.cc
@@ -31,6 +31,7 @@
 #include "host_tracker/host_cache.h"
 #include "host_tracker/host_cache_segmented.h"
 #include "main/snort_config.h"
+#include "main/thread_config.h"
 #include "managers/module_manager.h"
 
 #include <CppUTest/CommandLineTestRunner.h>
@@ -85,6 +86,9 @@ time_t packet_time() { return 0; }
 bool Snort::is_reloading() { return false; }
 void SnortConfig::register_reload_handler(ReloadResourceTuner* rrt) { delete rrt; }
 void FatalError(const char* fmt, ...) { (void)fmt; exit(1); }
+unsigned get_instance_id()
+{ return 0; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 } // end of namespace snort
 
 void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { }

diff --git a/src/host_tracker/test/host_tracker_module_test.cc b/src/host_tracker/test/host_tracker_module_test.cc
index 529201168c247a823319c33567e6ac72363ceb21..5d429d2beb9bf64c4cf154a5327fdabf74f116b5 100644 (file)
--- a/src/host_tracker/test/host_tracker_module_test.cc
+++ b/src/host_tracker/test/host_tracker_module_test.cc
@@ -29,6 +29,7 @@
 #include "host_tracker/host_cache_segmented.h"
 #include "host_tracker/host_tracker_module.h"
 #include "main/snort_config.h"
+#include "main/thread_config.h"
 #include "target_based/snort_protocols.h"
 
 #include <CppUTest/CommandLineTestRunner.h>
@@ -42,6 +43,9 @@ char* snort_strdup(const char* s)
 { return strdup(s); }
 time_t packet_time() { return 0; }
 void FatalError(const char* fmt, ...) { (void)fmt; exit(1); }
+unsigned get_instance_id()
+{ return 0; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 }
 
 //  Fake show_stats to avoid bringing in a ton of dependencies.

diff --git a/src/ips_options/ips_pcre.cc b/src/ips_options/ips_pcre.cc
index a1636ec1a2e150372cb958440e790f6299a71674..56c063706521ccc3eeec53f1a8ac12767ded952a 100644 (file)
--- a/src/ips_options/ips_pcre.cc
+++ b/src/ips_options/ips_pcre.cc
@@ -672,6 +672,9 @@ public:
 
     PcreData* get_data();
 
+    bool global_stats() const override
+    { return true; }
+
     Usage get_usage() const override
     { return DETECT; }
 

diff --git a/src/ips_options/test/ips_regex_test.cc b/src/ips_options/test/ips_regex_test.cc
index e2e8d2ae1f891df57ee4edcc4e47cb1ebccc24ac..f040d1ae7499c1d3c7daf1095bf50e0afaa9ecbf 100644 (file)
--- a/src/ips_options/test/ips_regex_test.cc
+++ b/src/ips_options/test/ips_regex_test.cc
@@ -30,6 +30,7 @@
 #include "framework/module.h"
 #include "log/messages.h"
 #include "main/snort_config.h"
+#include "main/thread_config.h"
 #include "ports/port_group.h"
 #include "profiler/profiler_defs.h"
 #include "protocols/packet.h"
@@ -96,6 +97,7 @@ void ParseWarning(WarningGroup, const char*, ...) { }
 
 unsigned get_instance_id()
 { return 0; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 
 char* snort_strdup(const char* s)
 { return strdup(s); }

diff --git a/src/main/snort.cc b/src/main/snort.cc
index 1cb0781ca0c22bf73c7c4ffc60e3faa68dd65a62..55e3a5c15ad0225a9b37fcb9fff0cb33b42155b1 100644 (file)
--- a/src/main/snort.cc
+++ b/src/main/snort.cc
@@ -172,6 +172,7 @@ void Snort::init(int argc, char** argv)
     HighAvailabilityManager::configure(sc->ha_config);
     memory::MemoryCap::init(sc->thread_config->get_instance_max());
 
+    ModuleManager::init_stats();
     ModuleManager::reset_stats(sc);
 
     if (sc->alert_before_pass())

diff --git a/src/managers/module_manager.cc b/src/managers/module_manager.cc
index 95127e5a33b0aec7587046f2811184f2e0618fee..f28d832f43e4aa43d10a7f0983c6f68411bbcb9a 100644 (file)
--- a/src/managers/module_manager.cc
+++ b/src/managers/module_manager.cc
@@ -73,6 +73,7 @@ static std::unordered_map<std::string, ModHook*> s_modules;
 static std::unordered_map<std::string, const Parameter*> s_pmap;
 
 static unsigned s_errors = 0;
+const char* ModuleManager::dynamic_stats_modules = "file_id appid";
 
 set<uint32_t> ModuleManager::gids;
 mutex ModuleManager::stats_mutex;
@@ -1431,11 +1432,36 @@ PegCount* ModuleManager::get_stats(const char* name)
     ModHook* mh = get_hook(name);
 
     if ( mh )
-        pc = &mh->mod->dump_stats_counts[0];
+        pc = &mh->mod->dump_stats_counts[0][0];
 
     return pc;
 }
 
+void ModuleManager::accumulate_dump_stats()
+{
+    auto mod_hooks = get_all_modhooks();
+    for ( auto* mh : mod_hooks )
+    {
+        mh->mod->main_accumulate_stats();
+    }
+}
+
+void ModuleManager::init_stats()
+{
+    auto mod_hooks = get_all_modhooks();
+    for ( auto* mh : mod_hooks )
+    {
+        mh->mod->init_stats();
+    }
+}
+
+void ModuleManager::add_thread_stats_entry(const char* name)
+{
+    ModHook* mh = get_hook(name);
+    if ( mh )
+        mh->mod->init_stats(true);
+}
+
 void ModuleManager::dump_stats(const char* skip, bool dynamic)
 {
     auto mod_hooks = get_all_modhooks();
@@ -1445,11 +1471,21 @@ void ModuleManager::dump_stats(const char* skip, bool dynamic)
     {
         if ( !skip || !strstr(skip, mh->mod->get_name()) )
         {
-            lock_guard<mutex> lock(stats_mutex);
-            if ( dynamic )
-                mh->mod->show_dynamic_stats();
+            if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats())
+            {
+                lock_guard<mutex> lock(stats_mutex);
+                if ( dynamic )
+                    mh->mod->show_dynamic_stats();
+                else
+                    mh->mod->show_stats();
+            }
             else
-                mh->mod->show_stats();
+            {
+                if ( dynamic )
+                    mh->mod->show_dynamic_stats();
+                else
+                    mh->mod->show_stats();
+            }
         }
     }
 }
@@ -1463,9 +1499,17 @@ void ModuleManager::accumulate(const char* except)
         if ( except and !strcmp(mh->mod->name, except) )
             continue;
 
-        lock_guard<mutex> lock(stats_mutex);
-        mh->mod->prep_counts(true);
-        mh->mod->sum_stats(true);
+        if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats())
+        {
+            lock_guard<mutex> lock(stats_mutex);
+            mh->mod->prep_counts(true);
+            mh->mod->sum_stats(true);
+        }
+        else
+        {
+            mh->mod->prep_counts(true);
+            mh->mod->sum_stats(true);
+        }
     }
 }
 
@@ -1474,9 +1518,17 @@ void ModuleManager::accumulate_module(const char* name)
     ModHook* mh = get_hook(name);
     if ( mh )
     {
-        lock_guard<mutex> lock(stats_mutex);
-        mh->mod->prep_counts(true);
-        mh->mod->sum_stats(true);
+        if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats())
+        {
+            lock_guard<mutex> lock(stats_mutex);
+            mh->mod->prep_counts(true);
+            mh->mod->sum_stats(true);
+        }
+        else
+        {
+            mh->mod->prep_counts(true);
+            mh->mod->sum_stats(true);
+        }
     }
 }
 
@@ -1486,8 +1538,15 @@ void ModuleManager::reset_stats(SnortConfig*)
 
     for ( auto* mh : mod_hooks )
     {
-        lock_guard<mutex> lock(stats_mutex);
-        mh->mod->reset_stats();
+        if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats())
+        {
+            lock_guard<mutex> lock(stats_mutex);
+            mh->mod->reset_stats();
+        }
+        else
+        {
+            mh->mod->reset_stats();
+        }
     }
 }
 
@@ -1507,8 +1566,15 @@ void ModuleManager::clear_global_active_counters()
 
     for ( auto* mh : mod_hooks )
     {
-        lock_guard<mutex> lock(stats_mutex);
-        mh->mod->clear_global_active_counters();
+        if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats())
+        {
+            lock_guard<mutex> lock(stats_mutex);
+            mh->mod->clear_global_active_counters();
+        }
+        else
+        {
+            mh->mod->clear_global_active_counters();
+        }
     }
 }
 
@@ -1519,8 +1585,15 @@ void ModuleManager::reset_stats(clear_counter_type_t type)
         ModHook* mh = get_hook(clear_counter_type_string_map[type]);
         if ( mh and mh->mod )
         {
-            lock_guard<mutex> lock(stats_mutex);
-            mh->mod->reset_stats();
+            if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats())
+            {
+                lock_guard<mutex> lock(stats_mutex);
+                mh->mod->reset_stats();
+            }
+            else
+            {
+                mh->mod->reset_stats();
+            }
         }
     }
     else
@@ -1543,8 +1616,15 @@ void ModuleManager::reset_stats(clear_counter_type_t type)
 
             if ( type == TYPE_ALL or !ignore )
             {
-                lock_guard<mutex> lock(stats_mutex);
-                mh->mod->reset_stats();
+                if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats())
+                {
+                    lock_guard<mutex> lock(stats_mutex);
+                    mh->mod->reset_stats();
+                }
+                else
+                {
+                    mh->mod->reset_stats();
+                }
             }
         }
     }

diff --git a/src/managers/module_manager.h b/src/managers/module_manager.h
index 5be30b73ceadc92d055925ca293367e93b093620..b78673d37cff452ef426cef9795113083abf978c 100644 (file)
--- a/src/managers/module_manager.h
+++ b/src/managers/module_manager.h
@@ -87,8 +87,11 @@ public:
 
     static PegCount* get_stats(const char* name);
     static void dump_stats(const char* skip = nullptr, bool dynamic = false);
+    static void accumulate_dump_stats();
+    static void init_stats();
+    static void add_thread_stats_entry(const char* name);
 
-    static void accumulate(const char* except = nullptr);
+    static void accumulate(const char* except = "snort");
     static void accumulate_module(const char* name);
 
     static void reset_stats(SnortConfig*);
@@ -102,6 +105,7 @@ public:
 
     static std::set<uint32_t> gids;
     SO_PUBLIC static std::mutex stats_mutex;
+    SO_PUBLIC static const char* dynamic_stats_modules;
 };
 }
 

diff --git a/src/managers/test/get_inspector_stubs.h b/src/managers/test/get_inspector_stubs.h
index ea69f12789fb881d742bd327ea42559336bc5105..f13fa64151c47a138ad31f163c67b2c116bad62c 100644 (file)
--- a/src/managers/test/get_inspector_stubs.h
+++ b/src/managers/test/get_inspector_stubs.h
@@ -74,6 +74,8 @@ uint8_t TraceApi::get_constraints_generation() { return 0; }
 void TraceApi::filter(const Packet&) { }
 PegCount Module::get_global_count(const char*) const { return 0; }
 void Module::sum_stats(bool) { }
+void Module::init_stats(bool) { }
+void Module::main_accumulate_stats() { }
 void Module::show_interval_stats(IndexVec&, FILE*) { }
 void Module::show_stats() { }
 void Module::reset_stats() { }

diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_sip_test.cc b/src/network_inspectors/appid/detector_plugins/test/detector_sip_test.cc
index 78dff30fff814a02c5de5470dcf1e0ec45627ccb..398625aad76e4a4b543cb63ebc9445808c8c789c 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/test/detector_sip_test.cc
+++ b/src/network_inspectors/appid/detector_plugins/test/detector_sip_test.cc
@@ -30,6 +30,7 @@
 #include "framework/data_bus.h"
 #include "framework/module.cc"
 #include "framework/mpse_batch.h"
+#include "main/thread_config.h"
 #include "network_inspectors/appid/appid_utils/sf_mlmp.cc"
 #include "protocols/protocol_ids.h"
 #include "service_inspectors/sip/sip_parser.h"
@@ -77,6 +78,9 @@ int SearchTool::find_all(const char*, unsigned, MpseMatch, bool, void*, const Sn
     assert(mpsegrp);
     return 0;
 }
+unsigned get_instance_id()
+{ return 0; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 }
 
 AppIdInspector::AppIdInspector(AppIdModule&) { }

diff --git a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc
index 4c3d81f44a4ee471fd771999645337c6b052d4e8..8a600e13d8544e525c455ce36a07bb5dbb1d3014 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc
+++ b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc
@@ -23,6 +23,7 @@
 
 #include "network_inspectors/appid/detector_plugins/http_url_patterns.cc"
 
+#include "main/thread_config.h"
 #include "protocols/protocol_ids.h"
 #include "framework/module.cc"
 #include "network_inspectors/appid/appid_utils/sf_mlmp.cc"
@@ -72,6 +73,9 @@ int SearchTool::find_all(const char*, unsigned, MpseMatch, bool, void* mp_arg, c
         memcpy(mp_arg, &mock_mp, sizeof(MatchedPatterns*));
     return 0;
 }
+unsigned get_instance_id()
+{ return 0; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 }
 
 void ApplicationDescriptor::set_id(const Packet&, AppIdSession&, AppidSessionDirection, AppId, AppidChangeBits&) { }

diff --git a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h
index c3e55bf48d7822da7b6382c28e684f2a95d2655b..6fb04cd84767427a478a92dfac0ed6eaa687b644 100644 (file)
--- a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h
+++ b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h
@@ -76,7 +76,8 @@ Module::Module(const char*, const char*, const Parameter*, bool)
 PegCount Module::get_global_count(char const*) const { return 0; }
 void Module::show_interval_stats(std::vector<unsigned int, std::allocator<unsigned int> >&, FILE*) {}
 void Module::show_stats(){}
-void Module::sum_stats(bool ){}
+void Module::sum_stats(bool){}
+void Module::main_accumulate_stats(){}
 void Module::reset_stats() {}
 
 AppIdSessionApi::AppIdSessionApi(const AppIdSession*, const SfIp&) :

diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc
index 2df6a4dd7f8485fb06dd927bd473c7069bbe881f..ac8731d0077c110d15badfa110a994947c4b60d3 100644 (file)
--- a/src/network_inspectors/appid/test/appid_discovery_test.cc
+++ b/src/network_inspectors/appid/test/appid_discovery_test.cc
@@ -77,7 +77,9 @@ Module::Module(char const*, char const*) {}
 void Module::sum_stats(bool) {}
 void Module::show_interval_stats(std::vector<unsigned>&, FILE*) {}
 void Module::show_stats() {}
+void Module::init_stats(bool) {}
 void Module::reset_stats() {}
+void Module::main_accumulate_stats() {}
 PegCount Module::get_global_count(char const*) const { return 0; }
 
 // Stubs for logs

diff --git a/src/network_inspectors/appid/test/appid_mock_inspector.h b/src/network_inspectors/appid/test/appid_mock_inspector.h
index 63a95af2aeeef9bf8b28ab4033a2109dd08c75c8..be225ef58e71e366fa2c09dd89daa25c3603702b 100644 (file)
--- a/src/network_inspectors/appid/test/appid_mock_inspector.h
+++ b/src/network_inspectors/appid/test/appid_mock_inspector.h
@@ -39,8 +39,10 @@ class StreamSplitter* Inspector::get_splitter(bool) { return nullptr; }
 
 Module::Module(char const*, char const*) {}
 void Module::sum_stats(bool) {}
+void Module::main_accumulate_stats() { }
 void Module::show_interval_stats(std::vector<unsigned int, std::allocator<unsigned int> >&, FILE*) {}
 void Module::show_stats() {}
+void Module::init_stats(bool) {}
 void Module::reset_stats() {}
 PegCount Module::get_global_count(char const*) const { return 0; }
 

diff --git a/src/network_inspectors/perf_monitor/base_tracker.cc b/src/network_inspectors/perf_monitor/base_tracker.cc
index e9bb4a61ea7b86f5c74eb4b755d113e5140addfc..ced852c5062b82407a1c108b8172954ea01d040b 100644 (file)
--- a/src/network_inspectors/perf_monitor/base_tracker.cc
+++ b/src/network_inspectors/perf_monitor/base_tracker.cc
@@ -58,8 +58,15 @@ void BaseTracker::process(bool summary)
     {
         for ( const ModuleConfig& mod : modules )
         {
-            lock_guard<mutex> lock(ModuleManager::stats_mutex);
-            mod.ptr->sum_stats(false);
+            if (strstr(ModuleManager::dynamic_stats_modules, mod.ptr->get_name()) || mod.ptr->global_stats())
+            {
+                lock_guard<mutex> lock(ModuleManager::stats_mutex);
+                mod.ptr->sum_stats(false);
+            }
+            else
+            {
+                mod.ptr->sum_stats(false);
+            }
         }
     }
 }
@@ -85,6 +92,8 @@ public:
 
     void sum_stats(bool) override {}
 
+    void init_stats(bool) override {Module::init_stats();}
+
     void real_sum_stats() { Module::sum_stats(false); }
 
     Usage get_usage() const override
@@ -124,6 +133,7 @@ TEST_CASE("module stats", "[BaseTracker]")
     config.format = PerfFormat::MOCK;
 
     MockModule mod;
+    mod.init_stats(false);
     ModuleConfig mod_cfg;
     mod_cfg.ptr = &mod;
     mod_cfg.pegs = {0, 2, 4};

diff --git a/src/network_inspectors/rna/test/rna_module_stubs.h b/src/network_inspectors/rna/test/rna_module_stubs.h
index 0f360b4fe158e90d0e8f849e643eb39f6735851e..34043ca35ae3013df08ec1d55bc0f939c85b4696 100644 (file)
--- a/src/network_inspectors/rna/test/rna_module_stubs.h
+++ b/src/network_inspectors/rna/test/rna_module_stubs.h
@@ -30,7 +30,9 @@ char* snort_strdup(const char* s)
 { return strdup(s); }
 
 void Module::sum_stats(bool) {}
+void Module::main_accumulate_stats() {}
 void Module::show_stats() {}
+void Module::init_stats(bool) {}
 void Module::reset_stats() {}
 PegCount Module::get_global_count(char const*) const
 { return 0; }

diff --git a/src/payload_injector/test/payload_injector_test.cc b/src/payload_injector/test/payload_injector_test.cc
index 74d47a9c2cfce113296aef140357113eeeb94505..1e1bbb17c4110834c97bbdecd06024f4044e46d8 100644 (file)
--- a/src/payload_injector/test/payload_injector_test.cc
+++ b/src/payload_injector/test/payload_injector_test.cc
@@ -28,6 +28,7 @@
 
 #include "detection/detection_engine.h"
 #include "flow/flow.h"
+#include "main/thread_config.h"
 #include "packet_io/active.h"
 #include "protocols/packet.h"
 #include "utils/util.h"
@@ -61,6 +62,9 @@ DataBus::DataBus() = default;
 DataBus::~DataBus() = default;
 SnortConfig::SnortConfig(snort::SnortConfig const*, const char*) { }
 SnortConfig::~SnortConfig() = default;
+unsigned get_instance_id()
+{ return 0; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 
 IpsContext ips_context;
 SnortConfig conf;

diff --git a/src/search_engines/test/hyper_tool_test.cc b/src/search_engines/test/hyper_tool_test.cc
index ef87faf921fdd325ed17173cbc2e5697f5a68c51..a9afc3e6d4c21855f0b0fb59d83fcac7e208ebc5 100644 (file)
--- a/src/search_engines/test/hyper_tool_test.cc
+++ b/src/search_engines/test/hyper_tool_test.cc
@@ -33,6 +33,7 @@
 #include "framework/base_api.h"
 #include "framework/mpse_batch.h"
 #include "main/snort_config.h"
+#include "main/thread_config.h"
 #include "managers/mpse_manager.h"
 
 #include "mpse_test_stubs.h"
@@ -43,6 +44,12 @@
 
 using namespace snort;
 
+namespace snort
+{
+    unsigned get_instance_id() { return 0; }
+    unsigned ThreadConfig::get_instance_max() { return 1; }
+}
+
 //-------------------------------------------------------------------------
 // stubs, spies, etc.
 //-------------------------------------------------------------------------

diff --git a/src/search_engines/test/hyperscan_test.cc b/src/search_engines/test/hyperscan_test.cc
index 1b7aae4bf62a10927e01c8bb263697a55fe46020..a3f8c16c2ddc7cc1877c623bd6133fa23f4d2fcc 100644 (file)
--- a/src/search_engines/test/hyperscan_test.cc
+++ b/src/search_engines/test/hyperscan_test.cc
@@ -30,6 +30,7 @@
 #include "framework/mpse.h"
 #include "framework/mpse_batch.h"
 #include "main/snort_config.h"
+#include "main/thread_config.h"
 
 #include "mpse_test_stubs.h"
 
@@ -39,6 +40,12 @@
 
 using namespace snort;
 
+namespace snort
+{
+    unsigned get_instance_id() { return 0; }
+    unsigned ThreadConfig::get_instance_max() { return 1; }
+}
+
 //-------------------------------------------------------------------------
 // stubs, spies, etc.
 //-------------------------------------------------------------------------

diff --git a/src/search_engines/test/mpse_test_stubs.cc b/src/search_engines/test/mpse_test_stubs.cc
index 85e98b3e7476c5292943456b623b03386257ef1a..74bc48349afcaea8945369fb2b1bd0e198c98452 100644 (file)
--- a/src/search_engines/test/mpse_test_stubs.cc
+++ b/src/search_engines/test/mpse_test_stubs.cc
@@ -76,9 +76,6 @@ void SnortConfig::release_scratch(int)
 DataBus::DataBus() = default;
 DataBus::~DataBus() = default;
 
-unsigned get_instance_id()
-{ return 0; }
-
 THREAD_LOCAL PatMatQStat pmqs;
 
 unsigned parse_errors = 0;

diff --git a/src/service_inspectors/http_inspect/test/http_module_test.cc b/src/service_inspectors/http_inspect/test/http_module_test.cc
index b0a909342b5808f26458858a8c6ab970240c5ac8..60a977d00ecd2b2c727eb85d9e4d94e49a4dbb2f 100755 (executable)
--- a/src/service_inspectors/http_inspect/test/http_module_test.cc
+++ b/src/service_inspectors/http_inspect/test/http_module_test.cc
@@ -26,6 +26,7 @@
 #include "helpers/literal_search.h"
 #include "log/messages.h"
 
+#include "main/thread_config.h"
 #include "service_inspectors/http_inspect/http_js_norm.h"
 #include "service_inspectors/http_inspect/http_module.h"
 #include "service_inspectors/http_inspect/http_str_to_code.h"
@@ -60,6 +61,9 @@ void DecodeConfig::set_decompress_zip(bool) {}
 void DecodeConfig::set_decompress_vba(bool) {}
 
 SearchTool::~SearchTool() {}
+unsigned get_instance_id()
+{ return 0; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 }
 
 void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { }

diff --git a/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc b/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc
index 02a5ffa58ac1ae3e4c16ffc22fd316e0ad2ee5b8..c7f5dfc18285f669d734c9124d531b16b1681ecd 100755 (executable)
--- a/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc
+++ b/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc
@@ -26,6 +26,7 @@
 #include "helpers/literal_search.h"
 #include "log/messages.h"
 
+#include "main/thread_config.h"
 #include "service_inspectors/http_inspect/http_js_norm.h"
 #include "service_inspectors/http_inspect/http_uri_norm.h"
 
@@ -54,6 +55,9 @@ void DecodeConfig::set_decompress_swf(bool) {}
 void DecodeConfig::set_decompress_zip(bool) {}
 void DecodeConfig::set_decompress_vba(bool) {}
 SearchTool::~SearchTool() {}
+unsigned get_instance_id()
+{ return 0; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 }
 
 snort::SearchTool* js_create_mpse_open_tag() { return nullptr; }

diff --git a/src/side_channel/test/side_channel_module_test.cc b/src/side_channel/test/side_channel_module_test.cc
index 188902e01dd987ca32570f56f3abbf9b6a6e9aa2..6c5ec19593cca4bbf71f0fba8339dd7d4dcc3ede 100644 (file)
--- a/src/side_channel/test/side_channel_module_test.cc
+++ b/src/side_channel/test/side_channel_module_test.cc
@@ -27,6 +27,7 @@
 #include "side_channel/side_channel_module.h"
 
 #include "log/messages.h"
+#include "main/thread_config.h"
 #include "profiler/profiler.h"
 
 #include <CppUTest/CommandLineTestRunner.h>
@@ -64,6 +65,9 @@ void ParseWarning(WarningGroup, const char*, ...) { }
 
 char* snort_strdup(const char* s)
 { return strdup(s); }
+unsigned get_instance_id()
+{ return 0; }
+unsigned ThreadConfig::get_instance_max() { return 1; }
 }
 
 TEST_GROUP(side_channel_module)

diff --git a/src/trace/trace.cc b/src/trace/trace.cc
index 2a8b220bcadd9b879de974f99ed47b5abbe882ab..c7ab4feabcaa176db052b76ce584573556bd0985 100644 (file)
--- a/src/trace/trace.cc
+++ b/src/trace/trace.cc
@@ -97,7 +97,9 @@ Module::Module(const char* s, const char* h, const Parameter* p, bool l) : name(
 PegCount Module::get_global_count(char const*) const { return 0; }
 void Module::show_interval_stats(std::vector<unsigned int, std::allocator<unsigned int> >&, FILE*) {}
 void Module::show_stats(){}
+void Module::init_stats(bool){}
 void Module::sum_stats(bool){}
+void Module::main_accumulate_stats(){}
 void Module::reset_stats() {}
 
 class TraceTestModule : public Module

diff --git a/src/utils/stats.cc b/src/utils/stats.cc
index b2f139d6a83ad8d22a071ee300fe9a8f0d6c349b..bdf3ba82caff8cb1e986fad17cf2a3e3dbd9c38e 100644 (file)
--- a/src/utils/stats.cc
+++ b/src/utils/stats.cc
@@ -263,6 +263,7 @@ const PegInfo proc_names[] =
 void DropStats(ControlConn* ctrlcon)
 {
     s_ctrlcon = ctrlcon;
+    ModuleManager::accumulate_dump_stats();
     LogLabel("Packet Statistics");
     ModuleManager::get_module("daq")->show_stats();
     PacketManager::dump_stats();