From: ARUNKUMAR KAYAMBU -X (akayambu - XORIANT CORPORATION at Cisco) Date: Mon, 4 Mar 2024 20:18:43 +0000 (+0000) Subject: Pull request #4095: Perf accumulate each thread data X-Git-Tag: 3.1.82.0~9 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0487ee1903a2bc469ca9d3e89723200ac1177cfa;p=thirdparty%2Fsnort3.git Pull request #4095: Perf accumulate each thread data Merge in SNORT/snort3 from ~AKAYAMBU/snort3:perf_accumulate_each_thread_data to master Squashed commit of the following: commit bc4dfd6566df0b9276141b33529c9c2e6d619231 Author: Arunkumar Kayambu Date: Fri Oct 27 11:19:09 2023 -0400 stats: perf improvement changes --- diff --git a/src/codecs/misc/test/geneve_codec_test.cc b/src/codecs/misc/test/geneve_codec_test.cc index 0c865c912..7082dfe73 100644 --- a/src/codecs/misc/test/geneve_codec_test.cc +++ b/src/codecs/misc/test/geneve_codec_test.cc @@ -20,6 +20,7 @@ #include "../cd_geneve.cc" #include "utils/endian.h" +#include "main/thread_config.h" #include #include @@ -33,6 +34,9 @@ namespace snort bool TextLog_Print(TextLog* const, const char*, ...) { return false; } void Codec::codec_event(const CodecData&, CodecSid) { } bool SnortConfig::tunnel_bypass_enabled(unsigned short) const { return false; } + unsigned get_instance_id() + { return 0; } + unsigned ThreadConfig::get_instance_max() { return 1; } } // Geneve data with 2 variable options. diff --git a/src/connectors/file_connector/test/file_connector_module_test.cc b/src/connectors/file_connector/test/file_connector_module_test.cc index 6c7bd84d4..63c3de64b 100644 --- a/src/connectors/file_connector/test/file_connector_module_test.cc +++ b/src/connectors/file_connector/test/file_connector_module_test.cc @@ -24,6 +24,7 @@ #endif #include "connectors/file_connector/file_connector_module.h" +#include "main/thread_config.h" #include "profiler/profiler.h" #include @@ -31,6 +32,7 @@ using namespace snort; + THREAD_LOCAL SimpleStats file_connector_stats; THREAD_LOCAL ProfileStats file_connector_perfstats; @@ -41,6 +43,9 @@ namespace snort { char* snort_strdup(const char* s) { return strdup(s); } +unsigned get_instance_id() +{ return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } } TEST_GROUP(file_connector_module) diff --git a/src/connectors/file_connector/test/file_connector_test.cc b/src/connectors/file_connector/test/file_connector_test.cc index a5a690605..5c4a3ce5e 100644 --- a/src/connectors/file_connector/test/file_connector_test.cc +++ b/src/connectors/file_connector/test/file_connector_test.cc @@ -25,12 +25,14 @@ #include "connectors/file_connector/file_connector.h" #include "connectors/file_connector/file_connector_module.h" +#include "main/thread_config.h" #include #include using namespace snort; + extern const BaseApi* file_connector; const ConnectorApi* fc_api = nullptr; @@ -55,6 +57,9 @@ namespace snort { const char* get_instance_file(std::string& file, const char* name) { file += name; return nullptr; } +unsigned get_instance_id() +{ return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } } FileConnectorModule::FileConnectorModule() : diff --git a/src/connectors/tcp_connector/test/tcp_connector_module_test.cc b/src/connectors/tcp_connector/test/tcp_connector_module_test.cc index 1c6cf684c..e944cbd47 100644 --- a/src/connectors/tcp_connector/test/tcp_connector_module_test.cc +++ b/src/connectors/tcp_connector/test/tcp_connector_module_test.cc @@ -24,6 +24,7 @@ #endif #include "connectors/tcp_connector/tcp_connector_module.h" +#include "main/thread_config.h" #include "profiler/profiler.h" #include @@ -31,6 +32,7 @@ using namespace snort; + THREAD_LOCAL SimpleStats tcp_connector_stats; THREAD_LOCAL ProfileStats tcp_connector_perfstats; @@ -41,6 +43,10 @@ namespace snort { char* snort_strdup(const char* s) { return strdup(s); } + +unsigned get_instance_id() +{ return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } } TEST_GROUP(tcp_connector_module) diff --git a/src/connectors/tcp_connector/test/tcp_connector_test.cc b/src/connectors/tcp_connector/test/tcp_connector_test.cc index edfc627c0..0a1798e9d 100644 --- a/src/connectors/tcp_connector/test/tcp_connector_test.cc +++ b/src/connectors/tcp_connector/test/tcp_connector_test.cc @@ -31,6 +31,8 @@ #include #include +#include "main/thread_config.h" + #include #include @@ -72,6 +74,7 @@ namespace snort { unsigned get_instance_id() { return s_instance; } +unsigned ThreadConfig::get_instance_max() { return 1; } void ErrorMessage(const char*, ...) { } void LogMessage(const char*, ...) { } diff --git a/src/detection/regex_offload.cc b/src/detection/regex_offload.cc index 69497b079..9ec335928 100644 --- a/src/detection/regex_offload.cc +++ b/src/detection/regex_offload.cc @@ -182,7 +182,11 @@ ThreadRegexOffload::ThreadRegexOffload(unsigned max) : RegexOffload(max) const SnortConfig* sc = SnortConfig::get_conf(); for ( auto* req : idle ) + { + ModuleManager::add_thread_stats_entry("search_engine"); + ModuleManager::add_thread_stats_entry("detection"); req->thread = new std::thread(worker, req, sc, i++); + } } ThreadRegexOffload::~ThreadRegexOffload() diff --git a/src/flow/test/flow_cache_test.cc b/src/flow/test/flow_cache_test.cc index 3d39419eb..85cdf800d 100644 --- a/src/flow/test/flow_cache_test.cc +++ b/src/flow/test/flow_cache_test.cc @@ -32,6 +32,7 @@ #include "flow/ha.h" #include "flow/session.h" #include "main/analyzer.h" +#include "main/thread_config.h" #include "managers/inspector_manager.h" #include "packet_io/active.h" #include "protocols/icmp4.h" diff --git a/src/framework/module.cc b/src/framework/module.cc index f95f5de61..ed0af5b53 100644 --- a/src/framework/module.cc +++ b/src/framework/module.cc @@ -22,6 +22,7 @@ #endif #include "module.h" +#include "main/thread_config.h" #include "trace/trace.h" @@ -77,18 +78,55 @@ void Module::clear_global_active_counters() const PegInfo* q = get_pegs(); assert(q); + for ( unsigned thread_index = 0; thread_index < dump_stats_counts.size(); thread_index++) + { + for ( int i = 0; i < num_counts; i++ ) + { + if ( q[i].type == CountType::NOW ) + { + dump_stats_counts[thread_index][i] = 0; + counts[thread_index][i] = 0; + dump_stats_results[i] = 0; + } + } + } +} - for ( int i = 0; i < num_counts; i++ ) +void Module::main_accumulate_stats() +{ + const PegInfo* q = get_pegs(); + + if ( !q ) + return; + + if (!global_stats()) { - if ( q[i].type == CountType::NOW ) - counts[i] = 0; + for ( int i = 0; i < num_counts; i++ ) + { + for ( unsigned thread_index = 0; thread_index < dump_stats_counts.size(); thread_index++) + { + if ( q[i].type == CountType::SUM || q[i].type == CountType::NOW) + { + dump_stats_results[i] += dump_stats_counts[thread_index][i]; + } + else if ( q[i].type == CountType::MAX) + { + if (dump_stats_counts[thread_index][i] > dump_stats_results[i]) + dump_stats_results[i] = dump_stats_counts[thread_index][i]; + } + } + } } } + void Module::sum_stats(bool dump_stats) { if ( num_counts < 0 ) + { + init_stats(); reset_stats(); + } PegCount* p = get_counts(); const PegInfo* q = get_pegs(); @@ -97,17 +135,25 @@ void Module::sum_stats(bool dump_stats) return; assert(q); - if(dump_stats && !dump_stats_initialized) + + unsigned instance_id = get_instance_id(); + if (dump_stats and !dump_stats_initialized[instance_id]) { - for (unsigned long i=0; i 0 ) { - ::show_stats(&dump_stats_counts[0], get_pegs(), num_counts, get_name()); - dump_stats_initialized = false; + ::show_stats(&dump_stats_results[0], get_pegs(), num_counts, get_name()); + dump_stats_initialized.assign(dump_stats_initialized.size(), 0); + std::fill(dump_stats_results.begin(), dump_stats_results.end(), 0); } } -void Module::reset_stats() +void Module::init_stats(bool new_thread) { - PegCount* p = get_counts(); - - if ( !p ) - return; - const PegInfo* pegs = get_pegs(); if ( !pegs ) @@ -170,15 +212,39 @@ void Module::reset_stats() while ( pegs[num_counts].name ) ++num_counts; + } + + unsigned number_of_threads = new_thread ? 1 : ThreadConfig::get_instance_max(); - counts.resize(num_counts); - dump_stats_counts.resize(num_counts); + for ( unsigned thread_index = 0; thread_index < number_of_threads; thread_index++) + { + std::vector stats(num_counts); + std::vector dump_stats(num_counts); + counts.push_back(stats); + dump_stats_counts.push_back(dump_stats); + dump_stats_initialized.push_back(0); } + dump_stats_results.resize(num_counts); +} + + +void Module::reset_stats() +{ + PegCount* p = get_counts(); + + if ( !p ) + return; + + const PegInfo* pegs = get_pegs(); + + if ( !pegs ) + return; + for ( int i = 0; i < num_counts; i++ ) { - counts[i] = 0; - dump_stats_counts[i] = 0; + counts[get_instance_id()][i] = 0; + dump_stats_counts[get_instance_id()][i] = 0; if ( pegs[i].type != CountType::NOW ) p[i] = 0; @@ -193,7 +259,7 @@ PegCount Module::get_global_count(const char* name) const for ( unsigned i = 0; infos[i].name; i++ ) { if ( strcmp(name, infos[i].name) == 0 ) - return counts[i]; + return dump_stats_results[i]; } assert(false); // wrong name = programmer error return 0; diff --git a/src/framework/module.h b/src/framework/module.h index 390b5f8b6..24b87b537 100644 --- a/src/framework/module.h +++ b/src/framework/module.h @@ -175,6 +175,8 @@ public: virtual void show_interval_stats(IndexVec&, FILE*); virtual void show_stats(); virtual void reset_stats(); + virtual void init_stats(bool new_thread=false); + virtual void main_accumulate_stats(); virtual void show_dynamic_stats() {} void clear_global_active_counters(); @@ -203,35 +205,50 @@ protected: void set_params(const Parameter* p) { params = p; } - - bool dump_stats_initialized = false; - - std::vector counts; - std::vector dump_stats_counts; + std::vector dump_stats_initialized; + std::vector> counts; + std::vector> dump_stats_counts; + std::vector dump_stats_results; int num_counts = -1; - void set_peg_count(int index, PegCount value, bool dump_stats = false) + void set_peg_count(int index, PegCount value, bool dump_stats) + { + assert(index < num_counts); + if (dump_stats) + dump_stats_counts[get_instance_id()][index] = value; + else + counts[get_instance_id()][index] = value; + } + + void set_max_peg_count(int index, PegCount value, bool dump_stats) { assert(index < num_counts); - if(dump_stats) - dump_stats_counts[index] = value; + if (dump_stats) + { + if(value > dump_stats_counts[get_instance_id()][index]) + dump_stats_counts[get_instance_id()][index] = value; + } else - counts[index] = value; + { + if(value > counts[get_instance_id()][index]) + counts[get_instance_id()][index] = value; + } } - void add_peg_count(int index, PegCount value, bool dump_stats = false) + void add_peg_count(int index, PegCount value, bool dump_stats) { assert(index < num_counts); - if(dump_stats) - dump_stats_counts[index] += value; + if (dump_stats) + dump_stats_counts[get_instance_id()][index] += value; else - counts[index] += value; + counts[get_instance_id()][index] += value; } private: friend ModuleManager; void init(const char*, const char* = nullptr); + const char* name; const char* help; @@ -239,20 +256,6 @@ private: bool list; int table_level = 0; - void set_max_peg_count(int index, PegCount value, bool dump_stats = false) - { - assert(index < num_counts); - if(dump_stats) - { - if(value > dump_stats_counts[index]) - dump_stats_counts[index] = value; - } - else - { - if(value > counts[index]) - counts[index] = value; - } - } }; } #endif diff --git a/src/host_tracker/host_cache_module.cc b/src/host_tracker/host_cache_module.cc index 8ed5adb82..9612d28df 100644 --- a/src/host_tracker/host_cache_module.cc +++ b/src/host_tracker/host_cache_module.cc @@ -580,7 +580,6 @@ void HostCacheModule::prep_counts(bool) void HostCacheModule::sum_stats(bool dump_stats) { Module::sum_stats(dump_stats); - host_cache.sum_stats(dump_stats); } void HostCacheModule::reset_stats() diff --git a/src/host_tracker/host_cache_segmented.h b/src/host_tracker/host_cache_segmented.h index 3f0d9fdb5..01e660109 100644 --- a/src/host_tracker/host_cache_segmented.h +++ b/src/host_tracker/host_cache_segmented.h @@ -65,7 +65,6 @@ public: void invalidate(); void update_counts(); void reset_counts(); - void sum_stats(bool dump_stats); std::shared_ptr operator[](const Key& key); @@ -278,27 +277,6 @@ void HostCacheSegmented::reset_counts() } } -template -void HostCacheSegmented::sum_stats(bool dump_stats) -{ - std::lock_guard guard(stats_lock); - if ( !dump_stats ) - { - const PegInfo* pegs = get_pegs(); - - for (auto cache : seg_list) - { - PegCount* cache_counts = reinterpret_cast (&cache->stats); - cache->lock(); - for (int i = 0; pegs[i].type != CountType::END; i++) - { - if ( pegs[i].type == CountType::SUM ) - cache_counts[i] = 0; - } - cache->unlock(); - } - } -} template std::shared_ptr HostCacheSegmented:: find_else_create(const Key& key, bool* new_data) diff --git a/src/host_tracker/test/host_cache_module_test.cc b/src/host_tracker/test/host_cache_module_test.cc index 8b8f62944..bfa4c4955 100644 --- a/src/host_tracker/test/host_cache_module_test.cc +++ b/src/host_tracker/test/host_cache_module_test.cc @@ -31,6 +31,7 @@ #include "host_tracker/host_cache.h" #include "host_tracker/host_cache_segmented.h" #include "main/snort_config.h" +#include "main/thread_config.h" #include "managers/module_manager.h" #include @@ -85,6 +86,9 @@ time_t packet_time() { return 0; } bool Snort::is_reloading() { return false; } void SnortConfig::register_reload_handler(ReloadResourceTuner* rrt) { delete rrt; } void FatalError(const char* fmt, ...) { (void)fmt; exit(1); } +unsigned get_instance_id() +{ return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } } // end of namespace snort void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { } diff --git a/src/host_tracker/test/host_tracker_module_test.cc b/src/host_tracker/test/host_tracker_module_test.cc index 529201168..5d429d2be 100644 --- a/src/host_tracker/test/host_tracker_module_test.cc +++ b/src/host_tracker/test/host_tracker_module_test.cc @@ -29,6 +29,7 @@ #include "host_tracker/host_cache_segmented.h" #include "host_tracker/host_tracker_module.h" #include "main/snort_config.h" +#include "main/thread_config.h" #include "target_based/snort_protocols.h" #include @@ -42,6 +43,9 @@ char* snort_strdup(const char* s) { return strdup(s); } time_t packet_time() { return 0; } void FatalError(const char* fmt, ...) { (void)fmt; exit(1); } +unsigned get_instance_id() +{ return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } } // Fake show_stats to avoid bringing in a ton of dependencies. diff --git a/src/ips_options/ips_pcre.cc b/src/ips_options/ips_pcre.cc index a1636ec1a..56c063706 100644 --- a/src/ips_options/ips_pcre.cc +++ b/src/ips_options/ips_pcre.cc @@ -672,6 +672,9 @@ public: PcreData* get_data(); + bool global_stats() const override + { return true; } + Usage get_usage() const override { return DETECT; } diff --git a/src/ips_options/test/ips_regex_test.cc b/src/ips_options/test/ips_regex_test.cc index e2e8d2ae1..f040d1ae7 100644 --- a/src/ips_options/test/ips_regex_test.cc +++ b/src/ips_options/test/ips_regex_test.cc @@ -30,6 +30,7 @@ #include "framework/module.h" #include "log/messages.h" #include "main/snort_config.h" +#include "main/thread_config.h" #include "ports/port_group.h" #include "profiler/profiler_defs.h" #include "protocols/packet.h" @@ -96,6 +97,7 @@ void ParseWarning(WarningGroup, const char*, ...) { } unsigned get_instance_id() { return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } char* snort_strdup(const char* s) { return strdup(s); } diff --git a/src/main/snort.cc b/src/main/snort.cc index 1cb0781ca..55e3a5c15 100644 --- a/src/main/snort.cc +++ b/src/main/snort.cc @@ -172,6 +172,7 @@ void Snort::init(int argc, char** argv) HighAvailabilityManager::configure(sc->ha_config); memory::MemoryCap::init(sc->thread_config->get_instance_max()); + ModuleManager::init_stats(); ModuleManager::reset_stats(sc); if (sc->alert_before_pass()) diff --git a/src/managers/module_manager.cc b/src/managers/module_manager.cc index 95127e5a3..f28d832f4 100644 --- a/src/managers/module_manager.cc +++ b/src/managers/module_manager.cc @@ -73,6 +73,7 @@ static std::unordered_map s_modules; static std::unordered_map s_pmap; static unsigned s_errors = 0; +const char* ModuleManager::dynamic_stats_modules = "file_id appid"; set ModuleManager::gids; mutex ModuleManager::stats_mutex; @@ -1431,11 +1432,36 @@ PegCount* ModuleManager::get_stats(const char* name) ModHook* mh = get_hook(name); if ( mh ) - pc = &mh->mod->dump_stats_counts[0]; + pc = &mh->mod->dump_stats_counts[0][0]; return pc; } +void ModuleManager::accumulate_dump_stats() +{ + auto mod_hooks = get_all_modhooks(); + for ( auto* mh : mod_hooks ) + { + mh->mod->main_accumulate_stats(); + } +} + +void ModuleManager::init_stats() +{ + auto mod_hooks = get_all_modhooks(); + for ( auto* mh : mod_hooks ) + { + mh->mod->init_stats(); + } +} + +void ModuleManager::add_thread_stats_entry(const char* name) +{ + ModHook* mh = get_hook(name); + if ( mh ) + mh->mod->init_stats(true); +} + void ModuleManager::dump_stats(const char* skip, bool dynamic) { auto mod_hooks = get_all_modhooks(); @@ -1445,11 +1471,21 @@ void ModuleManager::dump_stats(const char* skip, bool dynamic) { if ( !skip || !strstr(skip, mh->mod->get_name()) ) { - lock_guard lock(stats_mutex); - if ( dynamic ) - mh->mod->show_dynamic_stats(); + if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats()) + { + lock_guard lock(stats_mutex); + if ( dynamic ) + mh->mod->show_dynamic_stats(); + else + mh->mod->show_stats(); + } else - mh->mod->show_stats(); + { + if ( dynamic ) + mh->mod->show_dynamic_stats(); + else + mh->mod->show_stats(); + } } } } @@ -1463,9 +1499,17 @@ void ModuleManager::accumulate(const char* except) if ( except and !strcmp(mh->mod->name, except) ) continue; - lock_guard lock(stats_mutex); - mh->mod->prep_counts(true); - mh->mod->sum_stats(true); + if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats()) + { + lock_guard lock(stats_mutex); + mh->mod->prep_counts(true); + mh->mod->sum_stats(true); + } + else + { + mh->mod->prep_counts(true); + mh->mod->sum_stats(true); + } } } @@ -1474,9 +1518,17 @@ void ModuleManager::accumulate_module(const char* name) ModHook* mh = get_hook(name); if ( mh ) { - lock_guard lock(stats_mutex); - mh->mod->prep_counts(true); - mh->mod->sum_stats(true); + if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats()) + { + lock_guard lock(stats_mutex); + mh->mod->prep_counts(true); + mh->mod->sum_stats(true); + } + else + { + mh->mod->prep_counts(true); + mh->mod->sum_stats(true); + } } } @@ -1486,8 +1538,15 @@ void ModuleManager::reset_stats(SnortConfig*) for ( auto* mh : mod_hooks ) { - lock_guard lock(stats_mutex); - mh->mod->reset_stats(); + if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats()) + { + lock_guard lock(stats_mutex); + mh->mod->reset_stats(); + } + else + { + mh->mod->reset_stats(); + } } } @@ -1507,8 +1566,15 @@ void ModuleManager::clear_global_active_counters() for ( auto* mh : mod_hooks ) { - lock_guard lock(stats_mutex); - mh->mod->clear_global_active_counters(); + if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats()) + { + lock_guard lock(stats_mutex); + mh->mod->clear_global_active_counters(); + } + else + { + mh->mod->clear_global_active_counters(); + } } } @@ -1519,8 +1585,15 @@ void ModuleManager::reset_stats(clear_counter_type_t type) ModHook* mh = get_hook(clear_counter_type_string_map[type]); if ( mh and mh->mod ) { - lock_guard lock(stats_mutex); - mh->mod->reset_stats(); + if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats()) + { + lock_guard lock(stats_mutex); + mh->mod->reset_stats(); + } + else + { + mh->mod->reset_stats(); + } } } else @@ -1543,8 +1616,15 @@ void ModuleManager::reset_stats(clear_counter_type_t type) if ( type == TYPE_ALL or !ignore ) { - lock_guard lock(stats_mutex); - mh->mod->reset_stats(); + if (strstr(dynamic_stats_modules, mh->mod->get_name()) || mh->mod->global_stats()) + { + lock_guard lock(stats_mutex); + mh->mod->reset_stats(); + } + else + { + mh->mod->reset_stats(); + } } } } diff --git a/src/managers/module_manager.h b/src/managers/module_manager.h index 5be30b73c..b78673d37 100644 --- a/src/managers/module_manager.h +++ b/src/managers/module_manager.h @@ -87,8 +87,11 @@ public: static PegCount* get_stats(const char* name); static void dump_stats(const char* skip = nullptr, bool dynamic = false); + static void accumulate_dump_stats(); + static void init_stats(); + static void add_thread_stats_entry(const char* name); - static void accumulate(const char* except = nullptr); + static void accumulate(const char* except = "snort"); static void accumulate_module(const char* name); static void reset_stats(SnortConfig*); @@ -102,6 +105,7 @@ public: static std::set gids; SO_PUBLIC static std::mutex stats_mutex; + SO_PUBLIC static const char* dynamic_stats_modules; }; } diff --git a/src/managers/test/get_inspector_stubs.h b/src/managers/test/get_inspector_stubs.h index ea69f1278..f13fa6415 100644 --- a/src/managers/test/get_inspector_stubs.h +++ b/src/managers/test/get_inspector_stubs.h @@ -74,6 +74,8 @@ uint8_t TraceApi::get_constraints_generation() { return 0; } void TraceApi::filter(const Packet&) { } PegCount Module::get_global_count(const char*) const { return 0; } void Module::sum_stats(bool) { } +void Module::init_stats(bool) { } +void Module::main_accumulate_stats() { } void Module::show_interval_stats(IndexVec&, FILE*) { } void Module::show_stats() { } void Module::reset_stats() { } diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_sip_test.cc b/src/network_inspectors/appid/detector_plugins/test/detector_sip_test.cc index 78dff30ff..398625aad 100644 --- a/src/network_inspectors/appid/detector_plugins/test/detector_sip_test.cc +++ b/src/network_inspectors/appid/detector_plugins/test/detector_sip_test.cc @@ -30,6 +30,7 @@ #include "framework/data_bus.h" #include "framework/module.cc" #include "framework/mpse_batch.h" +#include "main/thread_config.h" #include "network_inspectors/appid/appid_utils/sf_mlmp.cc" #include "protocols/protocol_ids.h" #include "service_inspectors/sip/sip_parser.h" @@ -77,6 +78,9 @@ int SearchTool::find_all(const char*, unsigned, MpseMatch, bool, void*, const Sn assert(mpsegrp); return 0; } +unsigned get_instance_id() +{ return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } } AppIdInspector::AppIdInspector(AppIdModule&) { } diff --git a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc index 4c3d81f44..8a600e13d 100644 --- a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc +++ b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc @@ -23,6 +23,7 @@ #include "network_inspectors/appid/detector_plugins/http_url_patterns.cc" +#include "main/thread_config.h" #include "protocols/protocol_ids.h" #include "framework/module.cc" #include "network_inspectors/appid/appid_utils/sf_mlmp.cc" @@ -72,6 +73,9 @@ int SearchTool::find_all(const char*, unsigned, MpseMatch, bool, void* mp_arg, c memcpy(mp_arg, &mock_mp, sizeof(MatchedPatterns*)); return 0; } +unsigned get_instance_id() +{ return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } } void ApplicationDescriptor::set_id(const Packet&, AppIdSession&, AppidSessionDirection, AppId, AppidChangeBits&) { } diff --git a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h index c3e55bf48..6fb04cd84 100644 --- a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h +++ b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h @@ -76,7 +76,8 @@ Module::Module(const char*, const char*, const Parameter*, bool) PegCount Module::get_global_count(char const*) const { return 0; } void Module::show_interval_stats(std::vector >&, FILE*) {} void Module::show_stats(){} -void Module::sum_stats(bool ){} +void Module::sum_stats(bool){} +void Module::main_accumulate_stats(){} void Module::reset_stats() {} AppIdSessionApi::AppIdSessionApi(const AppIdSession*, const SfIp&) : diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc index 2df6a4dd7..ac8731d00 100644 --- a/src/network_inspectors/appid/test/appid_discovery_test.cc +++ b/src/network_inspectors/appid/test/appid_discovery_test.cc @@ -77,7 +77,9 @@ Module::Module(char const*, char const*) {} void Module::sum_stats(bool) {} void Module::show_interval_stats(std::vector&, FILE*) {} void Module::show_stats() {} +void Module::init_stats(bool) {} void Module::reset_stats() {} +void Module::main_accumulate_stats() {} PegCount Module::get_global_count(char const*) const { return 0; } // Stubs for logs diff --git a/src/network_inspectors/appid/test/appid_mock_inspector.h b/src/network_inspectors/appid/test/appid_mock_inspector.h index 63a95af2a..be225ef58 100644 --- a/src/network_inspectors/appid/test/appid_mock_inspector.h +++ b/src/network_inspectors/appid/test/appid_mock_inspector.h @@ -39,8 +39,10 @@ class StreamSplitter* Inspector::get_splitter(bool) { return nullptr; } Module::Module(char const*, char const*) {} void Module::sum_stats(bool) {} +void Module::main_accumulate_stats() { } void Module::show_interval_stats(std::vector >&, FILE*) {} void Module::show_stats() {} +void Module::init_stats(bool) {} void Module::reset_stats() {} PegCount Module::get_global_count(char const*) const { return 0; } diff --git a/src/network_inspectors/perf_monitor/base_tracker.cc b/src/network_inspectors/perf_monitor/base_tracker.cc index e9bb4a61e..ced852c50 100644 --- a/src/network_inspectors/perf_monitor/base_tracker.cc +++ b/src/network_inspectors/perf_monitor/base_tracker.cc @@ -58,8 +58,15 @@ void BaseTracker::process(bool summary) { for ( const ModuleConfig& mod : modules ) { - lock_guard lock(ModuleManager::stats_mutex); - mod.ptr->sum_stats(false); + if (strstr(ModuleManager::dynamic_stats_modules, mod.ptr->get_name()) || mod.ptr->global_stats()) + { + lock_guard lock(ModuleManager::stats_mutex); + mod.ptr->sum_stats(false); + } + else + { + mod.ptr->sum_stats(false); + } } } } @@ -85,6 +92,8 @@ public: void sum_stats(bool) override {} + void init_stats(bool) override {Module::init_stats();} + void real_sum_stats() { Module::sum_stats(false); } Usage get_usage() const override @@ -124,6 +133,7 @@ TEST_CASE("module stats", "[BaseTracker]") config.format = PerfFormat::MOCK; MockModule mod; + mod.init_stats(false); ModuleConfig mod_cfg; mod_cfg.ptr = &mod; mod_cfg.pegs = {0, 2, 4}; diff --git a/src/network_inspectors/rna/test/rna_module_stubs.h b/src/network_inspectors/rna/test/rna_module_stubs.h index 0f360b4fe..34043ca35 100644 --- a/src/network_inspectors/rna/test/rna_module_stubs.h +++ b/src/network_inspectors/rna/test/rna_module_stubs.h @@ -30,7 +30,9 @@ char* snort_strdup(const char* s) { return strdup(s); } void Module::sum_stats(bool) {} +void Module::main_accumulate_stats() {} void Module::show_stats() {} +void Module::init_stats(bool) {} void Module::reset_stats() {} PegCount Module::get_global_count(char const*) const { return 0; } diff --git a/src/payload_injector/test/payload_injector_test.cc b/src/payload_injector/test/payload_injector_test.cc index 74d47a9c2..1e1bbb17c 100644 --- a/src/payload_injector/test/payload_injector_test.cc +++ b/src/payload_injector/test/payload_injector_test.cc @@ -28,6 +28,7 @@ #include "detection/detection_engine.h" #include "flow/flow.h" +#include "main/thread_config.h" #include "packet_io/active.h" #include "protocols/packet.h" #include "utils/util.h" @@ -61,6 +62,9 @@ DataBus::DataBus() = default; DataBus::~DataBus() = default; SnortConfig::SnortConfig(snort::SnortConfig const*, const char*) { } SnortConfig::~SnortConfig() = default; +unsigned get_instance_id() +{ return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } IpsContext ips_context; SnortConfig conf; diff --git a/src/search_engines/test/hyper_tool_test.cc b/src/search_engines/test/hyper_tool_test.cc index ef87faf92..a9afc3e6d 100644 --- a/src/search_engines/test/hyper_tool_test.cc +++ b/src/search_engines/test/hyper_tool_test.cc @@ -33,6 +33,7 @@ #include "framework/base_api.h" #include "framework/mpse_batch.h" #include "main/snort_config.h" +#include "main/thread_config.h" #include "managers/mpse_manager.h" #include "mpse_test_stubs.h" @@ -43,6 +44,12 @@ using namespace snort; +namespace snort +{ + unsigned get_instance_id() { return 0; } + unsigned ThreadConfig::get_instance_max() { return 1; } +} + //------------------------------------------------------------------------- // stubs, spies, etc. //------------------------------------------------------------------------- diff --git a/src/search_engines/test/hyperscan_test.cc b/src/search_engines/test/hyperscan_test.cc index 1b7aae4bf..a3f8c16c2 100644 --- a/src/search_engines/test/hyperscan_test.cc +++ b/src/search_engines/test/hyperscan_test.cc @@ -30,6 +30,7 @@ #include "framework/mpse.h" #include "framework/mpse_batch.h" #include "main/snort_config.h" +#include "main/thread_config.h" #include "mpse_test_stubs.h" @@ -39,6 +40,12 @@ using namespace snort; +namespace snort +{ + unsigned get_instance_id() { return 0; } + unsigned ThreadConfig::get_instance_max() { return 1; } +} + //------------------------------------------------------------------------- // stubs, spies, etc. //------------------------------------------------------------------------- diff --git a/src/search_engines/test/mpse_test_stubs.cc b/src/search_engines/test/mpse_test_stubs.cc index 85e98b3e7..74bc48349 100644 --- a/src/search_engines/test/mpse_test_stubs.cc +++ b/src/search_engines/test/mpse_test_stubs.cc @@ -76,9 +76,6 @@ void SnortConfig::release_scratch(int) DataBus::DataBus() = default; DataBus::~DataBus() = default; -unsigned get_instance_id() -{ return 0; } - THREAD_LOCAL PatMatQStat pmqs; unsigned parse_errors = 0; diff --git a/src/service_inspectors/http_inspect/test/http_module_test.cc b/src/service_inspectors/http_inspect/test/http_module_test.cc index b0a909342..60a977d00 100755 --- a/src/service_inspectors/http_inspect/test/http_module_test.cc +++ b/src/service_inspectors/http_inspect/test/http_module_test.cc @@ -26,6 +26,7 @@ #include "helpers/literal_search.h" #include "log/messages.h" +#include "main/thread_config.h" #include "service_inspectors/http_inspect/http_js_norm.h" #include "service_inspectors/http_inspect/http_module.h" #include "service_inspectors/http_inspect/http_str_to_code.h" @@ -60,6 +61,9 @@ void DecodeConfig::set_decompress_zip(bool) {} void DecodeConfig::set_decompress_vba(bool) {} SearchTool::~SearchTool() {} +unsigned get_instance_id() +{ return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } } void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { } diff --git a/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc b/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc index 02a5ffa58..c7f5dfc18 100755 --- a/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc +++ b/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc @@ -26,6 +26,7 @@ #include "helpers/literal_search.h" #include "log/messages.h" +#include "main/thread_config.h" #include "service_inspectors/http_inspect/http_js_norm.h" #include "service_inspectors/http_inspect/http_uri_norm.h" @@ -54,6 +55,9 @@ void DecodeConfig::set_decompress_swf(bool) {} void DecodeConfig::set_decompress_zip(bool) {} void DecodeConfig::set_decompress_vba(bool) {} SearchTool::~SearchTool() {} +unsigned get_instance_id() +{ return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } } snort::SearchTool* js_create_mpse_open_tag() { return nullptr; } diff --git a/src/side_channel/test/side_channel_module_test.cc b/src/side_channel/test/side_channel_module_test.cc index 188902e01..6c5ec1959 100644 --- a/src/side_channel/test/side_channel_module_test.cc +++ b/src/side_channel/test/side_channel_module_test.cc @@ -27,6 +27,7 @@ #include "side_channel/side_channel_module.h" #include "log/messages.h" +#include "main/thread_config.h" #include "profiler/profiler.h" #include @@ -64,6 +65,9 @@ void ParseWarning(WarningGroup, const char*, ...) { } char* snort_strdup(const char* s) { return strdup(s); } +unsigned get_instance_id() +{ return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } } TEST_GROUP(side_channel_module) diff --git a/src/trace/trace.cc b/src/trace/trace.cc index 2a8b220bc..c7ab4feab 100644 --- a/src/trace/trace.cc +++ b/src/trace/trace.cc @@ -97,7 +97,9 @@ Module::Module(const char* s, const char* h, const Parameter* p, bool l) : name( PegCount Module::get_global_count(char const*) const { return 0; } void Module::show_interval_stats(std::vector >&, FILE*) {} void Module::show_stats(){} +void Module::init_stats(bool){} void Module::sum_stats(bool){} +void Module::main_accumulate_stats(){} void Module::reset_stats() {} class TraceTestModule : public Module diff --git a/src/utils/stats.cc b/src/utils/stats.cc index b2f139d6a..bdf3ba82c 100644 --- a/src/utils/stats.cc +++ b/src/utils/stats.cc @@ -263,6 +263,7 @@ const PegInfo proc_names[] = void DropStats(ControlConn* ctrlcon) { s_ctrlcon = ctrlcon; + ModuleManager::accumulate_dump_stats(); LogLabel("Packet Statistics"); ModuleManager::get_module("daq")->show_stats(); PacketManager::dump_stats();