From: Masud Hasan (mashasan) Date: Tue, 19 Apr 2022 17:19:55 +0000 (+0000) Subject: Pull request #3378: Peg counts for bytes and number of items in use for various caches X-Git-Tag: 3.1.28.0~9 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9c38b8eba38fc092b03853b52a0a2311b3c51b3f;p=thirdparty%2Fsnort3.git Pull request #3378: Peg counts for bytes and number of items in use for various caches Merge in SNORT/snort3 from ~SMINUT/snort3:memory_pegs_now to master Squashed commit of the following: commit b229d5b046d97cba62377ea028f0a4892c1cd82a Author: Silviu Minut Date: Sun Apr 17 08:40:40 2022 -0400 module_manager: fix memory pegs display issue during packet processing, while also correctly computing the memory pegs in Analyzer::term commit ac3e69171f9a9dc5e13bbe416418893ed791f1ee Author: Silviu Minut Date: Mon Mar 28 18:04:19 2022 -0400 stream: add current_flows, uni_flows and uni_ip_flows peg counts commit 014af9aa055dffae340d7e789258535ba820cf40 Author: Silviu Minut Date: Thu Mar 24 20:54:28 2022 -0400 appid: add bytes_in_use and items_in_use peg counts commit b23c2063e089dfd6388bab6ff68737d9b94f706e Author: Silviu Minut Date: Thu Mar 24 19:35:12 2022 -0400 host_cache: bytes_in_use and items_in_use peg counts --- diff --git a/src/flow/flow_cache.cc b/src/flow/flow_cache.cc index f76c9071a..850c2ec29 100644 --- a/src/flow/flow_cache.cc +++ b/src/flow/flow_cache.cc @@ -115,15 +115,15 @@ void FlowCache::link_uni(Flow* flow) { if ( flow->key->pkt_type == PktType::IP ) { - debug_logf(stream_trace, TRACE_FLOW, nullptr, - "linking unidirectional flow (IP) to list of size: %u\n", + debug_logf(stream_trace, TRACE_FLOW, nullptr, + "linking unidirectional flow (IP) to list of size: %u\n", uni_ip_flows->get_count()); uni_ip_flows->link_uni(flow); } else { - debug_logf(stream_trace, TRACE_FLOW, nullptr, - "linking unidirectional flow (non-IP) to list of size: %u\n", + debug_logf(stream_trace, TRACE_FLOW, nullptr, + "linking unidirectional flow (non-IP) to list of size: %u\n", uni_flows->get_count()); uni_flows->link_uni(flow); } @@ -136,8 +136,8 @@ void FlowCache::unlink_uni(Flow* flow) { if ( uni_ip_flows->unlink_uni(flow) ) { - debug_logf(stream_trace, TRACE_FLOW, nullptr, - "unlinked unidirectional flow (IP) from list, size: %u\n", + debug_logf(stream_trace, TRACE_FLOW, nullptr, + "unlinked unidirectional flow (IP) from list, size: %u\n", uni_ip_flows->get_count()); } } @@ -145,7 +145,7 @@ void FlowCache::unlink_uni(Flow* flow) { if ( uni_flows->unlink_uni(flow) ) { - debug_logf(stream_trace, TRACE_FLOW, nullptr, + debug_logf(stream_trace, TRACE_FLOW, nullptr, "unlinked unidirectional flow (non-IP) from list, size: %u\n", uni_flows->get_count()); } @@ -526,3 +526,18 @@ unsigned FlowCache::purge() return retired; } + +size_t FlowCache::uni_flows_size() const +{ + return uni_flows ? uni_flows->get_count() : 0; +} + +size_t FlowCache::uni_ip_flows_size() const +{ + return uni_ip_flows ? uni_ip_flows->get_count() : 0; +} + +size_t FlowCache::flows_size() const +{ + return hash_table->get_num_nodes(); +} diff --git a/src/flow/flow_cache.h b/src/flow/flow_cache.h index 7c81da004..9854aac66 100644 --- a/src/flow/flow_cache.h +++ b/src/flow/flow_cache.h @@ -100,6 +100,10 @@ public: static bool is_pruning_in_progress() { return pruning_in_progress; } + size_t uni_flows_size() const; + size_t uni_ip_flows_size() const; + size_t flows_size() const; + private: void delete_uni(); void push(snort::Flow*); diff --git a/src/flow/flow_control.cc b/src/flow/flow_control.cc index ddbc45bbd..28decb08c 100644 --- a/src/flow/flow_control.cc +++ b/src/flow/flow_control.cc @@ -79,6 +79,16 @@ void FlowControl::clear_counts() num_flows = 0; } +PegCount FlowControl::get_uni_flows() const +{ return cache->uni_flows_size(); } + +PegCount FlowControl::get_uni_ip_flows() const +{ return cache->uni_ip_flows_size(); } + +PegCount FlowControl::get_num_flows() const +{ return cache->flows_size(); } + + //------------------------------------------------------------------------- // cache foo //------------------------------------------------------------------------- diff --git a/src/flow/flow_control.h b/src/flow/flow_control.h index fd87d67fc..d955239f4 100644 --- a/src/flow/flow_control.h +++ b/src/flow/flow_control.h @@ -94,6 +94,10 @@ public: PegCount get_deletes(FlowDeleteState state) const; void clear_counts(); + PegCount get_uni_flows() const; + PegCount get_uni_ip_flows() const; + PegCount get_num_flows() const; + private: void set_key(snort::FlowKey*, snort::Packet*); unsigned process(snort::Flow*, snort::Packet*); diff --git a/src/flow/test/flow_control_test.cc b/src/flow/test/flow_control_test.cc index 6512e53ac..b1fef5347 100644 --- a/src/flow/test/flow_control_test.cc +++ b/src/flow/test/flow_control_test.cc @@ -81,6 +81,9 @@ void FlowCache::push(Flow*) { } bool FlowCache::prune_one(PruneReason, bool) { return true; } unsigned FlowCache::delete_flows(unsigned) { return 0; } unsigned FlowCache::timeout(unsigned, time_t) { return 1; } +size_t FlowCache::uni_flows_size() const { return 0; } +size_t FlowCache::uni_ip_flows_size() const { return 0; } +size_t FlowCache::flows_size() const { return 0; } void Flow::init(PktType) { } void DataBus::publish(const char*, const uint8_t*, unsigned, Flow*) { } void DataBus::publish(const char*, Packet*, Flow*) { } diff --git a/src/hash/lru_cache_shared.cc b/src/hash/lru_cache_shared.cc index f30275d95..cde037e79 100644 --- a/src/hash/lru_cache_shared.cc +++ b/src/hash/lru_cache_shared.cc @@ -28,6 +28,8 @@ const PegInfo lru_cache_shared_peg_names[] = { { CountType::SUM, "adds", "lru cache added new entry" }, { CountType::SUM, "alloc_prunes", "lru cache pruned entry to make space for new entry" }, + { CountType::NOW, "bytes_in_use", "current number of bytes in use" }, + { CountType::NOW, "items_in_use", "current number of items in the cache" }, { CountType::SUM, "find_hits", "lru cache found entry in cache" }, { CountType::SUM, "find_misses", "lru cache did not find entry in cache" }, { CountType::SUM, "reload_prunes", "lru cache pruned entry for lower memcap during reload" }, diff --git a/src/hash/lru_cache_shared.h b/src/hash/lru_cache_shared.h index 7e2d9d13b..0f47dc86f 100644 --- a/src/hash/lru_cache_shared.h +++ b/src/hash/lru_cache_shared.h @@ -41,6 +41,8 @@ struct LruCacheSharedStats { PegCount adds = 0; // an insert that added new entry PegCount alloc_prunes = 0; // when an old entry is removed to make room for a new entry + PegCount bytes_in_use = 0; // current bytes in use + PegCount items_in_use = 0; // current items in cache PegCount find_hits = 0; // found entry in cache PegCount find_misses = 0; // did not find entry in cache PegCount reload_prunes = 0; // when an old entry is removed due to lower memcap during reload diff --git a/src/hash/test/lru_cache_shared_test.cc b/src/hash/test/lru_cache_shared_test.cc index 9d7f1abc0..1c5af92e5 100644 --- a/src/hash/test/lru_cache_shared_test.cc +++ b/src/hash/test/lru_cache_shared_test.cc @@ -195,19 +195,23 @@ TEST(lru_cache_shared, stats_test) CHECK(stats[0] == 10); // adds CHECK(stats[1] == 7); // alloc prunes - CHECK(stats[2] == 3); // find hits - CHECK(stats[3] == 12); // find misses - CHECK(stats[4] == 0); // reload prunes - CHECK(stats[5] == 1); // removes + CHECK(stats[2] == 0); // bytes in use + CHECK(stats[3] == 0); // items in use + CHECK(stats[4] == 3); // find hits + CHECK(stats[5] == 12); // find misses + CHECK(stats[6] == 0); // reload prunes + CHECK(stats[7] == 1); // removes // Check statistics names. const PegInfo* pegs = lru_cache.get_pegs(); CHECK(!strcmp(pegs[0].name, "adds")); CHECK(!strcmp(pegs[1].name, "alloc_prunes")); - CHECK(!strcmp(pegs[2].name, "find_hits")); - CHECK(!strcmp(pegs[3].name, "find_misses")); - CHECK(!strcmp(pegs[4].name, "reload_prunes")); - CHECK(!strcmp(pegs[5].name, "removes")); + CHECK(!strcmp(pegs[2].name, "bytes_in_use")); + CHECK(!strcmp(pegs[3].name, "items_in_use")); + CHECK(!strcmp(pegs[4].name, "find_hits")); + CHECK(!strcmp(pegs[5].name, "find_misses")); + CHECK(!strcmp(pegs[6].name, "reload_prunes")); + CHECK(!strcmp(pegs[7].name, "removes")); } int main(int argc, char** argv) diff --git a/src/host_tracker/host_cache_module.cc b/src/host_tracker/host_cache_module.cc index cca05b71d..a81341fe3 100644 --- a/src/host_tracker/host_cache_module.cc +++ b/src/host_tracker/host_cache_module.cc @@ -456,6 +456,9 @@ string HostCacheModule::get_host_cache_stats() host_cache.lock(); + host_cache.stats.bytes_in_use = host_cache.current_size; + host_cache.stats.items_in_use = host_cache.list.size(); + PegCount* counts = (PegCount*) host_cache.get_counts(); const PegInfo* pegs = host_cache.get_pegs(); @@ -483,6 +486,11 @@ PegCount* HostCacheModule::get_counts() const void HostCacheModule::sum_stats(bool accumulate_now_stats) { host_cache.lock(); + // These could be set in prep_counts but we set them here + // to save an extra cache lock. + host_cache.stats.bytes_in_use = host_cache.current_size; + host_cache.stats.items_in_use = host_cache.list.size(); + Module::sum_stats(accumulate_now_stats); host_cache.unlock(); } diff --git a/src/host_tracker/host_cache_module.h b/src/host_tracker/host_cache_module.h index 7fdc182aa..30e01ca55 100644 --- a/src/host_tracker/host_cache_module.h +++ b/src/host_tracker/host_cache_module.h @@ -63,6 +63,10 @@ public: PegCount* get_counts() const override; void sum_stats(bool) override; + // in sum_stats, just populate the counts vector with whatever we have now + bool global_stats() const override + { return true; } + Usage get_usage() const override { return GLOBAL; } diff --git a/src/host_tracker/test/host_cache_module_test.cc b/src/host_tracker/test/host_cache_module_test.cc index 3e46d2da6..9c1724dfd 100644 --- a/src/host_tracker/test/host_cache_module_test.cc +++ b/src/host_tracker/test/host_cache_module_test.cc @@ -109,12 +109,18 @@ TEST(host_cache_module, misc) CHECK(!strcmp(ht_pegs[0].name, "adds")); CHECK(!strcmp(ht_pegs[1].name, "alloc_prunes")); - CHECK(!strcmp(ht_pegs[2].name, "find_hits")); - CHECK(!strcmp(ht_pegs[3].name, "find_misses")); - CHECK(!strcmp(ht_pegs[4].name, "reload_prunes")); - CHECK(!strcmp(ht_pegs[5].name, "removes")); - CHECK(!strcmp(ht_pegs[6].name, "replaced")); - CHECK(!ht_pegs[7].name); + CHECK(!strcmp(ht_pegs[2].name, "bytes_in_use")); + CHECK(!strcmp(ht_pegs[3].name, "items_in_use")); + CHECK(!strcmp(ht_pegs[4].name, "find_hits")); + CHECK(!strcmp(ht_pegs[5].name, "find_misses")); + CHECK(!strcmp(ht_pegs[6].name, "reload_prunes")); + CHECK(!strcmp(ht_pegs[7].name, "removes")); + CHECK(!strcmp(ht_pegs[8].name, "replaced")); + CHECK(!ht_pegs[9].name); + + // call this to set up the counts vector, before inserting hosts into the + // cache, because sum_stats resets the pegs. + module.sum_stats(true); // add 3 entries SfIp ip1, ip2, ip3; @@ -124,24 +130,40 @@ TEST(host_cache_module, misc) host_cache.find_else_create(ip1, nullptr); host_cache.find_else_create(ip2, nullptr); host_cache.find_else_create(ip3, nullptr); + module.sum_stats(true); // does not call reset CHECK(ht_stats[0] == 3); + CHECK(ht_stats[2] == 1992); // bytes_in_use + CHECK(ht_stats[3] == 3); // items_in_use // no pruning needed for resizing higher than current size CHECK(host_cache.reload_resize(host_cache.mem_chunk * 10) == false); + module.sum_stats(true); + CHECK(ht_stats[2] == 1992); // bytes_in_use unchanged + CHECK(ht_stats[3] == 3); // items_in_use unchanged // pruning needed for resizing lower than current size CHECK(host_cache.reload_resize(host_cache.mem_chunk * 1.5) == true); + module.sum_stats(true); + CHECK(ht_stats[2] == 1992); // bytes_in_use still unchanged + CHECK(ht_stats[3] == 3); // items_in_use still unchanged // pruning in thread is not done when reload_mutex is already locked host_cache.reload_mutex.lock(); std::thread test_negative(try_reload_prune, false); test_negative.join(); host_cache.reload_mutex.unlock(); + module.sum_stats(true); + CHECK(ht_stats[2] == 1992); // no pruning yet + CHECK(ht_stats[3] == 3); // no pruning_yet // prune 2 entries in thread when reload_mutex is not locked std::thread test_positive(try_reload_prune, true); test_positive.join(); + module.sum_stats(true); + CHECK(ht_stats[2] == 664); + CHECK(ht_stats[3] == 1); // one left + // alloc_prune 1 entry host_cache.find_else_create(ip1, nullptr); @@ -149,12 +171,15 @@ TEST(host_cache_module, misc) host_cache.find_else_create(ip1, nullptr); host_cache.remove(ip1); + module.sum_stats(true); CHECK(ht_stats[0] == 4); // 4 adds CHECK(ht_stats[1] == 1); // 1 alloc_prunes - CHECK(ht_stats[2] == 1); // 1 hit - CHECK(ht_stats[3] == 4); // 4 misses - CHECK(ht_stats[4] == 2); // 2 reload_prunes - CHECK(ht_stats[5] == 1); // 1 remove + CHECK(ht_stats[2] == 0); // 0 bytes_in_use + CHECK(ht_stats[3] == 0); // 0 items_in_use + CHECK(ht_stats[4] == 1); // 1 hit + CHECK(ht_stats[5] == 4); // 4 misses + CHECK(ht_stats[6] == 2); // 2 reload_prunes + CHECK(ht_stats[7] == 1); // 1 remove ht_stats = module.get_counts(); CHECK(ht_stats[0] == 4); diff --git a/src/main/analyzer.cc b/src/main/analyzer.cc index 46cb06fc0..cc5297bff 100644 --- a/src/main/analyzer.cc +++ b/src/main/analyzer.cc @@ -654,7 +654,7 @@ void Analyzer::term() DetectionEngine::idle(); InspectorManager::thread_stop(sc); InspectorManager::thread_term(); - ModuleManager::accumulate(); + ModuleManager::accumulate("memory"); ActionManager::thread_term(); IpsManager::clear_options(sc); diff --git a/src/main/test/distill_verdict_stubs.h b/src/main/test/distill_verdict_stubs.h index 45c8c297e..dd4a9851e 100644 --- a/src/main/test/distill_verdict_stubs.h +++ b/src/main/test/distill_verdict_stubs.h @@ -132,7 +132,7 @@ MemoryContext::MemoryContext(MemoryTracker&) : saved(nullptr) { } MemoryContext::~MemoryContext() = default; Packet::Packet(bool) { - memset(this , 0, sizeof(*this)); + memset((char*) this , 0, sizeof(*this)); ip_proto_next = IpProtocol::PROTO_NOT_SET; packet_flags = PKT_FROM_CLIENT; } @@ -206,7 +206,7 @@ void InspectorManager::thread_term() { } void InspectorManager::thread_stop(const SnortConfig*) { } void InspectorManager::thread_reinit(const SnortConfig*) { } void InspectorManager::thread_stop_removed(const SnortConfig*) { } -void ModuleManager::accumulate() { } +void ModuleManager::accumulate(const char*) { } void ModuleManager::accumulate_module(const char*) { } void Stream::handle_timeouts(bool) { } void Stream::purge_flows() { } diff --git a/src/managers/module_manager.cc b/src/managers/module_manager.cc index 357c95f24..044c38bd5 100644 --- a/src/managers/module_manager.cc +++ b/src/managers/module_manager.cc @@ -1394,13 +1394,13 @@ void ModuleManager::dump_stats(const char* skip, bool dynamic) } } -void ModuleManager::accumulate() +void ModuleManager::accumulate(const char* except) { auto mod_hooks = get_all_modhooks(); for ( auto* mh : mod_hooks ) { - if ( !strcmp(mh->mod->name, "memory") ) + if ( except and !strcmp(mh->mod->name, except) ) continue; lock_guard lock(stats_mutex); diff --git a/src/managers/module_manager.h b/src/managers/module_manager.h index 2dcd3f79e..dd5877327 100644 --- a/src/managers/module_manager.h +++ b/src/managers/module_manager.h @@ -85,7 +85,7 @@ public: static void dump_stats(const char* skip = nullptr, bool dynamic = false); - static void accumulate(); + static void accumulate(const char* except = nullptr); static void accumulate_module(const char* name); static void reset_stats(SnortConfig*); static void reset_stats(clear_counter_type_t); diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index 79c93716a..3e74e34f3 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -430,6 +430,8 @@ static const PegInfo appid_pegs[] = { CountType::SUM, "service_cache_removes", "number of times an item was removed from the service cache" }, { CountType::SUM, "odp_reload_ignored_pkts", "count of packets ignored after open detector package is reloaded" }, { CountType::SUM, "tp_reload_ignored_pkts", "count of packets ignored after third-party module is reloaded" }, + { CountType::NOW, "bytes_in_use", "number of bytes in use in the cache" }, + { CountType::NOW, "items_in_use", "items in use in the cache" }, { CountType::END, nullptr, nullptr }, }; diff --git a/src/network_inspectors/appid/appid_peg_counts.h b/src/network_inspectors/appid/appid_peg_counts.h index 9fb148284..1aa1fc37a 100644 --- a/src/network_inspectors/appid/appid_peg_counts.h +++ b/src/network_inspectors/appid/appid_peg_counts.h @@ -53,6 +53,8 @@ struct AppIdStats PegCount service_cache_removes; PegCount odp_reload_ignored_pkts; PegCount tp_reload_ignored_pkts; + PegCount bytes_in_use; + PegCount items_in_use; }; class AppIdPegCounts diff --git a/src/network_inspectors/appid/service_state.h b/src/network_inspectors/appid/service_state.h index 37171913b..3dc07645d 100644 --- a/src/network_inspectors/appid/service_state.h +++ b/src/network_inspectors/appid/service_state.h @@ -213,6 +213,8 @@ public: mem_used += sz; ss->qptr = --q.end(); // remember our place in the queue appid_stats.service_cache_adds++; + appid_stats.bytes_in_use = mem_used; + appid_stats.items_in_use = m.size(); if ( mem_used > memcap ) remove( q.front() ); @@ -248,6 +250,8 @@ public: delete it->second; m.erase(it); // then from cache appid_stats.service_cache_removes++; + appid_stats.bytes_in_use = mem_used; + appid_stats.items_in_use = m.size(); return true; } diff --git a/src/stream/base/stream_base.cc b/src/stream/base/stream_base.cc index 963ac2ea3..6452ba7b6 100644 --- a/src/stream/base/stream_base.cc +++ b/src/stream/base/stream_base.cc @@ -75,6 +75,9 @@ const PegInfo base_pegs[] = { CountType::SUM, "reload_allowed_deletes", "number of allowed flows deleted by config reloads" }, { CountType::SUM, "reload_blocked_deletes", "number of blocked flows deleted by config reloads" }, { CountType::SUM, "reload_offloaded_deletes", "number of offloaded flows deleted by config reloads" }, + { CountType::NOW, "current_flows", "current number of flows in cache" }, + { CountType::NOW, "uni_flows", "number of uni flows in cache" }, + { CountType::NOW, "uni_ip_flows", "number of uni ip flows in cache" }, { CountType::END, nullptr, nullptr } }; @@ -96,6 +99,9 @@ void base_prep() stream_base_stats.reload_allowed_flow_deletes = flow_con->get_deletes(FlowDeleteState::ALLOWED); stream_base_stats.reload_offloaded_flow_deletes= flow_con->get_deletes(FlowDeleteState::OFFLOADED); stream_base_stats.reload_blocked_flow_deletes= flow_con->get_deletes(FlowDeleteState::BLOCKED); + stream_base_stats.current_flows = flow_con->get_num_flows(); + stream_base_stats.uni_flows = flow_con->get_uni_flows(); + stream_base_stats.uni_ip_flows = flow_con->get_uni_ip_flows(); ExpectCache* exp_cache = flow_con->get_exp_cache(); if ( exp_cache ) diff --git a/src/stream/base/stream_module.h b/src/stream/base/stream_module.h index 2023dd11f..a5341b160 100644 --- a/src/stream/base/stream_module.h +++ b/src/stream/base/stream_module.h @@ -74,6 +74,10 @@ struct BaseStats PegCount reload_allowed_flow_deletes; PegCount reload_blocked_flow_deletes; PegCount reload_offloaded_flow_deletes; + PegCount current_flows; + PegCount uni_flows; + PegCount uni_ip_flows; + }; extern const PegInfo base_pegs[];