]> git.ipfire.org Git - thirdparty/snort3.git/commitdiff
Pull request #3378: Peg counts for bytes and number of items in use for various caches
authorMasud Hasan (mashasan) <mashasan@cisco.com>
Tue, 19 Apr 2022 17:19:55 +0000 (17:19 +0000)
committerMasud Hasan (mashasan) <mashasan@cisco.com>
Tue, 19 Apr 2022 17:19:55 +0000 (17:19 +0000)
Merge in SNORT/snort3 from ~SMINUT/snort3:memory_pegs_now to master

Squashed commit of the following:

commit b229d5b046d97cba62377ea028f0a4892c1cd82a
Author: Silviu Minut <sminut@cisco.com>
Date:   Sun Apr 17 08:40:40 2022 -0400

    module_manager: fix memory pegs display issue during packet processing, while also correctly computing the memory pegs in Analyzer::term

commit ac3e69171f9a9dc5e13bbe416418893ed791f1ee
Author: Silviu Minut <sminut@cisco.com>
Date:   Mon Mar 28 18:04:19 2022 -0400

    stream: add current_flows, uni_flows and uni_ip_flows peg counts

commit 014af9aa055dffae340d7e789258535ba820cf40
Author: Silviu Minut <sminut@cisco.com>
Date:   Thu Mar 24 20:54:28 2022 -0400

    appid: add bytes_in_use and items_in_use peg counts

commit b23c2063e089dfd6388bab6ff68737d9b94f706e
Author: Silviu Minut <sminut@cisco.com>
Date:   Thu Mar 24 19:35:12 2022 -0400

    host_cache: bytes_in_use and items_in_use peg counts

20 files changed:
src/flow/flow_cache.cc
src/flow/flow_cache.h
src/flow/flow_control.cc
src/flow/flow_control.h
src/flow/test/flow_control_test.cc
src/hash/lru_cache_shared.cc
src/hash/lru_cache_shared.h
src/hash/test/lru_cache_shared_test.cc
src/host_tracker/host_cache_module.cc
src/host_tracker/host_cache_module.h
src/host_tracker/test/host_cache_module_test.cc
src/main/analyzer.cc
src/main/test/distill_verdict_stubs.h
src/managers/module_manager.cc
src/managers/module_manager.h
src/network_inspectors/appid/appid_module.cc
src/network_inspectors/appid/appid_peg_counts.h
src/network_inspectors/appid/service_state.h
src/stream/base/stream_base.cc
src/stream/base/stream_module.h

index f76c9071a05321494acb0b5bf11180e82d9e4e00..850c2ec2905db6526e09ed574df527a6c4750804 100644 (file)
@@ -115,15 +115,15 @@ void FlowCache::link_uni(Flow* flow)
 {
     if ( flow->key->pkt_type == PktType::IP )
     {
-        debug_logf(stream_trace, TRACE_FLOW, nullptr, 
-            "linking unidirectional flow (IP) to list of size: %u\n", 
+        debug_logf(stream_trace, TRACE_FLOW, nullptr,
+            "linking unidirectional flow (IP) to list of size: %u\n",
             uni_ip_flows->get_count());
         uni_ip_flows->link_uni(flow);
     }
     else
     {
-        debug_logf(stream_trace, TRACE_FLOW, nullptr, 
-            "linking unidirectional flow (non-IP) to list of size: %u\n", 
+        debug_logf(stream_trace, TRACE_FLOW, nullptr,
+            "linking unidirectional flow (non-IP) to list of size: %u\n",
             uni_flows->get_count());
         uni_flows->link_uni(flow);
     }
@@ -136,8 +136,8 @@ void FlowCache::unlink_uni(Flow* flow)
     {
         if ( uni_ip_flows->unlink_uni(flow) )
         {
-            debug_logf(stream_trace, TRACE_FLOW, nullptr, 
-                "unlinked unidirectional flow (IP) from list, size: %u\n", 
+            debug_logf(stream_trace, TRACE_FLOW, nullptr,
+                "unlinked unidirectional flow (IP) from list, size: %u\n",
                 uni_ip_flows->get_count());
         }
     }
@@ -145,7 +145,7 @@ void FlowCache::unlink_uni(Flow* flow)
     {
         if ( uni_flows->unlink_uni(flow) )
         {
-            debug_logf(stream_trace, TRACE_FLOW, nullptr, 
+            debug_logf(stream_trace, TRACE_FLOW, nullptr,
                 "unlinked unidirectional flow (non-IP) from list, size: %u\n",
                 uni_flows->get_count());
         }
@@ -526,3 +526,18 @@ unsigned FlowCache::purge()
 
     return retired;
 }
+
+size_t FlowCache::uni_flows_size() const
+{
+    return uni_flows ? uni_flows->get_count() : 0;
+}
+
+size_t FlowCache::uni_ip_flows_size() const
+{
+    return uni_ip_flows ? uni_ip_flows->get_count() : 0;
+}
+
+size_t FlowCache::flows_size() const
+{
+    return hash_table->get_num_nodes();
+}
index 7c81da004c7fe4d680f0a8ae82c0079fe69fdf37..9854aac66560a9171d2dada932ef4b7662c379a8 100644 (file)
@@ -100,6 +100,10 @@ public:
     static bool is_pruning_in_progress()
     { return pruning_in_progress; }
 
+    size_t uni_flows_size() const;
+    size_t uni_ip_flows_size() const;
+    size_t flows_size() const;
+
 private:
     void delete_uni();
     void push(snort::Flow*);
index ddbc45bbd8f2db828416a2e5cf02c1c105d1f9fe..28decb08cf73e40d4276d7fcf9044b2e88be8b7f 100644 (file)
@@ -79,6 +79,16 @@ void FlowControl::clear_counts()
     num_flows = 0;
 }
 
+PegCount FlowControl::get_uni_flows() const
+{ return cache->uni_flows_size(); }
+
+PegCount FlowControl::get_uni_ip_flows() const
+{ return cache->uni_ip_flows_size(); }
+
+PegCount FlowControl::get_num_flows() const
+{ return cache->flows_size(); }
+
+
 //-------------------------------------------------------------------------
 // cache foo
 //-------------------------------------------------------------------------
index fd87d67fcb7f29c28396131673c5e10ede07827d..d955239f48c096673221c5931c2ecbf97a76d814 100644 (file)
@@ -94,6 +94,10 @@ public:
     PegCount get_deletes(FlowDeleteState state) const;
     void clear_counts();
 
+    PegCount get_uni_flows() const;
+    PegCount get_uni_ip_flows() const;
+    PegCount get_num_flows() const;
+
 private:
     void set_key(snort::FlowKey*, snort::Packet*);
     unsigned process(snort::Flow*, snort::Packet*);
index 6512e53ace91a83216499318329535d309ca3be8..b1fef5347bc157dcd2d092c1d0cfcd911adaee93 100644 (file)
@@ -81,6 +81,9 @@ void FlowCache::push(Flow*) { }
 bool FlowCache::prune_one(PruneReason, bool) { return true; }
 unsigned FlowCache::delete_flows(unsigned) { return 0; }
 unsigned FlowCache::timeout(unsigned, time_t) { return 1; }
+size_t FlowCache::uni_flows_size() const { return 0; }
+size_t FlowCache::uni_ip_flows_size() const { return 0; }
+size_t FlowCache::flows_size() const { return 0; }
 void Flow::init(PktType) { }
 void DataBus::publish(const char*, const uint8_t*, unsigned, Flow*) { }
 void DataBus::publish(const char*, Packet*, Flow*) { }
index f30275d9584ebeeef0f30422d06931b8d0655335..cde037e79ba12f50c11d8e6ae67ac6489ec5d026 100644 (file)
@@ -28,6 +28,8 @@ const PegInfo lru_cache_shared_peg_names[] =
 {
     { CountType::SUM, "adds", "lru cache added new entry" },
     { CountType::SUM, "alloc_prunes", "lru cache pruned entry to make space for new entry" },
+    { CountType::NOW, "bytes_in_use", "current number of bytes in use" },
+    { CountType::NOW, "items_in_use", "current number of items in the cache" },
     { CountType::SUM, "find_hits", "lru cache found entry in cache" },
     { CountType::SUM, "find_misses", "lru cache did not find entry in cache" },
     { CountType::SUM, "reload_prunes", "lru cache pruned entry for lower memcap during reload" },
index 7e2d9d13b87913305db521e971ca03917addfede..0f47dc86fc08fd7953d3d54df504c168f9067bbc 100644 (file)
@@ -41,6 +41,8 @@ struct LruCacheSharedStats
 {
     PegCount adds = 0;          // an insert that added new entry
     PegCount alloc_prunes = 0;  // when an old entry is removed to make room for a new entry
+    PegCount bytes_in_use = 0;  // current bytes in use
+    PegCount items_in_use = 0;  // current items in cache
     PegCount find_hits = 0;     // found entry in cache
     PegCount find_misses = 0;   // did not find entry in cache
     PegCount reload_prunes = 0; // when an old entry is removed due to lower memcap during reload
index 9d7f1abc0883ab5cecbe434a16f412466397e668..1c5af92e598f77c0a8b0f3f9d3ce317c02a86230 100644 (file)
@@ -195,19 +195,23 @@ TEST(lru_cache_shared, stats_test)
 
     CHECK(stats[0] == 10);  //  adds
     CHECK(stats[1] == 7);   //  alloc prunes
-    CHECK(stats[2] == 3);   //  find hits
-    CHECK(stats[3] == 12);  //  find misses
-    CHECK(stats[4] == 0);   //  reload prunes
-    CHECK(stats[5] == 1);   //  removes
+    CHECK(stats[2] == 0);   //  bytes in use
+    CHECK(stats[3] == 0);   //  items in use
+    CHECK(stats[4] == 3);   //  find hits
+    CHECK(stats[5] == 12);  //  find misses
+    CHECK(stats[6] == 0);   //  reload prunes
+    CHECK(stats[7] == 1);   //  removes
 
     // Check statistics names.
     const PegInfo* pegs = lru_cache.get_pegs();
     CHECK(!strcmp(pegs[0].name, "adds"));
     CHECK(!strcmp(pegs[1].name, "alloc_prunes"));
-    CHECK(!strcmp(pegs[2].name, "find_hits"));
-    CHECK(!strcmp(pegs[3].name, "find_misses"));
-    CHECK(!strcmp(pegs[4].name, "reload_prunes"));
-    CHECK(!strcmp(pegs[5].name, "removes"));
+    CHECK(!strcmp(pegs[2].name, "bytes_in_use"));
+    CHECK(!strcmp(pegs[3].name, "items_in_use"));
+    CHECK(!strcmp(pegs[4].name, "find_hits"));
+    CHECK(!strcmp(pegs[5].name, "find_misses"));
+    CHECK(!strcmp(pegs[6].name, "reload_prunes"));
+    CHECK(!strcmp(pegs[7].name, "removes"));
 }
 
 int main(int argc, char** argv)
index cca05b71d8172f031b8f3460a89323d64fe6a304..a81341fe3f291d1687b17ce44c93012174b12197 100644 (file)
@@ -456,6 +456,9 @@ string HostCacheModule::get_host_cache_stats()
 
     host_cache.lock();
 
+    host_cache.stats.bytes_in_use = host_cache.current_size;
+    host_cache.stats.items_in_use = host_cache.list.size();
+
     PegCount* counts = (PegCount*) host_cache.get_counts();
     const PegInfo* pegs = host_cache.get_pegs();
 
@@ -483,6 +486,11 @@ PegCount* HostCacheModule::get_counts() const
 void HostCacheModule::sum_stats(bool accumulate_now_stats)
 {
     host_cache.lock();
+    // These could be set in prep_counts but we set them here
+    // to save an extra cache lock.
+    host_cache.stats.bytes_in_use = host_cache.current_size;
+    host_cache.stats.items_in_use = host_cache.list.size();
+
     Module::sum_stats(accumulate_now_stats);
     host_cache.unlock();
 }
index 7fdc182aa8e159d32e12d9c179a6f86e482ba4ec..30e01ca551bf5c68857a71b7346b4cbfcf75e946 100644 (file)
@@ -63,6 +63,10 @@ public:
     PegCount* get_counts() const override;
     void sum_stats(bool) override;
 
+    // in sum_stats, just populate the counts vector with whatever we have now
+    bool global_stats() const override
+    { return true; }
+
     Usage get_usage() const override
     { return GLOBAL; }
 
index 3e46d2da66841aa5af695a3e3bf69af8bfcef4bf..9c1724dfd5881a54b54d1a6a4f0ff7b73d5ee7fc 100644 (file)
@@ -109,12 +109,18 @@ TEST(host_cache_module, misc)
 
     CHECK(!strcmp(ht_pegs[0].name, "adds"));
     CHECK(!strcmp(ht_pegs[1].name, "alloc_prunes"));
-    CHECK(!strcmp(ht_pegs[2].name, "find_hits"));
-    CHECK(!strcmp(ht_pegs[3].name, "find_misses"));
-    CHECK(!strcmp(ht_pegs[4].name, "reload_prunes"));
-    CHECK(!strcmp(ht_pegs[5].name, "removes"));
-    CHECK(!strcmp(ht_pegs[6].name, "replaced"));
-    CHECK(!ht_pegs[7].name);
+    CHECK(!strcmp(ht_pegs[2].name, "bytes_in_use"));
+    CHECK(!strcmp(ht_pegs[3].name, "items_in_use"));
+    CHECK(!strcmp(ht_pegs[4].name, "find_hits"));
+    CHECK(!strcmp(ht_pegs[5].name, "find_misses"));
+    CHECK(!strcmp(ht_pegs[6].name, "reload_prunes"));
+    CHECK(!strcmp(ht_pegs[7].name, "removes"));
+    CHECK(!strcmp(ht_pegs[8].name, "replaced"));
+    CHECK(!ht_pegs[9].name);
+
+    // call this to set up the counts vector, before inserting hosts into the
+    // cache, because sum_stats resets the pegs.
+    module.sum_stats(true);
 
     // add 3 entries
     SfIp ip1, ip2, ip3;
@@ -124,24 +130,40 @@ TEST(host_cache_module, misc)
     host_cache.find_else_create(ip1, nullptr);
     host_cache.find_else_create(ip2, nullptr);
     host_cache.find_else_create(ip3, nullptr);
+    module.sum_stats(true);      // does not call reset
     CHECK(ht_stats[0] == 3);
+    CHECK(ht_stats[2] == 1992);  // bytes_in_use
+    CHECK(ht_stats[3] == 3);     // items_in_use
 
     // no pruning needed for resizing higher than current size
     CHECK(host_cache.reload_resize(host_cache.mem_chunk * 10) == false);
+    module.sum_stats(true);
+    CHECK(ht_stats[2] == 1992);  // bytes_in_use unchanged
+    CHECK(ht_stats[3] == 3);     // items_in_use unchanged
 
     // pruning needed for resizing lower than current size
     CHECK(host_cache.reload_resize(host_cache.mem_chunk * 1.5) == true);
+    module.sum_stats(true);
+    CHECK(ht_stats[2] == 1992);  // bytes_in_use still unchanged
+    CHECK(ht_stats[3] == 3);     // items_in_use still unchanged
 
     // pruning in thread is not done when reload_mutex is already locked
     host_cache.reload_mutex.lock();
     std::thread test_negative(try_reload_prune, false);
     test_negative.join();
     host_cache.reload_mutex.unlock();
+    module.sum_stats(true);
+    CHECK(ht_stats[2] == 1992);   // no pruning yet
+    CHECK(ht_stats[3] == 3);      // no pruning_yet
 
     // prune 2 entries in thread when reload_mutex is not locked
     std::thread test_positive(try_reload_prune, true);
     test_positive.join();
 
+    module.sum_stats(true);
+    CHECK(ht_stats[2] == 664);
+    CHECK(ht_stats[3] == 1);     // one left
+
     // alloc_prune 1 entry
     host_cache.find_else_create(ip1, nullptr);
 
@@ -149,12 +171,15 @@ TEST(host_cache_module, misc)
     host_cache.find_else_create(ip1, nullptr);
     host_cache.remove(ip1);
 
+    module.sum_stats(true);
     CHECK(ht_stats[0] == 4); // 4 adds
     CHECK(ht_stats[1] == 1); // 1 alloc_prunes
-    CHECK(ht_stats[2] == 1); // 1 hit
-    CHECK(ht_stats[3] == 4); // 4 misses
-    CHECK(ht_stats[4] == 2); // 2 reload_prunes
-    CHECK(ht_stats[5] == 1); // 1 remove
+    CHECK(ht_stats[2] == 0); // 0 bytes_in_use
+    CHECK(ht_stats[3] == 0); // 0 items_in_use
+    CHECK(ht_stats[4] == 1); // 1 hit
+    CHECK(ht_stats[5] == 4); // 4 misses
+    CHECK(ht_stats[6] == 2); // 2 reload_prunes
+    CHECK(ht_stats[7] == 1); // 1 remove
 
     ht_stats = module.get_counts();
     CHECK(ht_stats[0] == 4);
index 46cb06fc0817cfd999ff941b84a4fed5d055dff9..cc5297bff4b259ac97aba84a39263da630a217a8 100644 (file)
@@ -654,7 +654,7 @@ void Analyzer::term()
     DetectionEngine::idle();
     InspectorManager::thread_stop(sc);
     InspectorManager::thread_term();
-    ModuleManager::accumulate();
+    ModuleManager::accumulate("memory");
     ActionManager::thread_term();
 
     IpsManager::clear_options(sc);
index 45c8c297e0e938096c0da685c68f018f11a38c08..dd4a9851e7a016cc08ceec89088a9285dcd0e72a 100644 (file)
@@ -132,7 +132,7 @@ MemoryContext::MemoryContext(MemoryTracker&) : saved(nullptr) { }
 MemoryContext::~MemoryContext() = default;
 Packet::Packet(bool)
 {
-    memset(this , 0, sizeof(*this));
+    memset((char*) this , 0, sizeof(*this));
     ip_proto_next = IpProtocol::PROTO_NOT_SET;
     packet_flags = PKT_FROM_CLIENT;
 }
@@ -206,7 +206,7 @@ void InspectorManager::thread_term() { }
 void InspectorManager::thread_stop(const SnortConfig*) { }
 void InspectorManager::thread_reinit(const SnortConfig*) { }
 void InspectorManager::thread_stop_removed(const SnortConfig*) { }
-void ModuleManager::accumulate() { }
+void ModuleManager::accumulate(const char*) { }
 void ModuleManager::accumulate_module(const char*) { }
 void Stream::handle_timeouts(bool) { }
 void Stream::purge_flows() { }
index 357c95f246f351e80ed88265435ffd5b407f698d..044c38bd531b44e4709c5f2c60840829fa46b11c 100644 (file)
@@ -1394,13 +1394,13 @@ void ModuleManager::dump_stats(const char* skip, bool dynamic)
     }
 }
 
-void ModuleManager::accumulate()
+void ModuleManager::accumulate(const char* except)
 {
     auto mod_hooks = get_all_modhooks();
 
     for ( auto* mh : mod_hooks )
     {
-        if ( !strcmp(mh->mod->name, "memory") )
+        if ( except and !strcmp(mh->mod->name, except) )
             continue;
 
         lock_guard<mutex> lock(stats_mutex);
index 2dcd3f79e3a525cbd0bf7091a0c582465bc1bff6..dd5877327b9c7f94770bb9595241250f6b1204cc 100644 (file)
@@ -85,7 +85,7 @@ public:
 
     static void dump_stats(const char* skip = nullptr, bool dynamic = false);
 
-    static void accumulate();
+    static void accumulate(const char* except = nullptr);
     static void accumulate_module(const char* name);
     static void reset_stats(SnortConfig*);
     static void reset_stats(clear_counter_type_t);
index 79c93716a0832c19b7cbab8757a772200ed16f91..3e74e34f30ab410a27a62e67a1461473b5d6ad51 100644 (file)
@@ -430,6 +430,8 @@ static const PegInfo appid_pegs[] =
     { CountType::SUM, "service_cache_removes", "number of times an item was removed from the service cache" },
     { CountType::SUM, "odp_reload_ignored_pkts", "count of packets ignored after open detector package is reloaded" },
     { CountType::SUM, "tp_reload_ignored_pkts", "count of packets ignored after third-party module is reloaded" },
+    { CountType::NOW, "bytes_in_use", "number of bytes in use in the cache" },
+    { CountType::NOW, "items_in_use", "items in use in the cache" },
     { CountType::END, nullptr, nullptr },
 };
 
index 9fb148284a697a8da25d588a29f2a063546b50c9..1aa1fc37a2428bf9ca3dd067744c71f4a33a0b04 100644 (file)
@@ -53,6 +53,8 @@ struct AppIdStats
     PegCount service_cache_removes;
     PegCount odp_reload_ignored_pkts;
     PegCount tp_reload_ignored_pkts;
+    PegCount bytes_in_use;
+    PegCount items_in_use;
 };
 
 class AppIdPegCounts
index 37171913b8accddcbcc9450cc6f013075dc7055d..3dc07645da95278f8c2d675f777a57e47eebd87b 100644 (file)
@@ -213,6 +213,8 @@ public:
             mem_used += sz;
             ss->qptr = --q.end(); // remember our place in the queue
             appid_stats.service_cache_adds++;
+            appid_stats.bytes_in_use = mem_used;
+            appid_stats.items_in_use = m.size();
 
             if ( mem_used > memcap )
                 remove( q.front() );
@@ -248,6 +250,8 @@ public:
             delete it->second;
             m.erase(it);                // then from cache
             appid_stats.service_cache_removes++;
+            appid_stats.bytes_in_use = mem_used;
+            appid_stats.items_in_use = m.size();
 
             return true;
         }
index 963ac2ea3ec472aa0db6852caf4acf9d69426e02..6452ba7b6f6798f2a43fb3be55a35ab77852f4f9 100644 (file)
@@ -75,6 +75,9 @@ const PegInfo base_pegs[] =
     { CountType::SUM, "reload_allowed_deletes", "number of allowed flows deleted by config reloads" },
     { CountType::SUM, "reload_blocked_deletes", "number of blocked flows deleted by config reloads" },
     { CountType::SUM, "reload_offloaded_deletes", "number of offloaded flows deleted by config reloads" },
+    { CountType::NOW, "current_flows", "current number of flows in cache" },
+    { CountType::NOW, "uni_flows", "number of uni flows in cache" },
+    { CountType::NOW, "uni_ip_flows", "number of uni ip flows in cache" },
     { CountType::END, nullptr, nullptr }
 };
 
@@ -96,6 +99,9 @@ void base_prep()
     stream_base_stats.reload_allowed_flow_deletes = flow_con->get_deletes(FlowDeleteState::ALLOWED);
     stream_base_stats.reload_offloaded_flow_deletes= flow_con->get_deletes(FlowDeleteState::OFFLOADED);
     stream_base_stats.reload_blocked_flow_deletes= flow_con->get_deletes(FlowDeleteState::BLOCKED);
+    stream_base_stats.current_flows = flow_con->get_num_flows();
+    stream_base_stats.uni_flows = flow_con->get_uni_flows();
+    stream_base_stats.uni_ip_flows = flow_con->get_uni_ip_flows();
     ExpectCache* exp_cache = flow_con->get_exp_cache();
 
     if ( exp_cache )
index 2023dd11fa102dd2b1d6986f7898a91685c683e9..a5341b1604cffb070931b0a8b49fdf192aebe8dc 100644 (file)
@@ -74,6 +74,10 @@ struct BaseStats
      PegCount reload_allowed_flow_deletes;
      PegCount reload_blocked_flow_deletes;
      PegCount reload_offloaded_flow_deletes;
+     PegCount current_flows;
+     PegCount uni_flows;
+     PegCount uni_ip_flows;
+
 };
 
 extern const PegInfo base_pegs[];