Pull request #3814: Forward-port: (master) add extra jemalloc stats

Merge in SNORT/snort3 from ~MMATIRKO/snort3:mem_counts_master to master

Squashed commit of the following:

commit 9a5d8dabaf88dadbe29cd01b54602b5631b1a9bd
Author: Russ Combs <rucombs@cisco.com>
Date:   Wed Mar 15 14:22:22 2023 -0400

    memory: add extra jemalloc counts for tracking

commit 1c078c5fa8c4fd0a99469677269d92f7b7837891
Author: Russ Combs <rucombs@cisco.com>
Date:   Tue Mar 14 22:24:37 2023 -0400

    memory: use jemalloc stats.mapped for process total

diff --git a/src/memory/heap_interface.cc b/src/memory/heap_interface.cc
index d4542bde3336437ec357651c03148f6e2d27de9c..70132b741132f35a32e6af1163432ae207f10063 100644 (file)
--- a/src/memory/heap_interface.cc
+++ b/src/memory/heap_interface.cc
@@ -52,6 +52,7 @@ class JemallocInterface : public HeapInterface
 
     void print_stats(ControlConn*) override;
 
+    void get_aux_counts(uint64_t&, uint64_t&, uint64_t&, uint64_t&) override;
 };
 
 static size_t stats_mib[2], mib_len = 2;
@@ -81,7 +82,7 @@ static void log_jem_stats(void *,const char *buf)
 
 void JemallocInterface::main_init()
 {
-    mallctlnametomib("stats.allocated", stats_mib, &mib_len);
+    mallctlnametomib("stats.mapped", stats_mib, &mib_len);
 }
 
 void JemallocInterface::thread_init()
@@ -122,6 +123,16 @@ void JemallocInterface::print_stats(ControlConn* ctrlcon)
     malloc_stats_print(log_jem_stats, nullptr, nullptr);
 }
 
+void JemallocInterface::get_aux_counts(uint64_t& all, uint64_t& act, uint64_t& res, uint64_t& ret)
+{
+    size_t sz = sizeof(all);
+
+    mallctl("stats.allocated", (void*)&all, &sz, nullptr, 0);
+    mallctl("stats.active", (void*)&act, &sz, nullptr, 0);
+    mallctl("stats.resident", (void*)&res, &sz, nullptr, 0);
+    mallctl("stats.retained", (void*)&ret, &sz, nullptr, 0);
+}
+
 //--------------------------------------------------------------------------
 #else  // disabled interface
 //--------------------------------------------------------------------------

diff --git a/src/memory/heap_interface.h b/src/memory/heap_interface.h
index 070c28647774d04ba02c8a5930ce6036690e926e..ffa03fc140b47b0dadabf5bcdc333dc81c011265 100644 (file)
--- a/src/memory/heap_interface.h
+++ b/src/memory/heap_interface.h
@@ -39,6 +39,9 @@ public:
     virtual void get_thread_allocs(uint64_t& alloc, uint64_t& dealloc) = 0;
 
     virtual void print_stats(ControlConn*) { }
+    virtual void get_aux_counts(uint64_t& app_all, uint64_t& active, uint64_t& resident, uint64_t& retained)
+    { app_all = active = resident = retained = 0; }
+
     static HeapInterface* get_instance();
 
 protected:

diff --git a/src/memory/memory_cap.cc b/src/memory/memory_cap.cc
index ad3d8a6474f4b662891f978bc4a9a8dbe426a828..800b5a23932a281c1dbe0fc02d4d3bb0071ab242 100644 (file)
--- a/src/memory/memory_cap.cc
+++ b/src/memory/memory_cap.cc
@@ -88,6 +88,15 @@ static void epoch_check(void*)
 
     mc.cur_in_use = total;
     mc.epochs++;
+
+    // for reporting / tracking only
+    uint64_t all, act, res, ret;
+    heap->get_aux_counts(all, act, res, ret);
+
+    mc.app_all = all;
+    mc.active = act;
+    mc.resident = res;
+    mc.retained = ret;
 }
 
 // -----------------------------------------------------------------------------

diff --git a/src/memory/memory_cap.h b/src/memory/memory_cap.h
index 1d03dee128c56cc4c1e80049f70b35b1ea0b5c85..b4045bad97e13adc46592bdf369eabd5ac49091c 100644 (file)
--- a/src/memory/memory_cap.h
+++ b/src/memory/memory_cap.h
@@ -48,6 +48,11 @@ struct MemoryCounts
     PegCount reap_aborts;
     PegCount reap_decrease;
     PegCount reap_increase;
+    // reporting only
+    PegCount app_all;
+    PegCount active;
+    PegCount resident;
+    PegCount retained;
 };
 
 typedef bool (*PruneHandler)();

diff --git a/src/memory/memory_module.cc b/src/memory/memory_module.cc
index b364a846f059eb678897a3b04a1fc244f0ea06c7..e25f12e66aea4e98a2a83f68f63d77b410255971 100644 (file)
--- a/src/memory/memory_module.cc
+++ b/src/memory/memory_module.cc
@@ -73,6 +73,11 @@ const PegInfo mem_pegs[] =
     { CountType::NOW, "reap_aborts", "abort pruning before target due to process under limit" },
     { CountType::NOW, "reap_decrease", "total amount of the decrease in thread memory while process over limit" },
     { CountType::NOW, "reap_increase", "total amount of the increase in thread memory while process over limit" },
+    { CountType::NOW, "app_all", "total bytes allocated by application" },
+    { CountType::NOW, "active", "total bytes allocated in active pages" },
+    { CountType::NOW, "resident", "maximum bytes physically resident" },
+    { CountType::NOW, "retained", "total bytes not returned to OS" },
+
     { CountType::END, nullptr, nullptr }
 };