Change the way getRealMemUsage() works on Linux (using statm)

If you want the old (expensive) way, use getSpecialMemUsage()
getSpecialMemUsage() is not included in getAllStatsMap()
Also, implement a getrusage() based getRealMemUsage() for non-Linux systems.

diff --git a/pdns/common_startup.cc b/pdns/common_startup.cc
index ccdfbe8b25b429e9e29cc7e7e1e45e6d14ce7836..e6c487169aaafa32ab024f3e32ac4c42ea2e1165 100644 (file)
--- a/pdns/common_startup.cc
+++ b/pdns/common_startup.cc
@@ -317,6 +317,7 @@ void declareStats(void)
 
   S.declare("uptime", "Uptime of process in seconds", uptimeOfProcess);
   S.declare("real-memory-usage", "Actual unique use of memory in bytes (approx)", getRealMemoryUsage);
+  S.declare("special-memory-usage", "Actual unique use of memory in bytes (approx)", getSpecialMemoryUsage);
   S.declare("fd-usage", "Number of open filedescriptors", getOpenFileDescriptors);
 #ifdef __linux__
   S.declare("udp-recvbuf-errors", "UDP 'recvbuf' errors", udpErrorStats);

diff --git a/pdns/dnsdist-snmp.cc b/pdns/dnsdist-snmp.cc
index 9eadeb2505c8838b1333f923dda47d6521a4283c..c0957d4e6f437673a8779f19383c50f151092e73 100644 (file)
--- a/pdns/dnsdist-snmp.cc
+++ b/pdns/dnsdist-snmp.cc
@@ -48,6 +48,7 @@ static const oid dynBlockedOID[] = { DNSDIST_STATS_OID, 35 };
 static const oid dynBlockedNMGSizeOID[] = { DNSDIST_STATS_OID, 36 };
 static const oid ruleServFailOID[] = { DNSDIST_STATS_OID, 37 };
 static const oid securityStatusOID[] = { DNSDIST_STATS_OID, 38 };
+static const oid specialMemoryUsageOID[] = { DNSDIST_STATS_OID, 39 };
 
 static std::unordered_map<oid, DNSDistStats::entry_t> s_statsMap;
 
@@ -576,12 +577,13 @@ DNSDistSNMPAgent::DNSDistSNMPAgent(const std::string& name, const std::string& m
   registerFloatStat("latencyAvg10000", latencyAvg10000OID, OID_LENGTH(latencyAvg10000OID), &g_stats.latencyAvg10000);
   registerFloatStat("latencyAvg1000000", latencyAvg1000000OID, OID_LENGTH(latencyAvg1000000OID), &g_stats.latencyAvg1000000);
   registerGauge64Stat("uptime", uptimeOID, OID_LENGTH(uptimeOID), &uptimeOfProcess);
-  registerGauge64Stat("realMemoryUsage", realMemoryUsageOID, OID_LENGTH(realMemoryUsageOID), &getRealMemoryUsage);
+  registerGauge64Stat("specialMemoryUsage", specialMemoryUsageOID, OID_LENGTH(specialMemoryUsageOID), &getSpecialMemoryUsage);
   registerGauge64Stat("cpuUserMSec", cpuUserMSecOID, OID_LENGTH(cpuUserMSecOID), &getCPUTimeUser);
   registerGauge64Stat("cpuSysMSec", cpuSysMSecOID, OID_LENGTH(cpuSysMSecOID), &getCPUTimeSystem);
   registerGauge64Stat("fdUsage", fdUsageOID, OID_LENGTH(fdUsageOID), &getOpenFileDescriptors);
   registerGauge64Stat("dynBlockedNMGSize", dynBlockedNMGSizeOID, OID_LENGTH(dynBlockedNMGSizeOID), [](const std::string&) { return g_dynblockNMG.getLocal()->size(); });
   registerGauge64Stat("securityStatus", securityStatusOID, OID_LENGTH(securityStatusOID), [](const std::string&) { return g_stats.securityStatus.load(); });
+  registerGauge64Stat("realMemoryUsage", realMemoryUsageOID, OID_LENGTH(realMemoryUsageOID), &getRealMemoryUsage);
 
 
   netsnmp_table_registration_info* table_info = SNMP_MALLOC_TYPEDEF(netsnmp_table_registration_info);

diff --git a/pdns/dnsdist-web.cc b/pdns/dnsdist-web.cc
index 8cd977384f867510b2c38eeaed5b98d1bc786c93..6359a1a99e54da69f557b6b21b1af18e373b48d4 100644 (file)
--- a/pdns/dnsdist-web.cc
+++ b/pdns/dnsdist-web.cc
@@ -323,6 +323,8 @@ static void connectionThread(int sock, ComboAddress remote)
         };
 
         for(const auto& e : g_stats.entries) {
+          if (e.first == "special-memory-usage")
+            continue; // Too expensive for get-all
           if(const auto& val = boost::get<DNSDistStats::stat_t*>(&e.second))
             obj.insert({e.first, (double)(*val)->load()});
           else if (const auto& dval = boost::get<double*>(&e.second))
@@ -405,6 +407,8 @@ static void connectionThread(int sock, ComboAddress remote)
 
         std::ostringstream output;
         for (const auto& e : g_stats.entries) {
+          if (e.first == "special-memory-usage")
+            continue; // Too expensive for get-all
           std::string metricName = std::get<0>(e);
 
           // Prometheus suggest using '_' instead of '-'
@@ -663,6 +667,9 @@ static void connectionThread(int sock, ComboAddress remote)
 
       Json::array doc;
       for(const auto& item : g_stats.entries) {
+        if (item.first == "special-memory-usage")
+          continue; // Too expensive for get-all
+
         if(const auto& val = boost::get<DNSDistStats::stat_t*>(&item.second)) {
           doc.push_back(Json::object {
               { "type", "StatisticItem" },

diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh
index 45f8d4706ecc31b8ce59ccb49aaad74c6f00baf7..21bae2d876d80a40e06ac937bf0ab4908641634f 100644 (file)
--- a/pdns/dnsdist.hh
+++ b/pdns/dnsdist.hh
@@ -256,6 +256,7 @@ struct DNSDistStats
     {"latency-avg1000000", &latencyAvg1000000},
     {"uptime", uptimeOfProcess},
     {"real-memory-usage", getRealMemoryUsage},
+    {"special-memory-usage", getSpecialMemoryUsage},
     {"noncompliant-queries", &nonCompliantQueries},
     {"noncompliant-responses", &nonCompliantResponses},
     {"rdqueries", &rdQueries},

diff --git a/pdns/dnsdistdist/DNSDIST-MIB.txt b/pdns/dnsdistdist/DNSDIST-MIB.txt
index 9d1f5026920064fad0b9e86daa13e3b485ef8913..f2c8b7e15cb2d43ca64b2c50981bbf90545ccf9a 100644 (file)
--- a/pdns/dnsdistdist/DNSDIST-MIB.txt
+++ b/pdns/dnsdistdist/DNSDIST-MIB.txt
@@ -326,6 +326,14 @@ ruleServFail OBJECT-TYPE
        "Number of ServFail responses returned because of a rule"
     ::= { stats 37 }
 
+specialMemoryUsage OBJECT-TYPE
+    SYNTAX CounterBasedGauge64
+    MAX-ACCESS read-only
+    STATUS current
+    DESCRIPTION
+       "Memory usage (more precise but expensive to retrieve)"
+    ::= { stats 38 }
+
 securityStatus OBJECT-TYPE
     SYNTAX CounterBasedGauge64
     MAX-ACCESS read-only

diff --git a/pdns/misc.cc b/pdns/misc.cc
index 10f799a74f4567c185e125a8824211b5ff31bbf8..05191dd11c7280b5e2b800fd224b284b7a43f370 100644 (file)
--- a/pdns/misc.cc
+++ b/pdns/misc.cc
@@ -1240,6 +1240,26 @@ uint64_t getOpenFileDescriptors(const std::string&)
 
 uint64_t getRealMemoryUsage(const std::string&)
 {
+#ifdef __linux__
+  ifstream ifs("/proc/"+std::to_string(getpid())+"/statm");
+  if(!ifs)
+    return 0;
+
+  uint64_t size, resident, shared, text, lib, data;
+  ifs >> size >> resident >> shared >> text >> lib >> data;
+
+  return data * getpagesize();
+#else
+  struct rusage ru;
+  if (getrusage(RUSAGE_SELF, &ru) != 0)
+    return 0;
+  return ru.ru_maxrss * 1024;
+#endif
+}
+
+
+uint64_t getSpecialMemoryUsage(const std::string&)
+{
 #ifdef __linux__
   ifstream ifs("/proc/"+std::to_string(getpid())+"/smaps");
   if(!ifs)

diff --git a/pdns/misc.hh b/pdns/misc.hh
index f04d7bb72c5252db766a311311b224c4e3e225bf..f57a010bc6041bbed5047595444ba26aef5a59e6 100644 (file)
--- a/pdns/misc.hh
+++ b/pdns/misc.hh
@@ -539,6 +539,7 @@ bool setCloseOnExec(int sock);
 uint64_t udpErrorStats(const std::string& str);
 
 uint64_t getRealMemoryUsage(const std::string&);
+uint64_t getSpecialMemoryUsage(const std::string&);
 uint64_t getOpenFileDescriptors(const std::string&);
 uint64_t getCPUTimeUser(const std::string&);
 uint64_t getCPUTimeSystem(const std::string&);

diff --git a/pdns/rec-snmp.cc b/pdns/rec-snmp.cc
index 8fa346b74d2189e1c12493426ebc3916cd167994..e73d7e7d23114fab22f191e8080c7e034ef38f3c 100644 (file)
--- a/pdns/rec-snmp.cc
+++ b/pdns/rec-snmp.cc
@@ -113,6 +113,7 @@ static const oid emptyQueriesOID[] = { RECURSOR_STATS_OID, 94 };
 static const oid dnssecAuthenticDataQueriesOID[] = { RECURSOR_STATS_OID, 95 };
 static const oid dnssecCheckDisabledQueriesOID[] = { RECURSOR_STATS_OID, 96 };
 static const oid variableResponsesOID[] = { RECURSOR_STATS_OID, 97 };
+static const oid specialMemoryUsageOID[] = { RECURSOR_STATS_OID, 98 };
 
 static std::unordered_map<oid, std::string> s_statsMap;
 
@@ -302,6 +303,7 @@ RecursorSNMPAgent::RecursorSNMPAgent(const std::string& name, const std::string&
   registerCounter64Stat("policy-result-nodata", policyResultNodataOID, OID_LENGTH(policyResultNodataOID));
   registerCounter64Stat("policy-result-truncate", policyResultTruncateOID, OID_LENGTH(policyResultTruncateOID));
   registerCounter64Stat("policy-result-custom", policyResultCustomOID, OID_LENGTH(policyResultCustomOID));
+  registerCounter64Stat("special-memory-usage", specialMemoryUsageOID, OID_LENGTH(specialMemoryUsageOID));
 
 #endif /* HAVE_NET_SNMP */
 }

diff --git a/pdns/rec_channel_rec.cc b/pdns/rec_channel_rec.cc
index 92da0a805f08b1e1c974920c04e89d7a7998b96a..277d341063c5f4055f39e517dd112597ca4488dc 100644 (file)
--- a/pdns/rec_channel_rec.cc
+++ b/pdns/rec_channel_rec.cc
@@ -113,9 +113,11 @@ map<string,string> getAllStatsMap()
     ret.insert(make_pair(atomic.first, std::to_string(atomic.second->load())));
   }
 
-  for(const auto& the64bitmembers :  d_get64bitmembers) { 
+  for(const auto& the64bitmembers :  d_get64bitmembers) {
     if(the64bitmembers.first == "cache-bytes" || the64bitmembers.first=="packetcache-bytes")
       continue; // too slow for 'get-all'
+    if(the64bitmembers.first == "special-memory-usage")
+      continue; // too slow for 'get-all'
     ret.insert(make_pair(the64bitmembers.first, std::to_string(the64bitmembers.second())));
   }
   Lock l(&d_dynmetricslock);
@@ -1022,7 +1024,8 @@ void registerAllStats()
 
   addGetStat("uptime", calculateUptime);
   addGetStat("real-memory-usage", boost::bind(getRealMemoryUsage, string()));
-  addGetStat("fd-usage", boost::bind(getOpenFileDescriptors, string()));  
+  addGetStat("special-memory-usage", boost::bind(getSpecialMemoryUsage, string()));
+  addGetStat("fd-usage", boost::bind(getOpenFileDescriptors, string()));
 
   //  addGetStat("query-rate", getQueryRate);
   addGetStat("user-msec", getUserTimeMsec);

diff --git a/pdns/recursordist/RECURSOR-MIB.txt b/pdns/recursordist/RECURSOR-MIB.txt
index 23f4287a81e7f2655b4b6bb33d535f44138574c0..625f8f4fc085b537e6de4b4161e68612c3eb8fba 100644 (file)
--- a/pdns/recursordist/RECURSOR-MIB.txt
+++ b/pdns/recursordist/RECURSOR-MIB.txt
@@ -809,6 +809,14 @@ variableResponses OBJECT-TYPE
         "Number of variable responses"
     ::= { stats 97 }
 
+specialMemoryUsage OBJECT-TYPE
+    SYNTAX CounterBasedGauge64
+    MAX-ACCESS read-only
+    STATUS current
+    DESCRIPTION
+        "Memory usage (more precise bbut expensive to retrieve)"
+    ::= { stats 98 }
+
 ---
 --- Traps / Notifications
 ---

diff --git a/regression-tests.nobackend/counters/command b/regression-tests.nobackend/counters/command
index c3938cbca7d409df0f61df93f8aea8622963ec4f..6475101104601d44467d513ebd0b37b23c58cd5a 100755 (executable)
--- a/regression-tests.nobackend/counters/command
+++ b/regression-tests.nobackend/counters/command
@@ -27,7 +27,7 @@ $SDIG ::1 $port example.com SOA >&2 >/dev/null
 $SDIG ::1 $port example.com SOA tcp >&2 >/dev/null
 
 $PDNSCONTROL --config-name= --no-config --socket-dir=./ 'show *' | \
-  tr ',' '\n'| grep -v -E '(user-msec|sys-msec|uptime|udp-noport-errors|udp-in-errors|real-memory-usage|udp-recvbuf-errors|udp-sndbuf-errors|-hit|-miss|fd-usage|latency)' | LC_ALL=C sort
+  tr ',' '\n'| grep -v -E '(user-msec|sys-msec|uptime|udp-noport-errors|udp-in-errors|real-memory-usage|special-memory-usage|udp-recvbuf-errors|udp-sndbuf-errors|-hit|-miss|fd-usage|latency)' | LC_ALL=C sort
 
 kill $(cat pdns*.pid)
 rm pdns*.pid