From: Otto Moerbeek Date: Tue, 19 Feb 2019 14:34:52 +0000 (+0100) Subject: Change the way getRealMemUsage() works on Linux (using statm) X-Git-Tag: auth-4.2.0-beta1~3^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=330dcb5c61d20d26c7d782c9eb7bd5872664e10a;p=thirdparty%2Fpdns.git Change the way getRealMemUsage() works on Linux (using statm) If you want the old (expensive) way, use getSpecialMemUsage() getSpecialMemUsage() is not included in getAllStatsMap() Also, implement a getrusage() based getRealMemUsage() for non-Linux systems. --- diff --git a/pdns/common_startup.cc b/pdns/common_startup.cc index ccdfbe8b25..e6c487169a 100644 --- a/pdns/common_startup.cc +++ b/pdns/common_startup.cc @@ -317,6 +317,7 @@ void declareStats(void) S.declare("uptime", "Uptime of process in seconds", uptimeOfProcess); S.declare("real-memory-usage", "Actual unique use of memory in bytes (approx)", getRealMemoryUsage); + S.declare("special-memory-usage", "Actual unique use of memory in bytes (approx)", getSpecialMemoryUsage); S.declare("fd-usage", "Number of open filedescriptors", getOpenFileDescriptors); #ifdef __linux__ S.declare("udp-recvbuf-errors", "UDP 'recvbuf' errors", udpErrorStats); diff --git a/pdns/dnsdist-snmp.cc b/pdns/dnsdist-snmp.cc index 9eadeb2505..c0957d4e6f 100644 --- a/pdns/dnsdist-snmp.cc +++ b/pdns/dnsdist-snmp.cc @@ -48,6 +48,7 @@ static const oid dynBlockedOID[] = { DNSDIST_STATS_OID, 35 }; static const oid dynBlockedNMGSizeOID[] = { DNSDIST_STATS_OID, 36 }; static const oid ruleServFailOID[] = { DNSDIST_STATS_OID, 37 }; static const oid securityStatusOID[] = { DNSDIST_STATS_OID, 38 }; +static const oid specialMemoryUsageOID[] = { DNSDIST_STATS_OID, 39 }; static std::unordered_map s_statsMap; @@ -576,12 +577,13 @@ DNSDistSNMPAgent::DNSDistSNMPAgent(const std::string& name, const std::string& m registerFloatStat("latencyAvg10000", latencyAvg10000OID, OID_LENGTH(latencyAvg10000OID), &g_stats.latencyAvg10000); registerFloatStat("latencyAvg1000000", latencyAvg1000000OID, OID_LENGTH(latencyAvg1000000OID), &g_stats.latencyAvg1000000); registerGauge64Stat("uptime", uptimeOID, OID_LENGTH(uptimeOID), &uptimeOfProcess); - registerGauge64Stat("realMemoryUsage", realMemoryUsageOID, OID_LENGTH(realMemoryUsageOID), &getRealMemoryUsage); + registerGauge64Stat("specialMemoryUsage", specialMemoryUsageOID, OID_LENGTH(specialMemoryUsageOID), &getSpecialMemoryUsage); registerGauge64Stat("cpuUserMSec", cpuUserMSecOID, OID_LENGTH(cpuUserMSecOID), &getCPUTimeUser); registerGauge64Stat("cpuSysMSec", cpuSysMSecOID, OID_LENGTH(cpuSysMSecOID), &getCPUTimeSystem); registerGauge64Stat("fdUsage", fdUsageOID, OID_LENGTH(fdUsageOID), &getOpenFileDescriptors); registerGauge64Stat("dynBlockedNMGSize", dynBlockedNMGSizeOID, OID_LENGTH(dynBlockedNMGSizeOID), [](const std::string&) { return g_dynblockNMG.getLocal()->size(); }); registerGauge64Stat("securityStatus", securityStatusOID, OID_LENGTH(securityStatusOID), [](const std::string&) { return g_stats.securityStatus.load(); }); + registerGauge64Stat("realMemoryUsage", realMemoryUsageOID, OID_LENGTH(realMemoryUsageOID), &getRealMemoryUsage); netsnmp_table_registration_info* table_info = SNMP_MALLOC_TYPEDEF(netsnmp_table_registration_info); diff --git a/pdns/dnsdist-web.cc b/pdns/dnsdist-web.cc index 8cd977384f..6359a1a99e 100644 --- a/pdns/dnsdist-web.cc +++ b/pdns/dnsdist-web.cc @@ -323,6 +323,8 @@ static void connectionThread(int sock, ComboAddress remote) }; for(const auto& e : g_stats.entries) { + if (e.first == "special-memory-usage") + continue; // Too expensive for get-all if(const auto& val = boost::get(&e.second)) obj.insert({e.first, (double)(*val)->load()}); else if (const auto& dval = boost::get(&e.second)) @@ -405,6 +407,8 @@ static void connectionThread(int sock, ComboAddress remote) std::ostringstream output; for (const auto& e : g_stats.entries) { + if (e.first == "special-memory-usage") + continue; // Too expensive for get-all std::string metricName = std::get<0>(e); // Prometheus suggest using '_' instead of '-' @@ -663,6 +667,9 @@ static void connectionThread(int sock, ComboAddress remote) Json::array doc; for(const auto& item : g_stats.entries) { + if (item.first == "special-memory-usage") + continue; // Too expensive for get-all + if(const auto& val = boost::get(&item.second)) { doc.push_back(Json::object { { "type", "StatisticItem" }, diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh index 45f8d4706e..21bae2d876 100644 --- a/pdns/dnsdist.hh +++ b/pdns/dnsdist.hh @@ -256,6 +256,7 @@ struct DNSDistStats {"latency-avg1000000", &latencyAvg1000000}, {"uptime", uptimeOfProcess}, {"real-memory-usage", getRealMemoryUsage}, + {"special-memory-usage", getSpecialMemoryUsage}, {"noncompliant-queries", &nonCompliantQueries}, {"noncompliant-responses", &nonCompliantResponses}, {"rdqueries", &rdQueries}, diff --git a/pdns/dnsdistdist/DNSDIST-MIB.txt b/pdns/dnsdistdist/DNSDIST-MIB.txt index 9d1f502692..f2c8b7e15c 100644 --- a/pdns/dnsdistdist/DNSDIST-MIB.txt +++ b/pdns/dnsdistdist/DNSDIST-MIB.txt @@ -326,6 +326,14 @@ ruleServFail OBJECT-TYPE "Number of ServFail responses returned because of a rule" ::= { stats 37 } +specialMemoryUsage OBJECT-TYPE + SYNTAX CounterBasedGauge64 + MAX-ACCESS read-only + STATUS current + DESCRIPTION + "Memory usage (more precise but expensive to retrieve)" + ::= { stats 38 } + securityStatus OBJECT-TYPE SYNTAX CounterBasedGauge64 MAX-ACCESS read-only diff --git a/pdns/misc.cc b/pdns/misc.cc index 10f799a74f..05191dd11c 100644 --- a/pdns/misc.cc +++ b/pdns/misc.cc @@ -1240,6 +1240,26 @@ uint64_t getOpenFileDescriptors(const std::string&) uint64_t getRealMemoryUsage(const std::string&) { +#ifdef __linux__ + ifstream ifs("/proc/"+std::to_string(getpid())+"/statm"); + if(!ifs) + return 0; + + uint64_t size, resident, shared, text, lib, data; + ifs >> size >> resident >> shared >> text >> lib >> data; + + return data * getpagesize(); +#else + struct rusage ru; + if (getrusage(RUSAGE_SELF, &ru) != 0) + return 0; + return ru.ru_maxrss * 1024; +#endif +} + + +uint64_t getSpecialMemoryUsage(const std::string&) +{ #ifdef __linux__ ifstream ifs("/proc/"+std::to_string(getpid())+"/smaps"); if(!ifs) diff --git a/pdns/misc.hh b/pdns/misc.hh index f04d7bb72c..f57a010bc6 100644 --- a/pdns/misc.hh +++ b/pdns/misc.hh @@ -539,6 +539,7 @@ bool setCloseOnExec(int sock); uint64_t udpErrorStats(const std::string& str); uint64_t getRealMemoryUsage(const std::string&); +uint64_t getSpecialMemoryUsage(const std::string&); uint64_t getOpenFileDescriptors(const std::string&); uint64_t getCPUTimeUser(const std::string&); uint64_t getCPUTimeSystem(const std::string&); diff --git a/pdns/rec-snmp.cc b/pdns/rec-snmp.cc index 8fa346b74d..e73d7e7d23 100644 --- a/pdns/rec-snmp.cc +++ b/pdns/rec-snmp.cc @@ -113,6 +113,7 @@ static const oid emptyQueriesOID[] = { RECURSOR_STATS_OID, 94 }; static const oid dnssecAuthenticDataQueriesOID[] = { RECURSOR_STATS_OID, 95 }; static const oid dnssecCheckDisabledQueriesOID[] = { RECURSOR_STATS_OID, 96 }; static const oid variableResponsesOID[] = { RECURSOR_STATS_OID, 97 }; +static const oid specialMemoryUsageOID[] = { RECURSOR_STATS_OID, 98 }; static std::unordered_map s_statsMap; @@ -302,6 +303,7 @@ RecursorSNMPAgent::RecursorSNMPAgent(const std::string& name, const std::string& registerCounter64Stat("policy-result-nodata", policyResultNodataOID, OID_LENGTH(policyResultNodataOID)); registerCounter64Stat("policy-result-truncate", policyResultTruncateOID, OID_LENGTH(policyResultTruncateOID)); registerCounter64Stat("policy-result-custom", policyResultCustomOID, OID_LENGTH(policyResultCustomOID)); + registerCounter64Stat("special-memory-usage", specialMemoryUsageOID, OID_LENGTH(specialMemoryUsageOID)); #endif /* HAVE_NET_SNMP */ } diff --git a/pdns/rec_channel_rec.cc b/pdns/rec_channel_rec.cc index 92da0a805f..277d341063 100644 --- a/pdns/rec_channel_rec.cc +++ b/pdns/rec_channel_rec.cc @@ -113,9 +113,11 @@ map getAllStatsMap() ret.insert(make_pair(atomic.first, std::to_string(atomic.second->load()))); } - for(const auto& the64bitmembers : d_get64bitmembers) { + for(const auto& the64bitmembers : d_get64bitmembers) { if(the64bitmembers.first == "cache-bytes" || the64bitmembers.first=="packetcache-bytes") continue; // too slow for 'get-all' + if(the64bitmembers.first == "special-memory-usage") + continue; // too slow for 'get-all' ret.insert(make_pair(the64bitmembers.first, std::to_string(the64bitmembers.second()))); } Lock l(&d_dynmetricslock); @@ -1022,7 +1024,8 @@ void registerAllStats() addGetStat("uptime", calculateUptime); addGetStat("real-memory-usage", boost::bind(getRealMemoryUsage, string())); - addGetStat("fd-usage", boost::bind(getOpenFileDescriptors, string())); + addGetStat("special-memory-usage", boost::bind(getSpecialMemoryUsage, string())); + addGetStat("fd-usage", boost::bind(getOpenFileDescriptors, string())); // addGetStat("query-rate", getQueryRate); addGetStat("user-msec", getUserTimeMsec); diff --git a/pdns/recursordist/RECURSOR-MIB.txt b/pdns/recursordist/RECURSOR-MIB.txt index 23f4287a81..625f8f4fc0 100644 --- a/pdns/recursordist/RECURSOR-MIB.txt +++ b/pdns/recursordist/RECURSOR-MIB.txt @@ -809,6 +809,14 @@ variableResponses OBJECT-TYPE "Number of variable responses" ::= { stats 97 } +specialMemoryUsage OBJECT-TYPE + SYNTAX CounterBasedGauge64 + MAX-ACCESS read-only + STATUS current + DESCRIPTION + "Memory usage (more precise bbut expensive to retrieve)" + ::= { stats 98 } + --- --- Traps / Notifications --- diff --git a/regression-tests.nobackend/counters/command b/regression-tests.nobackend/counters/command index c3938cbca7..6475101104 100755 --- a/regression-tests.nobackend/counters/command +++ b/regression-tests.nobackend/counters/command @@ -27,7 +27,7 @@ $SDIG ::1 $port example.com SOA >&2 >/dev/null $SDIG ::1 $port example.com SOA tcp >&2 >/dev/null $PDNSCONTROL --config-name= --no-config --socket-dir=./ 'show *' | \ - tr ',' '\n'| grep -v -E '(user-msec|sys-msec|uptime|udp-noport-errors|udp-in-errors|real-memory-usage|udp-recvbuf-errors|udp-sndbuf-errors|-hit|-miss|fd-usage|latency)' | LC_ALL=C sort + tr ',' '\n'| grep -v -E '(user-msec|sys-msec|uptime|udp-noport-errors|udp-in-errors|real-memory-usage|special-memory-usage|udp-recvbuf-errors|udp-sndbuf-errors|-hit|-miss|fd-usage|latency)' | LC_ALL=C sort kill $(cat pdns*.pid) rm pdns*.pid