From: Remi Gacogne Date: Wed, 20 Feb 2019 15:45:02 +0000 (+0100) Subject: rec: Make blacklisted statistics configurable per component X-Git-Tag: dnsdist-1.4.0-alpha1~58^2~2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=72259676bd47110d0322d1f9f3ca86a58ed94c11;p=thirdparty%2Fpdns.git rec: Make blacklisted statistics configurable per component This commit adds configuration parameters so it's possible to enable or disable specific statistic metrics depending on the way they are retrieved (API, Carbon, rec-control or SNMP). --- diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc index 80d30e5d11..02ec7ecfdf 100644 --- a/pdns/pdns_recursor.cc +++ b/pdns/pdns_recursor.cc @@ -3857,8 +3857,13 @@ static int serviceMain(int argc, char*argv[]) g_tcpMaxQueriesPerConn=::arg().asNum("max-tcp-queries-per-connection"); s_maxUDPQueriesPerRound=::arg().asNum("max-udp-queries-per-round"); + blacklistStats(StatComponent::API, ::arg()["blacklisted-stats-api"]); + blacklistStats(StatComponent::Carbon, ::arg()["blacklisted-stats-carbon"]); + blacklistStats(StatComponent::RecControl, ::arg()["blacklisted-stats-rec-control"]); + blacklistStats(StatComponent::SNMP, ::arg()["blacklisted-stats-snmp"]); + if (::arg().mustDo("snmp-agent")) { - g_snmpAgent = std::make_shared("recursor", ::arg()["snmp-master-socket"], ::arg().mustDo("snmp-enable-expensive-stats")); + g_snmpAgent = std::make_shared("recursor", ::arg()["snmp-master-socket"]); g_snmpAgent->run(); } @@ -4279,7 +4284,18 @@ int main(int argc, char **argv) ::arg().setSwitch("snmp-agent", "If set, register as an SNMP agent")="no"; ::arg().set("snmp-master-socket", "If set and snmp-agent is set, the socket to use to register to the SNMP master")=""; - ::arg().setSwitch("snmp-enable-expensive-stats", "If set and snmp-agent is set, even statistics whose reporting can have an impact on production will be enabled")="no"; + + std::string defaultBlacklistedStats = "cache-bytes, packetcache-bytes, special-memory-usage"; + for (size_t idx = 0; idx < 32; idx++) { + defaultBlacklistedStats += ", ecs-v4-response-bits-" + std::to_string(idx + 1); + } + for (size_t idx = 0; idx < 128; idx++) { + defaultBlacklistedStats += ", ecs-v6-response-bits-" + std::to_string(idx + 1); + } + ::arg().set("blacklisted-stats-api", "List of statistics that are disabled when retrieving the complete list of statistics via the API")=defaultBlacklistedStats; + ::arg().set("blacklisted-stats-carbon", "List of statistics that are prevented from being exported via Carbon")=defaultBlacklistedStats; + ::arg().set("blacklisted-stats-rec-control", "List of statistics that are prevented from being exported via rec_control get-all")=defaultBlacklistedStats; + ::arg().set("blacklisted-stats-snmp", "List of statistics that are prevented from being exported via SNMP")=defaultBlacklistedStats; ::arg().set("tcp-fast-open", "Enable TCP Fast Open support on the listening sockets, using the supplied numerical value as the queue size")="0"; ::arg().set("nsec3-max-iterations", "Maximum number of iterations allowed for an NSEC3 record")="2500"; diff --git a/pdns/rec-carbon.cc b/pdns/rec-carbon.cc index b8f4e258c6..5f2b92fa08 100644 --- a/pdns/rec-carbon.cc +++ b/pdns/rec-carbon.cc @@ -57,7 +57,7 @@ try if(msg.empty()) { typedef map all_t; - all_t all=getAllStatsMap(); + all_t all=getAllStatsMap(StatComponent::Carbon); ostringstream str; time_t now=time(0); diff --git a/pdns/rec-snmp.cc b/pdns/rec-snmp.cc index 41ae6b8caf..415a36ff30 100644 --- a/pdns/rec-snmp.cc +++ b/pdns/rec-snmp.cc @@ -164,7 +164,7 @@ static int handleDisabledCounter64Stats(netsnmp_mib_handler* handler, return RecursorSNMPAgent::setCounter64Value(requests, 0); } -static void registerCounter64Stat(const std::string& name, const oid statOID[], size_t statOIDLength, bool expensiveStats) +static void registerCounter64Stat(const std::string& name, const oid statOID[], size_t statOIDLength) { if (statOIDLength != OID_LENGTH(questionsOID)) { g_log< getAllStatsMap(); +enum class StatComponent { API, Carbon, RecControl, SNMP }; + +std::map getAllStatsMap(StatComponent component); extern pthread_mutex_t g_carbon_config_lock; std::vector >* pleaseGetQueryRing(); std::vector >* pleaseGetServfailQueryRing(); @@ -77,6 +79,9 @@ std::vector* pleaseGetTimeouts(); DNSName getRegisteredName(const DNSName& dom); std::atomic* getDynMetric(const std::string& str); optional getStatByName(const std::string& name); -bool isStatExpensive(const std::string& name); +bool isStatBlacklisted(StatComponent component, const std::string& name); +void blacklistStat(StatComponent component, const string& name); +void blacklistStats(StatComponent component, const string& stats); + void registerAllStats(); diff --git a/pdns/rec_channel_rec.cc b/pdns/rec_channel_rec.cc index 488697b19f..4b101b638a 100644 --- a/pdns/rec_channel_rec.cc +++ b/pdns/rec_channel_rec.cc @@ -45,16 +45,26 @@ static map > d_get64bitmembers; static pthread_mutex_t d_dynmetricslock = PTHREAD_MUTEX_INITIALIZER; static map* > d_dynmetrics; -static std::set s_expensiveStats = { "cache-bytes", "packetcache-bytes", "special-memory-usage" }; +static std::map> s_blacklistedStats; -bool isStatExpensive(const string& name) +bool isStatBlacklisted(StatComponent component, const string& name) { - return s_expensiveStats.count(name) != 0; + return s_blacklistedStats[component].count(name) != 0; } -void markStatAsExpensive(const string& name) +void blacklistStat(StatComponent component, const string& name) { - s_expensiveStats.insert(name); + s_blacklistedStats[component].insert(name); +} + +void blacklistStats(StatComponent component, const string& stats) +{ + std::vector blacklistedStats; + stringtok(blacklistedStats, stats, ", "); + auto& map = s_blacklistedStats[component]; + for (const auto &st : blacklistedStats) { + map.insert(st); + } } static void addGetStat(const string& name, const uint32_t* place) @@ -108,37 +118,44 @@ optional getStatByName(const std::string& name) return get(name); } -map getAllStatsMap() +map getAllStatsMap(StatComponent component) { map ret; - + const auto& blacklistMap = s_blacklistedStats.at(component); + for(const auto& the32bits : d_get32bitpointers) { - if (!isStatExpensive(the32bits.first)) { + if (blacklistMap.count(the32bits.first) == 0) { ret.insert(make_pair(the32bits.first, std::to_string(*the32bits.second))); } } for(const auto& atomic : d_getatomics) { - if (!isStatExpensive(atomic.first)) { + if (blacklistMap.count(atomic.first) == 0) { ret.insert(make_pair(atomic.first, std::to_string(atomic.second->load()))); } } - for(const auto& the64bitmembers : d_get64bitmembers) { - if (!isStatExpensive(the64bitmembers.first)) { + for(const auto& the64bitmembers : d_get64bitmembers) { + if (blacklistMap.count(the64bitmembers.first) == 0) { ret.insert(make_pair(the64bitmembers.first, std::to_string(the64bitmembers.second()))); } } - Lock l(&d_dynmetricslock); - for(const auto& a : d_dynmetrics) - ret.insert({a.first, std::to_string(*a.second)}); + { + Lock l(&d_dynmetricslock); + for(const auto& a : d_dynmetrics) { + if (blacklistMap.count(a.first) == 0) { + ret.insert({a.first, std::to_string(*a.second)}); + } + } + } + return ret; } -string getAllStats() +static string getAllStats() { typedef map varmap_t; - varmap_t varmap = getAllStatsMap(); + varmap_t varmap = getAllStatsMap(StatComponent::RecControl); string ret; for(varmap_t::value_type& tup : varmap) { ret += tup.first + "\t" + tup.second +"\n"; @@ -1061,12 +1078,10 @@ void registerAllStats() for (size_t idx = 0; idx < SyncRes::s_ecsResponsesBySubnetSize4.size(); idx++) { const std::string name = "ecs-v4-response-bits-" + std::to_string(idx + 1); addGetStat(name, &(SyncRes::s_ecsResponsesBySubnetSize4.at(idx))); - markStatAsExpensive(name); } for (size_t idx = 0; idx < SyncRes::s_ecsResponsesBySubnetSize6.size(); idx++) { const std::string name = "ecs-v6-response-bits-" + std::to_string(idx + 1); addGetStat(name, &(SyncRes::s_ecsResponsesBySubnetSize6.at(idx))); - markStatAsExpensive(name); } } diff --git a/pdns/recursordist/docs/settings.rst b/pdns/recursordist/docs/settings.rst index 803c6fedea..646a28970c 100644 --- a/pdns/recursordist/docs/settings.rst +++ b/pdns/recursordist/docs/settings.rst @@ -126,6 +126,46 @@ DNSSEC is not supported. Example: auth-zones=example.org=/var/zones/example.org, powerdns.com=/var/zones/powerdns.com +.. _setting-blacklisted-stats-api: + +``blacklisted-stats-api`` +------------------------- +.. versionadded:: 4.2.0 + +- String +- Default: "cache-bytes, packetcache-bytes, ecs-v4-response-bits-*, ecs-v6-response-bits-*" + +A list of comma-separated statistic names, that are disabled when retrieving the complete list of statistics via the API for performance reasons. +These statistics can still be retrieved individually by specifically asking for it. + +``blacklisted-stats-carbon`` +--------------------------- +.. versionadded:: 4.2.0 + +- String +- Default: "cache-bytes, packetcache-bytes, ecs-v4-response-bits-*, ecs-v6-response-bits-*" + +A list of comma-separated statistic names, that are prevented from being exported via carbon for performance reasons. + +``blacklisted-stats-rec-control`` +--------------------------------- +.. versionadded:: 4.2.0 + +- String +- Default: "cache-bytes, packetcache-bytes, ecs-v4-response-bits-*, ecs-v6-response-bits-*" + +A list of comma-separated statistic names, that are disabled when retrieving the complete list of statistics via `rec_control get-all`, for performance reasons. +These statistics can still be retrieved individually. + +``blacklisted-stats-snmp`` +-------------------------- +.. versionadded:: 4.2.0 + +- String +- Default: "cache-bytes, packetcache-bytes, ecs-v4-response-bits-*, ecs-v6-response-bits-*" + +A list of comma-separated statistic names, that are prevented from being exported via SNMP, for performance reasons. + .. _setting-carbon-interval: ``carbon-interval`` @@ -1225,17 +1265,6 @@ Use only a single socket for outgoing queries. If set to true and PowerDNS has been compiled with SNMP support, it will register as an SNMP agent to provide statistics and be able to send traps. -.. _setting-snmp-enable-expensive-stats: - -``snmp-enable-expensive-stats`` ----------------------- -.. versionadded:: 4.2.0 - -- Boolean -- Default: no - -If set and snmp-agent is set, even statistics whose reporting can have an impact on production will be enabled - .. _setting-snmp-master-socket: ``snmp-master-socket`` diff --git a/pdns/ws-recursor.cc b/pdns/ws-recursor.cc index a1889a3a67..dfb2368c4f 100644 --- a/pdns/ws-recursor.cc +++ b/pdns/ws-recursor.cc @@ -48,7 +48,7 @@ using json11::Json; void productServerStatisticsFetch(map& out) { - map stats = getAllStatsMap(); + map stats = getAllStatsMap(StatComponent::API); out.swap(stats); }