From 9472436f62be8f061a955b2e59dd0b9911186941 Mon Sep 17 00:00:00 2001 From: Otto Moerbeek Date: Mon, 29 Aug 2022 14:51:38 +0200 Subject: [PATCH] Add statistics command for remote logging stats --- pdns/fstrm_logger.hh | 6 ++++ pdns/rec_channel_rec.cc | 64 +++++++++++++++++++++++++++++++++++ pdns/recursordist/rec-main.cc | 9 +++++ pdns/recursordist/rec-main.hh | 3 ++ pdns/remote_logger.hh | 15 ++++++++ 5 files changed, 97 insertions(+) diff --git a/pdns/fstrm_logger.hh b/pdns/fstrm_logger.hh index c9890fc52d..2b1d69036b 100644 --- a/pdns/fstrm_logger.hh +++ b/pdns/fstrm_logger.hh @@ -39,6 +39,12 @@ public: FrameStreamLogger(int family, const std::string& address, bool connect, const std::unordered_map& options = std::unordered_map()); ~FrameStreamLogger(); [[nodiscard]] RemoteLoggerInterface::Result queueData(const std::string& data) override; + + std::string address() const override + { + return d_address; + } + const std::string name() const override { return "dnstap"; diff --git a/pdns/rec_channel_rec.cc b/pdns/rec_channel_rec.cc index 09b5854159..8b4bd6292f 100644 --- a/pdns/rec_channel_rec.cc +++ b/pdns/rec_channel_rec.cc @@ -939,6 +939,18 @@ static ProxyMappingStats_t* pleaseGetProxyMappingStats() return ret; } +static RemoteLoggerStats_t* pleaseGetRemoteLoggerStats() +{ + auto ret = new RemoteLoggerStats_t; + + if (t_protobufServers) { + for (const auto& s : *t_protobufServers) { + ret->emplace(std::make_pair(s->address(), s->getStats())); + } + } + return ret; +} + static string doGetProxyMappingStats() { ostringstream ret; @@ -950,6 +962,54 @@ static string doGetProxyMappingStats() return ret.str(); } +static RemoteLoggerStats_t* pleaseGetOutgoingRemoteLoggerStats() +{ + auto ret = new RemoteLoggerStats_t; + + if (t_outgoingProtobufServers) { + for (const auto& s : *t_outgoingProtobufServers) { + ret->emplace(std::make_pair(s->address(), s->getStats())); + } + } + return ret; +} + +static RemoteLoggerStats_t* pleaseGetFramestreamLoggerStats() +{ + auto ret = new RemoteLoggerStats_t; + + if (t_frameStreamServersInfo.servers) { + for (const auto& s : *t_frameStreamServersInfo.servers) { + ret->emplace(std::make_pair(s->address(), s->getStats())); + } + } + return ret; +} + +static void remoteLoggerStats(const string& name, const RemoteLoggerStats_t& stats, ostringstream& os) +{ + if (stats.size() > 0) { + std::string filler(name.size(), ' '); + os << name << "\tQueued\tPipe-\tToo-\tOther-\tAddress" << endl; + os << filler << "\t\tFull\tLarge\terror" << endl; + for (const auto& [key, entry]: stats) { + os << filler<< '\t' << entry.d_queued << '\t' << entry.d_pipeFull << '\t' << entry.d_tooLarge << '\t' << entry.d_otherError << '\t' << key << endl; + } + } +} + +static string getRemoteLoggerStats() +{ + ostringstream os; + auto stats = broadcastAccFunction(pleaseGetRemoteLoggerStats); + remoteLoggerStats("Protobuf ", stats, os); + stats = broadcastAccFunction(pleaseGetOutgoingRemoteLoggerStats); + remoteLoggerStats("OutProtobuf", stats, os); + stats = broadcastAccFunction(pleaseGetFramestreamLoggerStats); + remoteLoggerStats("Framestream", stats, os); + return os.str(); +} + static uint64_t calculateUptime() { return time(nullptr) - g_stats.startupTime; @@ -1994,6 +2054,7 @@ RecursorControlChannel::Answer RecursorControlParser::getAnswer(int s, const str "get-parameter [key1] [key2] .. get configuration parameters\n" "get-proxymapping-stats get proxy mapping statistics\n" "get-qtypelist get QType statistics\n" + "get-remotelogger-stats get remote logger statistics\n" " notice: queries from cache aren't being counted yet\n" "hash-password [work-factor] ask for a password then return the hashed version\n" "help get this list\n" @@ -2248,6 +2309,9 @@ RecursorControlChannel::Answer RecursorControlParser::getAnswer(int s, const str if (cmd == "get-proxymapping-stats") { return {0, doGetProxyMappingStats()}; } + if (cmd == "get-remotelogger-stats") { + return {0, getRemoteLoggerStats() }; + } return {1, "Unknown command '" + cmd + "', try 'help'\n"}; } diff --git a/pdns/recursordist/rec-main.cc b/pdns/recursordist/rec-main.cc index 42eea037bb..90f8b0cb43 100644 --- a/pdns/recursordist/rec-main.cc +++ b/pdns/recursordist/rec-main.cc @@ -1294,6 +1294,14 @@ static ProxyMappingStats_t& operator+=(ProxyMappingStats_t& a, const ProxyMappin return a; } +static RemoteLoggerStats_t& operator+=(RemoteLoggerStats_t &a, const RemoteLoggerStats_t& b) +{ + for (const auto& [key, entry] : b) { + a[key] += entry; + } + return a; +} + // This function should only be called by the handler to gather // metrics, wipe the cache, reload the Lua script (not the Lua config) // or change the current trace regex, and by the SNMP thread to gather @@ -1346,6 +1354,7 @@ template vector broadcastAccFunction(const std::function> broadcastAccFunction(const std::function>*()>& fun); // explicit instantiation template ThreadTimes broadcastAccFunction(const std::function& fun); template ProxyMappingStats_t broadcastAccFunction(const std::function& fun); +template RemoteLoggerStats_t broadcastAccFunction(const std::function& fun); static int serviceMain(int argc, char* argv[], Logr::log_t log) { diff --git a/pdns/recursordist/rec-main.hh b/pdns/recursordist/rec-main.hh index 8bb93431d9..59ead71291 100644 --- a/pdns/recursordist/rec-main.hh +++ b/pdns/recursordist/rec-main.hh @@ -188,6 +188,9 @@ typedef MTasker, PacketBuffer, PacketIDCompare> MT_t; extern thread_local std::unique_ptr MT; // the big MTasker extern thread_local std::unique_ptr t_packetCache; +using RemoteLoggerStats_t = std::unordered_map; + + extern bool g_logCommonErrors; extern size_t g_proxyProtocolMaximumSize; extern std::atomic g_quiet; diff --git a/pdns/remote_logger.hh b/pdns/remote_logger.hh index 98caa91af3..1b126b403c 100644 --- a/pdns/remote_logger.hh +++ b/pdns/remote_logger.hh @@ -66,6 +66,7 @@ public: virtual ~RemoteLoggerInterface() {}; virtual Result queueData(const std::string& data) = 0; + virtual std::string address() const = 0; virtual std::string toString() const = 0; virtual const std::string name() const = 0; bool logQueries(void) const { return d_logQueries; } @@ -79,6 +80,15 @@ public: uint64_t d_pipeFull{}; uint64_t d_tooLarge{}; uint64_t d_otherError{}; + + Stats& operator += (const Stats& rhs) + { + d_queued += rhs.d_queued; + d_pipeFull += rhs.d_pipeFull; + d_tooLarge += rhs.d_tooLarge; + d_otherError += rhs.d_otherError; + return *this; + } }; virtual Stats getStats() const = 0; @@ -102,6 +112,11 @@ public: bool asyncConnect=false); ~RemoteLogger(); + std::string address() const override + { + return d_remote.toStringWithPort(); + } + [[nodiscard]] Result queueData(const std::string& data) override; const std::string name() const override { -- 2.47.2