From 39e4242426d5ebffe4b3556eadfdf0b2a9fae7fa Mon Sep 17 00:00:00 2001 From: Otto Moerbeek Date: Wed, 22 Feb 2023 16:30:48 +0100 Subject: [PATCH] Initial integration with recursor --- pdns/recursordist/pdns_recursor.cc | 12 ++--- pdns/recursordist/rec-main.cc | 37 ++++++-------- pdns/recursordist/rec-main.hh | 2 +- pdns/recursordist/rec-tcounters.hh | 2 + pdns/recursordist/rec_channel_rec.cc | 56 +++------------------ pdns/recursordist/recpacketcache.cc | 75 ++++++++++++++++++++-------- pdns/recursordist/recpacketcache.hh | 18 +++---- pdns/recursordist/syncres.hh | 2 - 8 files changed, 90 insertions(+), 114 deletions(-) diff --git a/pdns/recursordist/pdns_recursor.cc b/pdns/recursordist/pdns_recursor.cc index 69c70f5d7e..071c09d201 100644 --- a/pdns/recursordist/pdns_recursor.cc +++ b/pdns/recursordist/pdns_recursor.cc @@ -49,8 +49,8 @@ thread_local ProtobufServersInfo t_outgoingProtobufServers; thread_local std::unique_ptr MT; // the big MTasker std::unique_ptr g_recCache; std::unique_ptr g_negCache; +std::unique_ptr g_packetCache; -thread_local std::unique_ptr t_packetCache; thread_local std::unique_ptr t_fdm; thread_local std::unique_ptr t_remotes, t_servfailremotes, t_largeanswerremotes, t_bogusremotes; thread_local std::unique_ptr>> t_queryring, t_servfailqueryring, t_bogusqueryring; @@ -1673,9 +1673,9 @@ void startDoResolve(void* p) #endif } - if (t_packetCache && !variableAnswer && !sr.wasVariable()) { + if (g_packetCache && !variableAnswer && !sr.wasVariable()) { minTTL = capPacketCacheTTL(*pw.getHeader(), minTTL, seenAuthSOA); - t_packetCache->insertResponsePacket(dc->d_tag, dc->d_qhash, std::move(dc->d_query), dc->d_mdp.d_qname, + g_packetCache->insertResponsePacket(dc->d_tag, dc->d_qhash, std::move(dc->d_query), dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_mdp.d_qclass, string((const char*)&*packet.begin(), packet.size()), g_now.tv_sec, @@ -1957,7 +1957,7 @@ bool checkForCacheHit(bool qnameParsed, unsigned int tag, const string& data, string& response, uint32_t& qhash, RecursorPacketCache::OptPBData& pbData, bool tcp, const ComboAddress& source, const ComboAddress& mappedSource) { - if (!t_packetCache) { + if (!g_packetCache) { return false; } bool cacheHit = false; @@ -1965,10 +1965,10 @@ bool checkForCacheHit(bool qnameParsed, unsigned int tag, const string& data, vState valState; if (qnameParsed) { - cacheHit = t_packetCache->getResponsePacket(tag, data, qname, qtype, qclass, now.tv_sec, &response, &age, &valState, &qhash, &pbData, tcp); + cacheHit = g_packetCache->getResponsePacket(tag, data, qname, qtype, qclass, now.tv_sec, &response, &age, &valState, &qhash, &pbData, tcp); } else { - cacheHit = t_packetCache->getResponsePacket(tag, data, qname, &qtype, &qclass, now.tv_sec, &response, &age, &valState, &qhash, &pbData, tcp); + cacheHit = g_packetCache->getResponsePacket(tag, data, qname, &qtype, &qclass, now.tv_sec, &response, &age, &valState, &qhash, &pbData, tcp); } if (cacheHit) { diff --git a/pdns/recursordist/rec-main.cc b/pdns/recursordist/rec-main.cc index 6b84332ba2..4720cca1b3 100644 --- a/pdns/recursordist/rec-main.cc +++ b/pdns/recursordist/rec-main.cc @@ -992,8 +992,8 @@ static void doStats(void) auto taskPushes = getTaskPushes(); auto taskExpired = getTaskExpired(); auto taskSize = getTaskSize(); - uint64_t pcSize = broadcastAccFunction(pleaseGetPacketCacheSize); - uint64_t pcHits = broadcastAccFunction(pleaseGetPacketCacheHits); + uint64_t pcSize = g_packetCache ? g_packetCache->size() : 0; + uint64_t pcHits = g_packetCache ? g_packetCache->getHits() : 0; auto log = g_slog->withName("stats"); @@ -2139,14 +2139,6 @@ static void houseKeeping(void*) t_Counters.updateSnap(now, g_regressionTestMode); // Below are the tasks that run for every recursorThread, including handler and taskThread - if (t_packetCache) { - static thread_local PeriodicTask packetCacheTask{"packetCacheTask", 5}; - packetCacheTask.runIfDue(now, []() { - size_t sz = g_maxPacketCacheEntries / (RecThreadInfo::numWorkers() + RecThreadInfo::numDistributors()); - t_packetCache->setMaxSize(sz); // g_maxPacketCacheEntries might have changed by rec_control - t_packetCache->doPruneTo(sz); - }); - } static thread_local PeriodicTask pruneTCPTask{"pruneTCPTask", 5}; pruneTCPTask.runIfDue(now, [now]() { @@ -2184,6 +2176,13 @@ static void houseKeeping(void*) }); } else if (info.isHandler()) { + if (g_packetCache) { + static PeriodicTask packetCacheTask{"packetCacheTask", 5}; + packetCacheTask.runIfDue(now, []() { + g_packetCache->setMaxSize(g_maxPacketCacheEntries); // g_maxPacketCacheEntries might have changed by rec_control + g_packetCache->doPruneTo(g_maxPacketCacheEntries); + }); + } static PeriodicTask recordCachePruneTask{"RecordCachePruneTask", 5}; recordCachePruneTask.runIfDue(now, []() { g_recCache->doPrune(g_maxCacheEntries); @@ -2362,11 +2361,6 @@ static void recursorThread() } } - if (!::arg().mustDo("disable-packetcache") && (threadInfo.isDistributor() || threadInfo.isWorker())) { - // Only enable packet cache for thread processing queries from the outside world - t_packetCache = std::make_unique(g_maxPacketCacheEntries / (RecThreadInfo::numWorkers() + RecThreadInfo::numDistributors())); - } - #ifdef NOD_ENABLED if (threadInfo.isWorker()) setupNODThread(log); @@ -3035,6 +3029,10 @@ int main(int argc, char** argv) g_recCache = std::make_unique(::arg().asNum("record-cache-shards")); g_negCache = std::make_unique(::arg().asNum("record-cache-shards") / 8); + if (!::arg().mustDo("disable-packetcache")) { + // Only enable packet cache for thread processing queries from the outside world + g_packetCache = std::make_unique(g_maxPacketCacheEntries /*, shards */); + } ret = serviceMain(argc, argv, startupLog); } @@ -3126,11 +3124,6 @@ string doTraceRegex(FDWrapper file, vector::const_iterator begin, vector return broadcastAccFunction([=] { return pleaseUseNewTraceRegex(begin != end ? *begin : "", fileno); }); } -static uint64_t* pleaseWipePacketCache(const DNSName& canon, bool subtree, uint16_t qtype) -{ - return new uint64_t(t_packetCache ? t_packetCache->doWipePacketCache(canon, qtype, subtree) : 0); -} - struct WipeCacheResult wipeCaches(const DNSName& canon, bool subtree, uint16_t qtype) { struct WipeCacheResult res; @@ -3138,7 +3131,9 @@ struct WipeCacheResult wipeCaches(const DNSName& canon, bool subtree, uint16_t q try { res.record_count = g_recCache->doWipeCache(canon, subtree, qtype); // scanbuild complains here about an allocated function object that is being leaked. Needs investigation - res.packet_count = broadcastAccFunction([=] { return pleaseWipePacketCache(canon, subtree, qtype); }); + if (g_packetCache) { + res.packet_count = g_packetCache->doWipePacketCache(canon, qtype, subtree); + } res.negative_record_count = g_negCache->wipe(canon, subtree); if (g_aggressiveNSECCache) { g_aggressiveNSECCache->removeZoneInfo(canon, subtree); diff --git a/pdns/recursordist/rec-main.hh b/pdns/recursordist/rec-main.hh index 1a1c9ee630..72ded229e7 100644 --- a/pdns/recursordist/rec-main.hh +++ b/pdns/recursordist/rec-main.hh @@ -187,7 +187,7 @@ enum class PaddingMode typedef MTasker, PacketBuffer, PacketIDCompare> MT_t; extern thread_local std::unique_ptr MT; // the big MTasker -extern thread_local std::unique_ptr t_packetCache; +extern std::unique_ptr g_packetCache; using RemoteLoggerStats_t = std::unordered_map; diff --git a/pdns/recursordist/rec-tcounters.hh b/pdns/recursordist/rec-tcounters.hh index 7d59bde46c..db5b311192 100644 --- a/pdns/recursordist/rec-tcounters.hh +++ b/pdns/recursordist/rec-tcounters.hh @@ -95,6 +95,8 @@ enum class Counter : uint8_t maintenanceUsec, maintenanceCalls, + pcHits, + pcMisses, numberOfCounters }; diff --git a/pdns/recursordist/rec_channel_rec.cc b/pdns/recursordist/rec_channel_rec.cc index 2766c7cf57..9bbc49df6e 100644 --- a/pdns/recursordist/rec_channel_rec.cc +++ b/pdns/recursordist/rec_channel_rec.cc @@ -334,11 +334,6 @@ static uint64_t dumpAggressiveNSECCache(int fd) return g_aggressiveNSECCache->dumpToFile(fp, now); } -static uint64_t* pleaseDump(int fd) -{ - return new uint64_t(t_packetCache ? t_packetCache->doDump(fd) : 0); -} - static uint64_t* pleaseDumpEDNSMap(int fd) { return new uint64_t(SyncRes::doEDNSDump(fd)); @@ -416,7 +411,8 @@ static RecursorControlChannel::Answer doDumpCache(int s) uint64_t total = 0; try { int fd = fdw; - total = g_recCache->doDump(fd, g_maxCacheEntries.load()) + g_negCache->doDump(fd, g_maxCacheEntries.load() / 8) + broadcastAccFunction([fd] { return pleaseDump(fd); }) + dumpAggressiveNSECCache(fd); + total = g_recCache->doDump(fd, g_maxCacheEntries.load()) + g_negCache->doDump(fd, g_maxCacheEntries.load() / 8) + + (g_packetCache ? g_packetCache->doDump(fd) : 0) + dumpAggressiveNSECCache(fd); } catch (...) { } @@ -1078,46 +1074,6 @@ static uint64_t doGetCacheMisses() return g_recCache->cacheMisses; } -uint64_t* pleaseGetPacketCacheSize() -{ - return new uint64_t(t_packetCache ? t_packetCache->size() : 0); -} - -static uint64_t* pleaseGetPacketCacheBytes() -{ - return new uint64_t(t_packetCache ? t_packetCache->bytes() : 0); -} - -static uint64_t doGetPacketCacheSize() -{ - return broadcastAccFunction(pleaseGetPacketCacheSize); -} - -static uint64_t doGetPacketCacheBytes() -{ - return broadcastAccFunction(pleaseGetPacketCacheBytes); -} - -uint64_t* pleaseGetPacketCacheHits() -{ - return new uint64_t(t_packetCache ? t_packetCache->d_hits : 0); -} - -static uint64_t doGetPacketCacheHits() -{ - return broadcastAccFunction(pleaseGetPacketCacheHits); -} - -static uint64_t* pleaseGetPacketCacheMisses() -{ - return new uint64_t(t_packetCache ? t_packetCache->d_misses : 0); -} - -static uint64_t doGetPacketCacheMisses() -{ - return broadcastAccFunction(pleaseGetPacketCacheMisses); -} - static uint64_t doGetMallocated() { // this turned out to be broken @@ -1306,10 +1262,10 @@ static void registerAllStats1() addGetStat("record-cache-contended", []() { return g_recCache->stats().first; }); addGetStat("record-cache-acquired", []() { return g_recCache->stats().second; }); - addGetStat("packetcache-hits", doGetPacketCacheHits); - addGetStat("packetcache-misses", doGetPacketCacheMisses); - addGetStat("packetcache-entries", doGetPacketCacheSize); - addGetStat("packetcache-bytes", doGetPacketCacheBytes); + addGetStat("packetcache-hits", [] { return g_packetCache ? g_packetCache->getHits() : 0; }); + addGetStat("packetcache-misses", [] { return g_packetCache ? g_packetCache->getMisses() : 0; }); + addGetStat("packetcache-entries", [] { return g_packetCache ? g_packetCache->size() : 0; }); + addGetStat("packetcache-bytes", [] { return g_packetCache ? g_packetCache->bytes() : 0; });; addGetStat("aggressive-nsec-cache-entries", []() { return g_aggressiveNSECCache ? g_aggressiveNSECCache->getEntriesCount() : 0; }); addGetStat("aggressive-nsec-cache-nsec-hits", []() { return g_aggressiveNSECCache ? g_aggressiveNSECCache->getNSECHits() : 0; }); diff --git a/pdns/recursordist/recpacketcache.cc b/pdns/recursordist/recpacketcache.cc index 054efc9714..10fa9cbd1d 100644 --- a/pdns/recursordist/recpacketcache.cc +++ b/pdns/recursordist/recpacketcache.cc @@ -12,6 +12,47 @@ unsigned int RecursorPacketCache::s_refresh_ttlperc{0}; +uint64_t RecursorPacketCache::size() const +{ + uint64_t count = 0; + for (const auto& map : d_maps) { + count += map.d_entriesCount; + } + return count; +} + +uint64_t RecursorPacketCache::bytes() +{ + uint64_t sum = 0; + for (auto& shard : d_maps) { + auto lock = shard.lock(); + for (const auto& entry : lock->d_map) { + sum += sizeof(entry) + entry.d_packet.length() + 4; + } + } + return sum; +} + +uint64_t RecursorPacketCache::getHits() +{ + uint64_t sum = 0; + for (auto& shard : d_maps) { + auto lock = shard.lock(); + sum += lock->d_hits; + } + return sum; +} + +uint64_t RecursorPacketCache::getMisses() +{ + uint64_t sum = 0; + for (auto& shard : d_maps) { + auto lock = shard.lock(); + sum += lock->d_misses; + } + return sum; +} + uint64_t RecursorPacketCache::doWipePacketCache(const DNSName& name, uint16_t qtype, bool subtree) { uint64_t count = 0; @@ -53,7 +94,7 @@ bool RecursorPacketCache::qrMatch(const packetCache_t::index::type::ite return queryMatches(iter->d_query, queryPacket, qname, optionsToSkip); } -bool RecursorPacketCache::checkResponseMatches(packetCache_t& shard, std::pair::type::iterator, packetCache_t::index::type::iterator> range, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, OptPBData* pbdata) +bool RecursorPacketCache::checkResponseMatches(MapCombo::LockedContent& shard, std::pair::type::iterator, packetCache_t::index::type::iterator> range, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, OptPBData* pbdata) { for (auto iter = range.first; iter != range.second; ++iter) { // the possibility is VERY real that we get hits that are not right - birthday paradox @@ -82,8 +123,8 @@ bool RecursorPacketCache::checkResponseMatches(packetCache_t& shard, std::pair

replace(sizeof(dnsheader), wirelength, queryPacket, sizeof(dnsheader), wirelength); } - d_hits++; - moveCacheItemToBack(shard, iter); + shard.d_hits++; + moveCacheItemToBack(shard.d_map, iter); if (pbdata != nullptr) { if (iter->d_pbdata) { @@ -98,7 +139,7 @@ bool RecursorPacketCache::checkResponseMatches(packetCache_t& shard, std::pair

d_map.get(); auto range = idx.equal_range(std::tie(tag, *qhash, tcp)); if (range.first == range.second) { - d_misses++; + shard->d_misses++; return false; } - return checkResponseMatches(shard->d_map, range, queryPacket, qname, qtype, qclass, now, responsePacket, age, valState, pbdata); + return checkResponseMatches(*shard, range, queryPacket, qname, qtype, qclass, now, responsePacket, age, valState, pbdata); } bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& queryPacket, DNSName& qname, uint16_t* qtype, uint16_t* qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, OptPBData* pbdata, bool tcp) { *qhash = canHashPacket(queryPacket, s_skipOptions); - auto shard = getMap(tag, *qhash, tcp).lock(); + auto& map = getMap(tag, *qhash, tcp); + auto shard = map.lock(); const auto& idx = shard->d_map.get(); auto range = idx.equal_range(std::tie(tag, *qhash, tcp)); if (range.first == range.second) { - d_misses++; + shard->d_misses++; return false; } qname = DNSName(queryPacket.c_str(), static_cast(queryPacket.length()), sizeof(dnsheader), false, qtype, qclass); - return checkResponseMatches(shard->d_map, range, queryPacket, qname, *qtype, *qclass, now, responsePacket, age, valState, pbdata); + return checkResponseMatches(*shard, range, queryPacket, qname, *qtype, *qclass, now, responsePacket, age, valState, pbdata); } void RecursorPacketCache::insertResponsePacket(unsigned int tag, uint32_t qhash, std::string&& query, const DNSName& qname, uint16_t qtype, uint16_t qclass, std::string&& responsePacket, time_t now, uint32_t ttl, const vState& valState, OptPBData&& pbdata, bool tcp) @@ -184,18 +227,6 @@ void RecursorPacketCache::insertResponsePacket(unsigned int tag, uint32_t qhash, assert(map.d_entriesCount == shard->d_map.size()); // XXX } -uint64_t RecursorPacketCache::bytes() -{ - uint64_t sum = 0; - for (auto& shard : d_maps) { - auto lock = shard.lock(); - for (const auto& entry : lock->d_map) { - sum += sizeof(entry) + entry.d_packet.length() + 4; - } - } - return sum; -} - void RecursorPacketCache::doPruneTo(size_t maxSize) { size_t cacheSize = size(); diff --git a/pdns/recursordist/recpacketcache.hh b/pdns/recursordist/recpacketcache.hh index 72f58d526a..8ebc46ecd4 100644 --- a/pdns/recursordist/recpacketcache.hh +++ b/pdns/recursordist/recpacketcache.hh @@ -95,18 +95,10 @@ public: d_maxSize = size; } - [[nodiscard]] uint64_t size() const - { - uint64_t count = 0; - for (const auto& map : d_maps) { - count += map.d_entriesCount; - } - return count; - } + [[nodiscard]] uint64_t size() const; [[nodiscard]] uint64_t bytes(); - - uint64_t d_hits{0}; - uint64_t d_misses{0}; + [[nodiscard]] uint64_t getHits(); + [[nodiscard]] uint64_t getMisses(); private: struct Entry @@ -169,6 +161,8 @@ private: struct LockedContent { packetCache_t d_map; + uint64_t d_hits{0}; + uint64_t d_misses{0}; uint64_t d_contended_count{0}; uint64_t d_acquired_count{0}; void invalidate() {} @@ -213,7 +207,7 @@ private: size_t d_maxSize; static bool qrMatch(const packetCache_t::index::type::iterator& iter, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass); - bool checkResponseMatches(packetCache_t& shard, std::pair::type::iterator, packetCache_t::index::type::iterator> range, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, OptPBData* pbdata); +bool checkResponseMatches(MapCombo::LockedContent& shard, std::pair::type::iterator, packetCache_t::index::type::iterator> range, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, OptPBData* pbdata); public: void preRemoval(MapCombo::LockedContent& map, const Entry& entry) { diff --git a/pdns/recursordist/syncres.hh b/pdns/recursordist/syncres.hh index 496ea28efe..fef7b96178 100644 --- a/pdns/recursordist/syncres.hh +++ b/pdns/recursordist/syncres.hh @@ -922,8 +922,6 @@ uint64_t* pleaseGetNsSpeedsSize(); uint64_t* pleaseGetFailedServersSize(); uint64_t* pleaseGetConcurrentQueries(); uint64_t* pleaseGetThrottleSize(); -uint64_t* pleaseGetPacketCacheHits(); -uint64_t* pleaseGetPacketCacheSize(); void doCarbonDump(void*); bool primeHints(time_t now = time(nullptr)); const char* isoDateTimeMillis(const struct timeval& tv, char* buf, size_t sz); -- 2.47.2