From: Otto Moerbeek Date: Mon, 2 Dec 2019 15:04:26 +0000 (+0100) Subject: wip X-Git-Tag: dnsdist-1.5.0-alpha1~21^2^2~12 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a7956123303e2df40101341315308c61ca1fad7d;p=thirdparty%2Fpdns.git wip --- diff --git a/pdns/auth-packetcache.hh b/pdns/auth-packetcache.hh index 0d3af47a25..bc3c2f35bd 100644 --- a/pdns/auth-packetcache.hh +++ b/pdns/auth-packetcache.hh @@ -89,6 +89,11 @@ private: uint32_t hash{0}; uint16_t qtype{0}; bool tcp{false}; + + time_t getTTD() const + { + return ttd; + } }; struct HashTag{}; @@ -145,4 +150,9 @@ private: bool d_cleanskipped{false}; static const unsigned int s_mincleaninterval=1000, s_maxcleaninterval=300000; + +public: + void preRemoval(const CacheEntry&) + { + } }; diff --git a/pdns/auth-querycache.hh b/pdns/auth-querycache.hh index 889634bb60..00465ad73e 100644 --- a/pdns/auth-querycache.hh +++ b/pdns/auth-querycache.hh @@ -68,6 +68,11 @@ private: mutable time_t ttd{0}; uint16_t qtype{0}; int zoneID{-1}; + + time_t getTTD() const + { + return ttd; + } }; struct HashTag{}; @@ -126,4 +131,9 @@ private: bool d_cleanskipped{false}; static const unsigned int s_mincleaninterval=1000, s_maxcleaninterval=300000; + +public: + void preRemoval(const CacheEntry&) + { + } }; diff --git a/pdns/cachecleaner.hh b/pdns/cachecleaner.hh index 732631d001..9d57ede009 100644 --- a/pdns/cachecleaner.hh +++ b/pdns/cachecleaner.hh @@ -21,6 +21,7 @@ */ #pragma once +#include #include "lock.hh" // this function can clean any cache that has a getTTD() method on its entries, a preRemoval() method and a 'sequence' index as its second index @@ -56,7 +57,7 @@ template void pruneCollection(C& container, for(; iter != sidx.end() && tried < lookAt ; ++tried) { if(iter->getTTD() < now) { container.preRemoval(*iter); - sidx.erase(iter++); + iter = sidx.erase(iter); erased++; } else @@ -132,7 +133,7 @@ template uint64_t pruneLockedCollectionsVector(vector(mc.d_map); uint64_t erased = 0, lookedAt = 0; for(auto i = sidx.begin(); i != sidx.end(); lookedAt++) { - if(i->ttd < now) { + if (i->getTTD() < now) { i = sidx.erase(i); erased++; } else { @@ -151,6 +152,69 @@ template uint64_t pruneLockedCollectionsVector(vector uint64_t pruneMutexCollectionsVector(C& container, vector& maps, uint64_t maxCached, uint64_t cacheSize) +{ + time_t now = time(nullptr); + uint64_t totErased = 0; + uint64_t toTrim = 0; + uint64_t lookAt = 0; + + // two modes - if toTrim is 0, just look through 10% of the cache and nuke everything that is expired + // otherwise, scan first 5*toTrim records, and stop once we've nuked enough + if (cacheSize > maxCached) { + toTrim = cacheSize - maxCached; + lookAt = 5 * toTrim; + } else { + lookAt = cacheSize / 10; + } + + for (auto& mc : maps) { + const std::lock_guard lock(mc.mutex); + auto& sidx = boost::multi_index::get(mc.d_map); + uint64_t erased = 0, lookedAt = 0; + for (auto i = sidx.begin(); i != sidx.end(); lookedAt++) { + if (i->getTTD() < now) { + container.preRemoval(*i); + i = sidx.erase(i); + erased++; + } else { + ++i; + } + + if (toTrim && erased >= toTrim / maps.size()) + break; + + if (lookedAt > lookAt / maps.size()) + break; + } + totErased += erased; + if (toTrim && totErased >= toTrim) + break; + } + + if (totErased >= toTrim) { // done + return totErased; + } + + toTrim -= totErased; + + // XXXX kinda desperate method + while (toTrim > 0) { + for (auto& mc : maps) { + const std::lock_guard lock(mc.mutex); + auto& sidx = boost::multi_index::get(mc.d_map); + auto i = sidx.begin(); + container.preRemoval(*i); + i = sidx.erase(i); + totErased++; + toTrim--; + if (toTrim == 0) + break; + } + } + return totErased; +} + template uint64_t purgeLockedCollectionsVector(vector& maps) { uint64_t delcount=0; diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc index 06d3c509c0..d8a09f3af6 100644 --- a/pdns/pdns_recursor.cc +++ b/pdns/pdns_recursor.cc @@ -127,7 +127,9 @@ static thread_local uint64_t t_frameStreamServersGeneration; #endif /* HAVE_FSTRM */ thread_local std::unique_ptr MT; // the big MTasker -thread_local std::unique_ptr t_RC; +std::unique_ptr s_RC = std::unique_ptr(new MemRecursorCache()); + + thread_local std::unique_ptr t_packetCache; thread_local FDMultiplexer* t_fdm{nullptr}; thread_local std::unique_ptr t_remotes, t_servfailremotes, t_largeanswerremotes, t_bogusremotes; @@ -1821,10 +1823,10 @@ static void startDoResolve(void *p) } if (sr.d_outqueries || sr.d_authzonequeries) { - t_RC->cacheMisses++; + s_RC->cacheMisses++; } else { - t_RC->cacheHits++; + s_RC->cacheHits++; } if(spent < 0.001) @@ -2913,7 +2915,6 @@ static void houseKeeping(void *) past = now; past.tv_sec -= 5; if (last_prune < past) { - t_RC->doPrune(g_maxCacheEntries / g_numThreads); // this function is local to a thread, so fine anyhow t_packetCache->doPruneTo(g_maxPacketCacheEntries / g_numWorkerThreads); SyncRes::pruneNegCache(g_maxCacheEntries / (g_numWorkerThreads * 10)); diff --git a/pdns/rec_channel_rec.cc b/pdns/rec_channel_rec.cc index 5eccb7583b..b09f3b6d7c 100644 --- a/pdns/rec_channel_rec.cc +++ b/pdns/rec_channel_rec.cc @@ -213,7 +213,7 @@ static uint64_t dumpNegCache(NegCache& negcache, int fd) static uint64_t* pleaseDump(int fd) { - return new uint64_t(t_RC->doDump(fd) + dumpNegCache(SyncRes::t_sstorage.negcache, fd) + t_packetCache->doDump(fd)); + return new uint64_t(s_RC->doDump(fd) + dumpNegCache(SyncRes::t_sstorage.negcache, fd) + t_packetCache->doDump(fd)); } static uint64_t* pleaseDumpEDNSMap(int fd) @@ -396,7 +396,7 @@ static string doDumpFailedServers(T begin, T end) uint64_t* pleaseWipeCache(const DNSName& canon, bool subtree, uint16_t qtype) { - return new uint64_t(t_RC->doWipeCache(canon, subtree, qtype)); + return new uint64_t(s_RC->doWipeCache(canon, subtree)); } uint64_t* pleaseWipePacketCache(const DNSName& canon, bool subtree, uint16_t qtype) @@ -934,12 +934,12 @@ static uint64_t getConcurrentQueries() uint64_t* pleaseGetCacheSize() { - return new uint64_t(t_RC ? t_RC->size() : 0); + return new uint64_t(s_RC ? s_RC->size() : 0); } static uint64_t* pleaseGetCacheBytes() { - return new uint64_t(t_RC ? t_RC->bytes() : 0); + return new uint64_t(s_RC ? s_RC->bytes() : 0); } static uint64_t doGetCacheSize() @@ -959,7 +959,7 @@ static uint64_t doGetCacheBytes() uint64_t* pleaseGetCacheHits() { - return new uint64_t(t_RC ? t_RC->cacheHits : 0); + return new uint64_t(s_RC->cacheHits); } static uint64_t doGetCacheHits() @@ -969,7 +969,7 @@ static uint64_t doGetCacheHits() uint64_t* pleaseGetCacheMisses() { - return new uint64_t(t_RC ? t_RC->cacheMisses : 0); + return new uint64_t(s_RC->cacheMisses); } static uint64_t doGetCacheMisses() diff --git a/pdns/recursor_cache.cc b/pdns/recursor_cache.cc index 0ea942ed49..cfe65173e9 100644 --- a/pdns/recursor_cache.cc +++ b/pdns/recursor_cache.cc @@ -11,35 +11,68 @@ #include "arguments.hh" #include "syncres.hh" #include "recursor_cache.hh" -#include "cachecleaner.hh" #include "namespaces.hh" +#include "cachecleaner.hh" -unsigned int MemRecursorCache::size() const +MemRecursorCache::MemRecursorCache(size_t mapsCount) : d_maps(mapsCount) { - return (unsigned int)d_cache.size(); } -size_t MemRecursorCache::ecsIndexSize() const +MemRecursorCache::~MemRecursorCache() { - return d_ecsIndex.size(); + try { + typedef std::unique_ptr> lock_t; + vector locks; + for (auto& map : d_maps) { + locks.push_back(lock_t(new std::lock_guard(map.mutex))); + } + } + catch(...) { + } } -// this function is too slow to poll! -unsigned int MemRecursorCache::bytes() const +size_t MemRecursorCache::size() { - unsigned int ret=0; + // XXX! + size_t count = 0; + for (auto& map : d_maps) { + const std::lock_guard lock(map.mutex); + count += map.d_map.size(); + } + return count; +} - for(const auto& i : d_cache) { - ret+=sizeof(struct CacheEntry); - ret+=(unsigned int)i.d_qname.toString().length(); - for(const auto& record : i.d_records) - ret+= sizeof(record); // XXX WRONG we don't know the stored size! +size_t MemRecursorCache::ecsIndexSize() +{ + // XXX! + size_t count = 0; + for (auto& map : d_maps) { + const std::lock_guard lock(map.mutex); + count += map.d_ecsIndex.size(); + } + return count; +} + +// this function is too slow to poll! +size_t MemRecursorCache::bytes() +{ + size_t ret = 0; + for (auto& map : d_maps) { + const std::lock_guard lock(map.mutex); + for (const auto& i : map.d_map) { + ret += sizeof(struct CacheEntry); + ret += i.d_qname.toString().length(); + for (const auto& record : i.d_records) { + ret += sizeof(record); // XXX WRONG we don't know the stored size! + } + } } return ret; } -int32_t MemRecursorCache::handleHit(MemRecursorCache::OrderedTagIterator_t& entry, const DNSName& qname, const ComboAddress& who, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth) +int32_t MemRecursorCache::handleHit(MapCombo& map, MemRecursorCache::OrderedTagIterator_t& entry, const DNSName& qname, const ComboAddress& who, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth) { + // MUTEX SHOULD BE ACQUIRED int32_t ttd = entry->d_ttd; if(variable && !entry->d_netmask.empty()) { @@ -78,16 +111,17 @@ int32_t MemRecursorCache::handleHit(MemRecursorCache::OrderedTagIterator_t& entr *wasAuth = entry->d_auth; } - moveCacheItemToBack(d_cache, entry); + moveCacheItemToBack(map.d_map, entry); return ttd; } -MemRecursorCache::cache_t::const_iterator MemRecursorCache::getEntryUsingECSIndex(time_t now, const DNSName &qname, uint16_t qtype, bool requireAuth, const ComboAddress& who) +MemRecursorCache::cache_t::const_iterator MemRecursorCache::getEntryUsingECSIndex(MapCombo& map, time_t now, const DNSName &qname, uint16_t qtype, bool requireAuth, const ComboAddress& who) { + // MUTEX SHOULD BE ACQUIRED auto ecsIndexKey = tie(qname, qtype); - auto ecsIndex = d_ecsIndex.find(ecsIndexKey); - if (ecsIndex != d_ecsIndex.end() && !ecsIndex->isEmpty()) { + auto ecsIndex = map.d_ecsIndex.find(ecsIndexKey); + if (ecsIndex != map.d_ecsIndex.end() && !ecsIndex->isEmpty()) { /* we have netmask-specific entries, let's see if we match one */ while (true) { const Netmask best = ecsIndex->lookupBestMatch(who); @@ -96,12 +130,12 @@ MemRecursorCache::cache_t::const_iterator MemRecursorCache::getEntryUsingECSInde break; } auto key = boost::make_tuple(qname, qtype, best); - auto entry = d_cache.find(key); - if (entry == d_cache.end()) { + auto entry = map.d_map.find(key); + if (entry == map.d_map.end()) { /* ecsIndex is not up-to-date */ ecsIndex->removeNetmask(best); if (ecsIndex->isEmpty()) { - d_ecsIndex.erase(ecsIndex); + map.d_ecsIndex.erase(ecsIndex); break; } continue; @@ -112,14 +146,14 @@ MemRecursorCache::cache_t::const_iterator MemRecursorCache::getEntryUsingECSInde return entry; } /* we need auth data and the best match is not authoritative */ - return d_cache.end(); + return map.d_map.end(); } else { /* this netmask-specific entry has expired */ - moveCacheItemToFront(d_cache, entry); + moveCacheItemToFront(map.d_map, entry); ecsIndex->removeNetmask(best); if (ecsIndex->isEmpty()) { - d_ecsIndex.erase(ecsIndex); + map.d_ecsIndex.erase(ecsIndex); break; } } @@ -128,40 +162,37 @@ MemRecursorCache::cache_t::const_iterator MemRecursorCache::getEntryUsingECSInde /* we have nothing specific, let's see if we have a generic one */ auto key = boost::make_tuple(qname, qtype, Netmask()); - auto entry = d_cache.find(key); - if (entry != d_cache.end()) { + auto entry = map.d_map.find(key); + if (entry != map.d_map.end()) { if (entry->d_ttd > now) { if (!requireAuth || entry->d_auth) { return entry; } } else { - moveCacheItemToFront(d_cache, entry); + moveCacheItemToFront(map.d_map, entry); } } /* nothing for you, sorry */ - return d_cache.end(); + return map.d_map.end(); } -// returns -1 for no hits -std::pair MemRecursorCache::getEntries(const DNSName &qname, const QType& qt) +std::pair MemRecursorCache::getEntries(MapCombo& map, const DNSName &qname, const QType& qt) { - // cerr<<"looking up "<< qname<<"|"+qt.getName()<<"\n"; - if(!d_cachecachevalid || d_cachedqname!= qname) { - // cerr<<"had cache cache miss"<(); - d_cachecache = idx.equal_range(qname); - d_cachecachevalid = true; + // MUTEX SHOULD BE ACQUIRED + if (!map.d_cachecachevalid || map.d_cachedqname != qname) { + map.d_cachedqname = qname; + const auto& idx = map.d_map.get(); + map.d_cachecache = idx.equal_range(qname); + map.d_cachecachevalid = true; } - // else cerr<<"had cache cache hit!"<d_auth) return false; @@ -178,21 +209,25 @@ int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, if(res) { res->clear(); } - const uint16_t qtype = qt.getCode(); + + auto& map = getMap(qname); + const std::lock_guard lock(map.mutex); + /* If we don't have any netmask-specific entries at all, let's just skip this to be able to use the nice d_cachecache hack. */ - if (qtype != QType::ANY && !d_ecsIndex.empty()) { + if (qtype != QType::ANY && !map.d_ecsIndex.empty()) { + if (qtype == QType::ADDR) { int32_t ret = -1; - auto entryA = getEntryUsingECSIndex(now, qname, QType::A, requireAuth, who); - if (entryA != d_cache.end()) { - ret = handleHit(entryA, qname, who, res, signatures, authorityRecs, variable, state, wasAuth); + auto entryA = getEntryUsingECSIndex(map, now, qname, QType::A, requireAuth, who); + if (entryA != map.d_map.end()) { + ret = handleHit(map, entryA, qname, who, res, signatures, authorityRecs, variable, state, wasAuth); } - auto entryAAAA = getEntryUsingECSIndex(now, qname, QType::AAAA, requireAuth, who); - if (entryAAAA != d_cache.end()) { - int32_t ttdAAAA = handleHit(entryAAAA, qname, who, res, signatures, authorityRecs, variable, state, wasAuth); + auto entryAAAA = getEntryUsingECSIndex(map, now, qname, QType::AAAA, requireAuth, who); + if (entryAAAA != map.d_map.end()) { + int32_t ttdAAAA = handleHit(map, entryAAAA, qname, who, res, signatures, authorityRecs, variable, state, wasAuth); if (ret > 0) { ret = std::min(ret, ttdAAAA); } else { @@ -202,29 +237,29 @@ int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, return ret > 0 ? static_cast(ret-now) : ret; } else { - auto entry = getEntryUsingECSIndex(now, qname, qtype, requireAuth, who); - if (entry != d_cache.end()) { - return static_cast(handleHit(entry, qname, who, res, signatures, authorityRecs, variable, state, wasAuth) - now); + auto entry = getEntryUsingECSIndex(map, now, qname, qtype, requireAuth, who); + if (entry != map.d_map.end()) { + return static_cast(handleHit(map, entry, qname, who, res, signatures, authorityRecs, variable, state, wasAuth) - now); } return -1; } } - auto entries = getEntries(qname, qt); + auto entries = getEntries(map, qname, qt); if(entries.first!=entries.second) { for(auto i=entries.first; i != entries.second; ++i) { - auto firstIndexIterator = d_cache.project(i); + auto firstIndexIterator = map.d_map.project(i); if (i->d_ttd <= now) { - moveCacheItemToFront(d_cache, firstIndexIterator); + moveCacheItemToFront(map.d_map, firstIndexIterator); continue; } if (!entryMatches(firstIndexIterator, qtype, requireAuth, who)) continue; - ttd = handleHit(firstIndexIterator, qname, who, res, signatures, authorityRecs, variable, state, wasAuth); + ttd = handleHit(map, firstIndexIterator, qname, who, res, signatures, authorityRecs, variable, state, wasAuth); if(qt.getCode()!=QType::ANY && qt.getCode()!=QType::ADDR) // normally if we have a hit, we are done break; @@ -238,16 +273,19 @@ int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, void MemRecursorCache::replace(time_t now, const DNSName &qname, const QType& qt, const vector& content, const vector>& signatures, const std::vector>& authorityRecs, bool auth, boost::optional ednsmask, vState state) { - d_cachecachevalid = false; + auto& map = getMap(qname); + const std::lock_guard lock(map.mutex); + + map.d_cachecachevalid = false; // cerr<<"Replacing "<toString() : "everyone") << endl; if (ednsmask) { ednsmask = ednsmask->getNormalized(); } auto key = boost::make_tuple(qname, qt.getCode(), ednsmask ? *ednsmask : Netmask()); bool isNew = false; - cache_t::iterator stored = d_cache.find(key); - if (stored == d_cache.end()) { - stored = d_cache.insert(CacheEntry(key, auth)).first; + cache_t::iterator stored = map.d_map.find(key); + if (stored == map.d_map.end()) { + stored = map.d_map.insert(CacheEntry(key, auth)).first; isNew = true; } @@ -260,9 +298,9 @@ void MemRecursorCache::replace(time_t now, const DNSName &qname, const QType& qt /* don't bother building an ecsIndex if we don't have any netmask-specific entries */ if (ednsmask && !ednsmask->empty()) { auto ecsIndexKey = boost::make_tuple(qname, qt.getCode()); - auto ecsIndex = d_ecsIndex.find(ecsIndexKey); - if (ecsIndex == d_ecsIndex.end()) { - ecsIndex = d_ecsIndex.insert(ECSIndexEntry(qname, qt.getCode())).first; + auto ecsIndex = map.d_ecsIndex.find(ecsIndexKey); + if (ecsIndex == map.d_ecsIndex.end()) { + ecsIndex = map.d_ecsIndex.insert(ECSIndexEntry(qname, qt.getCode())).first; } ecsIndex->addMask(*ednsmask); } @@ -314,94 +352,87 @@ void MemRecursorCache::replace(time_t now, const DNSName &qname, const QType& qt } if (!isNew) { - moveCacheItemToBack(d_cache, stored); + moveCacheItemToBack(map.d_map, stored); } - d_cache.replace(stored, ce); + map.d_map.replace(stored, ce); } -int MemRecursorCache::doWipeCache(const DNSName& name, bool sub, uint16_t qtype) +size_t MemRecursorCache::doWipeCache(const DNSName& name, bool sub, uint16_t qtype) { - int count=0; - d_cachecachevalid=false; - - if(!sub) { - auto& idx = d_cache.get(); - auto range = idx.equal_range(name); - for(auto& i=range.first; i != range.second; ) { - if (qtype == 0xffff || i->d_qtype == qtype) { - count++; - idx.erase(i++); + size_t count = 0; + + if (!sub) { + for (auto& map : d_maps) { + const std::lock_guard lock(map.mutex); + map.d_cachecachevalid = false; + auto& idx = map.d_map.get(); + count += idx.erase(name); + if (qtype == 0xffff) { + auto& ecsIdx = map.d_ecsIndex.get(); + auto ecsIndexRange = ecsIdx.equal_range(name); + ecsIdx.erase(ecsIndexRange.first, ecsIndexRange.second); } else { - ++i; - } - } - if (qtype == 0xffff) { - auto& ecsIdx = d_ecsIndex.get(); - auto ecsIndexRange = ecsIdx.equal_range(name); - for(auto i = ecsIndexRange.first; i != ecsIndexRange.second; ) { - ecsIdx.erase(i++); - } - } - else { - auto& ecsIdx = d_ecsIndex.get(); - auto ecsIndexRange = ecsIdx.equal_range(tie(name, qtype)); - for(auto i = ecsIndexRange.first; i != ecsIndexRange.second; ) { - ecsIdx.erase(i++); + auto& ecsIdx = map.d_ecsIndex.get(); + auto ecsIndexRange = ecsIdx.equal_range(tie(name, qtype)); + ecsIdx.erase(ecsIndexRange.first, ecsIndexRange.second); } } } else { - auto& idx = d_cache.get(); - auto& ecsIdx = d_ecsIndex.get(); - - for(auto iter = idx.lower_bound(name); iter != idx.end(); ) { - if(!iter->d_qname.isPartOf(name)) - break; - if(iter->d_qtype == qtype || qtype == 0xffff) { - count++; - idx.erase(iter++); - } - else - iter++; - } - for(auto iter = ecsIdx.lower_bound(name); iter != ecsIdx.end(); ) { - if(!iter->d_qname.isPartOf(name)) - break; - if(iter->d_qtype == qtype || qtype == 0xffff) { - ecsIdx.erase(iter++); + for (auto& map : d_maps) { + const std::lock_guard lock(map.mutex); + map.d_cachecachevalid = false; + auto& idx = map.d_map.get(); + for (auto i = idx.lower_bound(name); i != idx.end(); ) { + if (!i->d_qname.isPartOf(name)) + break; + if (i->d_qtype == qtype || qtype == 0xffff) { + count++; + i = idx.erase(i); + } else { + ++i; + } } - else { - iter++; + auto& ecsIdx = map.d_ecsIndex.get(); + for (auto i = ecsIdx.lower_bound(name); i != ecsIdx.end(); ) { + if (!i->d_qname.isPartOf(name)) + break; + if (i->d_qtype == qtype || qtype == 0xffff) { + i = ecsIdx.erase(i); + } else { + ++i; + } } } } return count; } +// Name should be doLimitTime or so bool MemRecursorCache::doAgeCache(time_t now, const DNSName& name, uint16_t qtype, uint32_t newTTL) { - cache_t::iterator iter = d_cache.find(tie(name, qtype)); - if(iter == d_cache.end()) { + auto& map = getMap(name); + const std::lock_guard lock(map.mutex); + cache_t::iterator iter = map.d_map.find(tie(name, qtype)); + if (iter == map.d_map.end()) { return false; } CacheEntry ce = *iter; - if(ce.d_ttd < now) + if (ce.d_ttd < now) return false; // would be dead anyhow uint32_t maxTTL = static_cast(ce.d_ttd - now); - if(maxTTL > newTTL) { - d_cachecachevalid=false; + if (maxTTL > newTTL) { + map.d_cachecachevalid = false; time_t newTTD = now + newTTL; - - if(ce.d_ttd > newTTD) // do never renew expired or older TTLs + if (ce.d_ttd > newTTD) { ce.d_ttd = newTTD; - - - d_cache.replace(iter, ce); + map.d_map.replace(iter, ce); + } return true; } return false; @@ -409,11 +440,14 @@ bool MemRecursorCache::doAgeCache(time_t now, const DNSName& name, uint16_t qtyp bool MemRecursorCache::updateValidationStatus(time_t now, const DNSName &qname, const QType& qt, const ComboAddress& who, bool requireAuth, vState newState, boost::optional capTTD) { + auto& map = getMap(qname); + const std::lock_guard lock(map.mutex); + bool updated = false; uint16_t qtype = qt.getCode(); - if (qtype != QType::ANY && qtype != QType::ADDR && !d_ecsIndex.empty()) { - auto entry = getEntryUsingECSIndex(now, qname, qtype, requireAuth, who); - if (entry == d_cache.end()) { + if (qtype != QType::ANY && qtype != QType::ADDR && !map.d_ecsIndex.empty()) { + auto entry = getEntryUsingECSIndex(map, now, qname, qtype, requireAuth, who); + if (entry == map.d_map.end()) { return false; } @@ -424,10 +458,10 @@ bool MemRecursorCache::updateValidationStatus(time_t now, const DNSName &qname, return true; } - auto entries = getEntries(qname, qt); + auto entries = getEntries(map, qname, qt); for(auto i = entries.first; i != entries.second; ++i) { - auto firstIndexIterator = d_cache.project(i); + auto firstIndexIterator = map.d_map.project(i); if (!entryMatches(firstIndexIterator, qtype, requireAuth, who)) continue; @@ -447,42 +481,52 @@ bool MemRecursorCache::updateValidationStatus(time_t now, const DNSName &qname, uint64_t MemRecursorCache::doDump(int fd) { - auto fp = std::unique_ptr(fdopen(dup(fd), "w"), fclose); + int newfd = dup(fd); + if (newfd == -1) { + return 0; + } + auto fp = std::unique_ptr(fdopen(newfd, "w"), fclose); if(!fp) { // dup probably failed + close(newfd); return 0; } - fprintf(fp.get(), "; main record cache dump from thread follows\n;\n"); - const auto& sidx=d_cache.get(); - uint64_t count=0; - time_t now=time(0); - for(const auto i : sidx) { - for(const auto j : i.d_records) { - count++; - try { - fprintf(fp.get(), "%s %" PRId64 " IN %s %s ; (%s) auth=%i %s\n", i.d_qname.toString().c_str(), static_cast(i.d_ttd - now), DNSRecordContent::NumberToType(i.d_qtype).c_str(), j->getZoneRepresentation().c_str(), vStates[i.d_state], i.d_auth, i.d_netmask.empty() ? "" : i.d_netmask.toString().c_str()); - } - catch(...) { - fprintf(fp.get(), "; error printing '%s'\n", i.d_qname.empty() ? "EMPTY" : i.d_qname.toString().c_str()); - } - } - for(const auto &sig : i.d_signatures) { - count++; - try { - fprintf(fp.get(), "%s %" PRId64 " IN RRSIG %s ; %s\n", i.d_qname.toString().c_str(), static_cast(i.d_ttd - now), sig->getZoneRepresentation().c_str(), i.d_netmask.empty() ? "" : i.d_netmask.toString().c_str()); + fprintf(fp.get(), "; main record cache dump follows\n;\n"); + uint64_t count = 0; + + for (auto& map : d_maps) { + const std::lock_guard lock(map.mutex); + const auto& sidx = map.d_map.get(); + + time_t now = time(0); + for (const auto i : sidx) { + for (const auto j : i.d_records) { + count++; + try { + fprintf(fp.get(), "%s %" PRId64 " IN %s %s ; (%s) auth=%i %s\n", i.d_qname.toString().c_str(), static_cast(i.d_ttd - now), DNSRecordContent::NumberToType(i.d_qtype).c_str(), j->getZoneRepresentation().c_str(), vStates[i.d_state], i.d_auth, i.d_netmask.empty() ? "" : i.d_netmask.toString().c_str()); + } + catch(...) { + fprintf(fp.get(), "; error printing '%s'\n", i.d_qname.empty() ? "EMPTY" : i.d_qname.toString().c_str()); + } } - catch(...) { - fprintf(fp.get(), "; error printing '%s'\n", i.d_qname.empty() ? "EMPTY" : i.d_qname.toString().c_str()); + for (const auto &sig : i.d_signatures) { + count++; + try { + fprintf(fp.get(), "%s %" PRId64 " IN RRSIG %s ; %s\n", i.d_qname.toString().c_str(), static_cast(i.d_ttd - now), sig->getZoneRepresentation().c_str(), i.d_netmask.empty() ? "" : i.d_netmask.toString().c_str()); + } + catch(...) { + fprintf(fp.get(), "; error printing '%s'\n", i.d_qname.empty() ? "EMPTY" : i.d_qname.toString().c_str()); + } } } } return count; } -void MemRecursorCache::doPrune(unsigned int keep) +void MemRecursorCache::doPrune(size_t keep) { - d_cachecachevalid=false; - - pruneCollection(*this, d_cache, keep); + //size_t maxCached = d_maxEntries; + size_t cacheSize = size(); + pruneMutexCollectionsVector(*this, d_maps, keep, cacheSize); } diff --git a/pdns/recursor_cache.hh b/pdns/recursor_cache.hh index 21c6991e89..48bcbb2954 100644 --- a/pdns/recursor_cache.hh +++ b/pdns/recursor_cache.hh @@ -22,6 +22,7 @@ #pragma once #include #include +#include #include "dns.hh" #include "qtype.hh" #include "misc.hh" @@ -46,26 +47,25 @@ using namespace ::boost::multi_index; class MemRecursorCache : public boost::noncopyable // : public RecursorCache { public: - MemRecursorCache() : d_cachecachevalid(false) - { - cacheHits = cacheMisses = 0; - } - unsigned int size() const; - unsigned int bytes() const; - size_t ecsIndexSize() const; + MemRecursorCache(size_t mapsCount = 1024); + ~MemRecursorCache(); + + size_t size(); + size_t bytes(); + size_t ecsIndexSize(); int32_t get(time_t, const DNSName &qname, const QType& qt, bool requireAuth, vector* res, const ComboAddress& who, vector>* signatures=nullptr, std::vector>* authorityRecs=nullptr, bool* variable=nullptr, vState* state=nullptr, bool* wasAuth=nullptr); void replace(time_t, const DNSName &qname, const QType& qt, const vector& content, const vector>& signatures, const std::vector>& authorityRecs, bool auth, boost::optional ednsmask=boost::none, vState state=Indeterminate); - void doPrune(unsigned int keep); + void doPrune(size_t keep); uint64_t doDump(int fd); - int doWipeCache(const DNSName& name, bool sub, uint16_t qtype=0xffff); + size_t doWipeCache(const DNSName& name, bool sub, uint16_t qtype=0xffff); bool doAgeCache(time_t now, const DNSName& name, uint16_t qtype, uint32_t newTTL); bool updateValidationStatus(time_t now, const DNSName &qname, const QType& qt, const ComboAddress& who, bool requireAuth, vState newState, boost::optional capTTD); - uint64_t cacheHits, cacheMisses; + std::atomic cacheHits{0}, cacheMisses{0}; private: @@ -94,7 +94,7 @@ private: }; /* The ECS Index (d_ecsIndex) keeps track of whether there is any ECS-specific - entry for a given (qname,qtype) entry in the cache (d_cache), and if so + entry for a given (qname,qtype) entry in the cache (d_map), and if so provides a NetmaskTree of those ECS entries. This allows figuring out quickly if we should look for an entry specific to the requestor IP, and if so which entry is the most @@ -112,14 +112,12 @@ private: Netmask lookupBestMatch(const ComboAddress& addr) const { - Netmask result = Netmask(); - const auto best = d_nmt.lookup(addr); if (best != nullptr) { - result = best->first; + return best->first; } - return result; + return Netmask(); } void addMask(const Netmask& nm) const @@ -190,16 +188,34 @@ private: > > ecsIndex_t; - cache_t d_cache; - ecsIndex_t d_ecsIndex; - std::pair d_cachecache; - DNSName d_cachedqname; - bool d_cachecachevalid; + struct MapCombo + { + MapCombo() + { + } + ~MapCombo() + { + } + MapCombo(const MapCombo &) = delete; + MapCombo & operator=(const MapCombo &) = delete; + cache_t d_map; + ecsIndex_t d_ecsIndex; + DNSName d_cachedqname; + std::pair d_cachecache; + bool d_cachecachevalid{false}; + std::mutex mutex; + }; + + vector d_maps; + MapCombo& getMap(const DNSName &qname) + { + return d_maps[qname.hash() % d_maps.size()]; + } bool entryMatches(OrderedTagIterator_t& entry, uint16_t qt, bool requireAuth, const ComboAddress& who); - std::pair getEntries(const DNSName &qname, const QType& qt); - cache_t::const_iterator getEntryUsingECSIndex(time_t now, const DNSName &qname, uint16_t qtype, bool requireAuth, const ComboAddress& who); - int32_t handleHit(OrderedTagIterator_t& entry, const DNSName& qname, const ComboAddress& who, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth); + std::pair getEntries(MapCombo& map, const DNSName &qname, const QType& qt); + cache_t::const_iterator getEntryUsingECSIndex(MapCombo& map, time_t now, const DNSName &qname, uint16_t qtype, bool requireAuth, const ComboAddress& who); + int32_t handleHit(MapCombo& map, OrderedTagIterator_t& entry, const DNSName& qname, const ComboAddress& who, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth); public: void preRemoval(const CacheEntry& entry) @@ -209,11 +225,12 @@ public: } auto key = tie(entry.d_qname, entry.d_qtype); - auto ecsIndexEntry = d_ecsIndex.find(key); - if (ecsIndexEntry != d_ecsIndex.end()) { + auto& map = getMap(entry.d_qname); + auto ecsIndexEntry = map.d_ecsIndex.find(key); + if (ecsIndexEntry != map.d_ecsIndex.end()) { ecsIndexEntry->removeNetmask(entry.d_netmask); if (ecsIndexEntry->isEmpty()) { - d_ecsIndex.erase(ecsIndexEntry); + map.d_ecsIndex.erase(ecsIndexEntry); } } } diff --git a/pdns/recursordist/test-recursorcache_cc.cc b/pdns/recursordist/test-recursorcache_cc.cc index 1e44e5c11d..1221593359 100644 --- a/pdns/recursordist/test-recursorcache_cc.cc +++ b/pdns/recursordist/test-recursorcache_cc.cc @@ -24,7 +24,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheSimple) MRC.replace(now, DNSName("hello"), QType(QType::A), records, signatures, authRecords, true, boost::none); BOOST_CHECK_EQUAL(MRC.size(), 1U); BOOST_CHECK_GT(MRC.bytes(), 1U); - BOOST_CHECK_EQUAL(MRC.doWipeCache(DNSName("hello"), false, QType::A), 1); + BOOST_CHECK_EQUAL(MRC.doWipeCache(DNSName("hello"), false, QType::A), 1U); BOOST_CHECK_EQUAL(MRC.size(), 0U); BOOST_CHECK_EQUAL(MRC.bytes(), 0U); @@ -45,7 +45,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheSimple) uint64_t delcounter = 0; for (delcounter = 0; delcounter < counter / 100; ++delcounter) { DNSName a = DNSName("hello ") + DNSName(std::to_string(delcounter)); - BOOST_CHECK_EQUAL(MRC.doWipeCache(a, false, QType::A), 1); + BOOST_CHECK_EQUAL(MRC.doWipeCache(a, false, QType::A), 1U); } BOOST_CHECK_EQUAL(MRC.size(), counter - delcounter); @@ -383,7 +383,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheGhost) BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingExpiredEntries) { - MemRecursorCache MRC; + MemRecursorCache MRC(1); std::vector records; std::vector> signatures; @@ -473,7 +473,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingExpiredEntries) BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingValidEntries) { - MemRecursorCache MRC; + MemRecursorCache MRC(1); std::vector records; std::vector> signatures; @@ -651,7 +651,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingValidEntries) BOOST_AUTO_TEST_CASE(test_RecursorCacheECSIndex) { - MemRecursorCache MRC; + MemRecursorCache MRC(1); const DNSName power("powerdns.com."); std::vector records; @@ -892,16 +892,16 @@ BOOST_AUTO_TEST_CASE(test_RecursorCache_Wipe) BOOST_CHECK_EQUAL(MRC.ecsIndexSize(), 5U); /* wipe everything under the powerdns.com domain */ - BOOST_CHECK_EQUAL(MRC.doWipeCache(power, true), 3); + BOOST_CHECK_EQUAL(MRC.doWipeCache(power, true), 3U); BOOST_CHECK_EQUAL(MRC.size(), 2U); BOOST_CHECK_EQUAL(MRC.ecsIndexSize(), 2U); /* now wipe the other domains too */ - BOOST_CHECK_EQUAL(MRC.doWipeCache(other1, true), 1); + BOOST_CHECK_EQUAL(MRC.doWipeCache(other1, true), 1U); BOOST_CHECK_EQUAL(MRC.size(), 1U); BOOST_CHECK_EQUAL(MRC.ecsIndexSize(), 1U); - BOOST_CHECK_EQUAL(MRC.doWipeCache(other2, true), 1); + BOOST_CHECK_EQUAL(MRC.doWipeCache(other2, true), 1U); BOOST_CHECK_EQUAL(MRC.size(), 0U); BOOST_CHECK_EQUAL(MRC.ecsIndexSize(), 0U); } diff --git a/pdns/recursordist/test-syncres_cc.cc b/pdns/recursordist/test-syncres_cc.cc index b354a943f1..dcdb0d48fb 100644 --- a/pdns/recursordist/test-syncres_cc.cc +++ b/pdns/recursordist/test-syncres_cc.cc @@ -10,7 +10,7 @@ RecursorStats g_stats; GlobalStateHolder g_luaconfs; GlobalStateHolder g_dontThrottleNames; GlobalStateHolder g_dontThrottleNetmasks; -thread_local std::unique_ptr t_RC{nullptr}; +std::unique_ptr s_RC{nullptr}; unsigned int g_numThreads = 1; bool g_lowercaseOutgoing = false; @@ -52,8 +52,8 @@ int asyncresolve(const ComboAddress& ip, const DNSName& domain, int type, bool d void primeHints(void) { vector nsset; - if (!t_RC) - t_RC = std::unique_ptr(new MemRecursorCache()); + if (!s_RC) + s_RC = std::unique_ptr(new MemRecursorCache()); DNSRecord arr, aaaarr, nsrr; nsrr.d_name = g_rootdnsname; @@ -72,18 +72,18 @@ void primeHints(void) arr.d_content = std::make_shared(ComboAddress(rootIps4[c - 'a'])); vector aset; aset.push_back(arr); - t_RC->replace(time(nullptr), DNSName(templ), QType(QType::A), aset, vector>(), vector>(), true); // auth, nuke it all + s_RC->replace(time(nullptr), DNSName(templ), QType(QType::A), aset, vector>(), vector>(), true); // auth, nuke it all if (rootIps6[c - 'a'] != NULL) { aaaarr.d_content = std::make_shared(ComboAddress(rootIps6[c - 'a'])); vector aaaaset; aaaaset.push_back(aaaarr); - t_RC->replace(time(nullptr), DNSName(templ), QType(QType::AAAA), aaaaset, vector>(), vector>(), true); + s_RC->replace(time(nullptr), DNSName(templ), QType(QType::AAAA), aaaaset, vector>(), vector>(), true); } nsset.push_back(nsrr); } - t_RC->replace(time(nullptr), g_rootdnsname, QType(QType::NS), nsset, vector>(), vector>(), false); // and stuff in the cache + s_RC->replace(time(nullptr), g_rootdnsname, QType(QType::NS), nsset, vector>(), vector>(), false); // and stuff in the cache } LuaConfigItems::LuaConfigItems() @@ -110,7 +110,7 @@ void initSR(bool debug) g_log.toConsole(Logger::Error); } - t_RC = std::unique_ptr(new MemRecursorCache()); + s_RC = std::unique_ptr(new MemRecursorCache()); SyncRes::s_maxqperq = 50; SyncRes::s_maxtotusec = 1000 * 7000; diff --git a/pdns/recursordist/test-syncres_cc2.cc b/pdns/recursordist/test-syncres_cc2.cc index adf2a476df..5728011378 100644 --- a/pdns/recursordist/test-syncres_cc2.cc +++ b/pdns/recursordist/test-syncres_cc2.cc @@ -934,7 +934,7 @@ BOOST_AUTO_TEST_CASE(test_ecs_cache_limit_allowed) /* should have been cached */ const ComboAddress who("192.0.2.128"); vector cached; - BOOST_REQUIRE_GT(t_RC->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_REQUIRE_GT(s_RC->get(now, target, QType(QType::A), true, &cached, who), 0); BOOST_REQUIRE_EQUAL(cached.size(), 1U); } @@ -973,7 +973,7 @@ BOOST_AUTO_TEST_CASE(test_ecs_cache_limit_no_ttl_limit_allowed) /* should have been cached because /24 is more specific than /16 but TTL limit is nof effective */ const ComboAddress who("192.0.2.128"); vector cached; - BOOST_REQUIRE_GT(t_RC->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_REQUIRE_GT(s_RC->get(now, target, QType(QType::A), true, &cached, who), 0); BOOST_REQUIRE_EQUAL(cached.size(), 1U); } @@ -1012,7 +1012,7 @@ BOOST_AUTO_TEST_CASE(test_ecs_cache_ttllimit_allowed) /* should have been cached */ const ComboAddress who("192.0.2.128"); vector cached; - BOOST_REQUIRE_GT(t_RC->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_REQUIRE_GT(s_RC->get(now, target, QType(QType::A), true, &cached, who), 0); BOOST_REQUIRE_EQUAL(cached.size(), 1U); } @@ -1052,7 +1052,7 @@ BOOST_AUTO_TEST_CASE(test_ecs_cache_ttllimit_and_scope_allowed) /* should have been cached */ const ComboAddress who("192.0.2.128"); vector cached; - BOOST_REQUIRE_GT(t_RC->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_REQUIRE_GT(s_RC->get(now, target, QType(QType::A), true, &cached, who), 0); BOOST_REQUIRE_EQUAL(cached.size(), 1U); } @@ -1092,7 +1092,7 @@ BOOST_AUTO_TEST_CASE(test_ecs_cache_ttllimit_notallowed) /* should have NOT been cached because TTL of 60 is too small and /24 is more specific than /16 */ const ComboAddress who("192.0.2.128"); vector cached; - BOOST_REQUIRE_LT(t_RC->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_REQUIRE_LT(s_RC->get(now, target, QType(QType::A), true, &cached, who), 0); BOOST_REQUIRE_EQUAL(cached.size(), 0U); } @@ -1200,7 +1200,7 @@ BOOST_AUTO_TEST_CASE(test_flawed_nsset) std::vector> sigs; addRecordToList(records, target, QType::NS, "pdns-public-ns1.powerdns.com.", DNSResourceRecord::AUTHORITY, now + 3600); - t_RC->replace(now, target, QType(QType::NS), records, sigs, vector>(), true, boost::optional()); + s_RC->replace(now, target, QType(QType::NS), records, sigs, vector>(), true, boost::optional()); vector ret; int res = sr->beginResolve(target, QType(QType::A), QClass::IN, ret); @@ -1263,7 +1263,7 @@ BOOST_AUTO_TEST_CASE(test_cache_hit) std::vector> sigs; addRecordToList(records, target, QType::A, "192.0.2.1", DNSResourceRecord::ANSWER, now + 3600); - t_RC->replace(now, target, QType(QType::A), records, sigs, vector>(), true, boost::optional()); + s_RC->replace(now, target, QType(QType::A), records, sigs, vector>(), true, boost::optional()); vector ret; int res = sr->beginResolve(target, QType(QType::A), QClass::IN, ret); @@ -1336,13 +1336,13 @@ BOOST_AUTO_TEST_CASE(test_cache_min_max_ttl) const ComboAddress who; vector cached; - BOOST_REQUIRE_GT(t_RC->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_REQUIRE_GT(s_RC->get(now, target, QType(QType::A), true, &cached, who), 0); BOOST_REQUIRE_EQUAL(cached.size(), 1U); BOOST_REQUIRE_GT(cached[0].d_ttl, now); BOOST_CHECK_EQUAL((cached[0].d_ttl - now), SyncRes::s_minimumTTL); cached.clear(); - BOOST_REQUIRE_GT(t_RC->get(now, target, QType(QType::NS), false, &cached, who), 0); + BOOST_REQUIRE_GT(s_RC->get(now, target, QType(QType::NS), false, &cached, who), 0); BOOST_REQUIRE_EQUAL(cached.size(), 1U); BOOST_REQUIRE_GT(cached[0].d_ttl, now); BOOST_CHECK_LE((cached[0].d_ttl - now), SyncRes::s_maxcachettl); @@ -1400,19 +1400,19 @@ BOOST_AUTO_TEST_CASE(test_cache_min_max_ecs_ttl) const ComboAddress who("192.0.2.128"); vector cached; - BOOST_REQUIRE_GT(t_RC->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_REQUIRE_GT(s_RC->get(now, target, QType(QType::A), true, &cached, who), 0); BOOST_REQUIRE_EQUAL(cached.size(), 1U); BOOST_REQUIRE_GT(cached[0].d_ttl, now); BOOST_CHECK_EQUAL((cached[0].d_ttl - now), SyncRes::s_minimumECSTTL); cached.clear(); - BOOST_REQUIRE_GT(t_RC->get(now, target, QType(QType::NS), false, &cached, who), 0); + BOOST_REQUIRE_GT(s_RC->get(now, target, QType(QType::NS), false, &cached, who), 0); BOOST_REQUIRE_EQUAL(cached.size(), 1U); BOOST_REQUIRE_GT(cached[0].d_ttl, now); BOOST_CHECK_LE((cached[0].d_ttl - now), SyncRes::s_maxcachettl); cached.clear(); - BOOST_REQUIRE_GT(t_RC->get(now, DNSName("a.gtld-servers.net."), QType(QType::A), false, &cached, who), 0); + BOOST_REQUIRE_GT(s_RC->get(now, DNSName("a.gtld-servers.net."), QType(QType::A), false, &cached, who), 0); BOOST_REQUIRE_EQUAL(cached.size(), 1U); BOOST_REQUIRE_GT(cached[0].d_ttl, now); BOOST_CHECK_LE((cached[0].d_ttl - now), SyncRes::s_minimumTTL); @@ -1452,7 +1452,7 @@ BOOST_AUTO_TEST_CASE(test_cache_expired_ttl) std::vector> sigs; addRecordToList(records, target, QType::A, "192.0.2.42", DNSResourceRecord::ANSWER, now - 60); - t_RC->replace(now - 3600, target, QType(QType::A), records, sigs, vector>(), true, boost::optional()); + s_RC->replace(now - 3600, target, QType(QType::A), records, sigs, vector>(), true, boost::optional()); vector ret; int res = sr->beginResolve(target, QType(QType::A), QClass::IN, ret); diff --git a/pdns/recursordist/test-syncres_cc3.cc b/pdns/recursordist/test-syncres_cc3.cc index bc6497c2d0..d0847bb4a3 100644 --- a/pdns/recursordist/test-syncres_cc3.cc +++ b/pdns/recursordist/test-syncres_cc3.cc @@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE(test_cache_auth) /* check that we correctly cached only the answer entry, not the additional one */ const ComboAddress who; vector cached; - BOOST_REQUIRE_GT(t_RC->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_REQUIRE_GT(s_RC->get(now, target, QType(QType::A), true, &cached, who), 0); BOOST_REQUIRE_EQUAL(cached.size(), 1U); BOOST_REQUIRE_EQUAL(QType(cached.at(0).d_type).getName(), QType(QType::A).getName()); BOOST_CHECK_EQUAL(getRR(cached.at(0))->getCA().toString(), ComboAddress("192.0.2.2").toString()); @@ -317,7 +317,7 @@ BOOST_AUTO_TEST_CASE(test_answer_no_aa) const ComboAddress who; vector cached; vector> signatures; - BOOST_REQUIRE_EQUAL(t_RC->get(now, target, QType(QType::A), false, &cached, who, &signatures), -1); + BOOST_REQUIRE_EQUAL(s_RC->get(now, target, QType(QType::A), false, &cached, who, &signatures), -1); } BOOST_AUTO_TEST_CASE(test_special_types) diff --git a/pdns/recursordist/test-syncres_cc8.cc b/pdns/recursordist/test-syncres_cc8.cc index a1f0b85e8e..9fb52c8899 100644 --- a/pdns/recursordist/test-syncres_cc8.cc +++ b/pdns/recursordist/test-syncres_cc8.cc @@ -817,7 +817,7 @@ BOOST_AUTO_TEST_CASE(test_dnssec_rrsig_cache_validity) const ComboAddress who; vector cached; vector> signatures; - BOOST_REQUIRE_EQUAL(t_RC->get(tnow, target, QType(QType::A), true, &cached, who, &signatures), 1); + BOOST_REQUIRE_EQUAL(s_RC->get(tnow, target, QType(QType::A), true, &cached, who, &signatures), 1); BOOST_REQUIRE_EQUAL(cached.size(), 1U); BOOST_REQUIRE_EQUAL(signatures.size(), 1U); BOOST_CHECK_EQUAL((cached[0].d_ttl - tnow), 1); diff --git a/pdns/recursordist/test-syncres_cc9.cc b/pdns/recursordist/test-syncres_cc9.cc index cf12fa2736..8364a872c0 100644 --- a/pdns/recursordist/test-syncres_cc9.cc +++ b/pdns/recursordist/test-syncres_cc9.cc @@ -937,7 +937,7 @@ BOOST_AUTO_TEST_CASE(test_cname_plus_authority_ns_ttl) vector cached; bool wasAuth = false; - auto ttl = t_RC->get(now, DNSName("powerdns.com."), QType(QType::NS), false, &cached, who, nullptr, nullptr, nullptr, nullptr, &wasAuth); + auto ttl = s_RC->get(now, DNSName("powerdns.com."), QType(QType::NS), false, &cached, who, nullptr, nullptr, nullptr, nullptr, &wasAuth); BOOST_REQUIRE_GE(ttl, 1); BOOST_REQUIRE_LE(ttl, 42); BOOST_CHECK_EQUAL(cached.size(), 1U); @@ -946,7 +946,7 @@ BOOST_AUTO_TEST_CASE(test_cname_plus_authority_ns_ttl) cached.clear(); /* Also check that the the part in additional is still not auth */ - BOOST_REQUIRE_GE(t_RC->get(now, DNSName("a.gtld-servers.net."), QType(QType::A), false, &cached, who, nullptr, nullptr, nullptr, nullptr, &wasAuth), -1); + BOOST_REQUIRE_GE(s_RC->get(now, DNSName("a.gtld-servers.net."), QType(QType::A), false, &cached, who, nullptr, nullptr, nullptr, nullptr, &wasAuth), -1); BOOST_CHECK_EQUAL(cached.size(), 1U); BOOST_CHECK_EQUAL(wasAuth, false); } @@ -987,14 +987,14 @@ BOOST_AUTO_TEST_CASE(test_records_sanitization_general) const ComboAddress who; vector cached; - BOOST_CHECK_GT(t_RC->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, target, QType(QType::A), true, &cached, who), 0); cached.clear(); - BOOST_CHECK_LT(t_RC->get(now, target, QType(QType::AAAA), true, &cached, who), 0); - BOOST_CHECK_EQUAL(t_RC->get(now, DNSName("not-sanitization.powerdns.com."), QType(QType::DNAME), true, &cached, who), -1); - BOOST_CHECK_LT(t_RC->get(now, target, QType(QType::MX), true, &cached, who), 0); - BOOST_CHECK_EQUAL(t_RC->get(now, DNSName("not-sanitization.powerdns.com."), QType(QType::SOA), true, &cached, who), -1); - BOOST_CHECK_LT(t_RC->get(now, target, QType(QType::TXT), false, &cached, who), 0); - BOOST_CHECK_EQUAL(t_RC->get(now, DNSName("powerdns.com."), QType(QType::AAAA), false, &cached, who), -1); + BOOST_CHECK_LT(s_RC->get(now, target, QType(QType::AAAA), true, &cached, who), 0); + BOOST_CHECK_EQUAL(s_RC->get(now, DNSName("not-sanitization.powerdns.com."), QType(QType::DNAME), true, &cached, who), -1); + BOOST_CHECK_LT(s_RC->get(now, target, QType(QType::MX), true, &cached, who), 0); + BOOST_CHECK_EQUAL(s_RC->get(now, DNSName("not-sanitization.powerdns.com."), QType(QType::SOA), true, &cached, who), -1); + BOOST_CHECK_LT(s_RC->get(now, target, QType(QType::TXT), false, &cached, who), 0); + BOOST_CHECK_EQUAL(s_RC->get(now, DNSName("powerdns.com."), QType(QType::AAAA), false, &cached, who), -1); } BOOST_AUTO_TEST_CASE(test_records_sanitization_keep_relevant_additional_aaaa) @@ -1022,11 +1022,11 @@ BOOST_AUTO_TEST_CASE(test_records_sanitization_keep_relevant_additional_aaaa) const ComboAddress who; vector cached; - BOOST_CHECK_GT(t_RC->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, target, QType(QType::A), true, &cached, who), 0); cached.clear(); /* not auth since it was in the additional section */ - BOOST_CHECK_LT(t_RC->get(now, target, QType(QType::AAAA), true, &cached, who), 0); - BOOST_CHECK_GT(t_RC->get(now, target, QType(QType::AAAA), false, &cached, who), 0); + BOOST_CHECK_LT(s_RC->get(now, target, QType(QType::AAAA), true, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, target, QType(QType::AAAA), false, &cached, who), 0); } BOOST_AUTO_TEST_CASE(test_records_sanitization_keep_glue) @@ -1080,17 +1080,17 @@ BOOST_AUTO_TEST_CASE(test_records_sanitization_keep_glue) const ComboAddress who; vector cached; - BOOST_CHECK_GT(t_RC->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, target, QType(QType::A), true, &cached, who), 0); cached.clear(); - BOOST_CHECK_GT(t_RC->get(now, DNSName("com."), QType(QType::NS), false, &cached, who), 0); - BOOST_CHECK_GT(t_RC->get(now, DNSName("a.gtld-servers.net."), QType(QType::A), false, &cached, who), 0); - BOOST_CHECK_GT(t_RC->get(now, DNSName("a.gtld-servers.net."), QType(QType::AAAA), false, &cached, who), 0); - BOOST_CHECK_GT(t_RC->get(now, DNSName("powerdns.com."), QType(QType::NS), false, &cached, who), 0); - BOOST_CHECK_GT(t_RC->get(now, DNSName("pdns-public-ns1.powerdns.com."), QType(QType::A), false, &cached, who), 0); - BOOST_CHECK_GT(t_RC->get(now, DNSName("pdns-public-ns1.powerdns.com."), QType(QType::AAAA), false, &cached, who), 0); - BOOST_CHECK_GT(t_RC->get(now, DNSName("pdns-public-ns2.powerdns.com."), QType(QType::A), false, &cached, who), 0); - BOOST_CHECK_GT(t_RC->get(now, DNSName("pdns-public-ns2.powerdns.com."), QType(QType::AAAA), false, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, DNSName("com."), QType(QType::NS), false, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, DNSName("a.gtld-servers.net."), QType(QType::A), false, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, DNSName("a.gtld-servers.net."), QType(QType::AAAA), false, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, DNSName("powerdns.com."), QType(QType::NS), false, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, DNSName("pdns-public-ns1.powerdns.com."), QType(QType::A), false, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, DNSName("pdns-public-ns1.powerdns.com."), QType(QType::AAAA), false, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, DNSName("pdns-public-ns2.powerdns.com."), QType(QType::A), false, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, DNSName("pdns-public-ns2.powerdns.com."), QType(QType::AAAA), false, &cached, who), 0); } BOOST_AUTO_TEST_CASE(test_records_sanitization_scrubs_ns_nxd) @@ -1120,12 +1120,12 @@ BOOST_AUTO_TEST_CASE(test_records_sanitization_scrubs_ns_nxd) const ComboAddress who; vector cached; - BOOST_CHECK_GT(t_RC->get(now, DNSName("powerdns.com."), QType(QType::SOA), true, &cached, who), 0); + BOOST_CHECK_GT(s_RC->get(now, DNSName("powerdns.com."), QType(QType::SOA), true, &cached, who), 0); cached.clear(); - BOOST_CHECK_LT(t_RC->get(now, DNSName("powerdns.com."), QType(QType::NS), false, &cached, who), 0); - BOOST_CHECK_LT(t_RC->get(now, DNSName("spoofed.ns."), QType(QType::A), false, &cached, who), 0); - BOOST_CHECK_LT(t_RC->get(now, DNSName("spoofed.ns."), QType(QType::AAAA), false, &cached, who), 0); + BOOST_CHECK_LT(s_RC->get(now, DNSName("powerdns.com."), QType(QType::NS), false, &cached, who), 0); + BOOST_CHECK_LT(s_RC->get(now, DNSName("spoofed.ns."), QType(QType::A), false, &cached, who), 0); + BOOST_CHECK_LT(s_RC->get(now, DNSName("spoofed.ns."), QType(QType::AAAA), false, &cached, who), 0); } BOOST_AUTO_TEST_SUITE_END() diff --git a/pdns/reczones.cc b/pdns/reczones.cc index 6cba83b5e0..fd64f6aee4 100644 --- a/pdns/reczones.cc +++ b/pdns/reczones.cc @@ -48,8 +48,6 @@ void primeHints(void) const vState validationState = Insecure; vector nsset; t_rootNSZones.clear(); - if(!t_RC) - t_RC = std::unique_ptr(new MemRecursorCache()); if(::arg()["hint-file"].empty()) { DNSRecord arr, aaaarr, nsrr; @@ -70,13 +68,13 @@ void primeHints(void) arr.d_content=std::make_shared(ComboAddress(rootIps4[c-'a'])); vector aset; aset.push_back(arr); - t_RC->replace(time(0), DNSName(templ), QType(QType::A), aset, vector>(), vector>(), true, boost::none, validationState); // auth, nuke it all + s_RC->replace(time(0), DNSName(templ), QType(QType::A), aset, vector>(), vector>(), true, boost::none, validationState); // auth, nuke it all if (rootIps6[c-'a'] != NULL) { aaaarr.d_content=std::make_shared(ComboAddress(rootIps6[c-'a'])); vector aaaaset; aaaaset.push_back(aaaarr); - t_RC->replace(time(0), DNSName(templ), QType(QType::AAAA), aaaaset, vector>(), vector>(), true, boost::none, validationState); + s_RC->replace(time(0), DNSName(templ), QType(QType::AAAA), aaaaset, vector>(), vector>(), true, boost::none, validationState); } nsset.push_back(nsrr); @@ -92,11 +90,11 @@ void primeHints(void) if(rr.qtype.getCode()==QType::A) { vector aset; aset.push_back(DNSRecord(rr)); - t_RC->replace(time(0), rr.qname, QType(QType::A), aset, vector>(), vector>(), true, boost::none, validationState); // auth, etc see above + s_RC->replace(time(0), rr.qname, QType(QType::A), aset, vector>(), vector>(), true, boost::none, validationState); // auth, etc see above } else if(rr.qtype.getCode()==QType::AAAA) { vector aaaaset; aaaaset.push_back(DNSRecord(rr)); - t_RC->replace(time(0), rr.qname, QType(QType::AAAA), aaaaset, vector>(), vector>(), true, boost::none, validationState); + s_RC->replace(time(0), rr.qname, QType(QType::AAAA), aaaaset, vector>(), vector>(), true, boost::none, validationState); } else if(rr.qtype.getCode()==QType::NS) { rr.content=toLower(rr.content); nsset.push_back(DNSRecord(rr)); @@ -104,8 +102,8 @@ void primeHints(void) insertIntoRootNSZones(rr.qname.getLastLabel()); } } - t_RC->doWipeCache(g_rootdnsname, false, QType::NS); - t_RC->replace(time(0), g_rootdnsname, QType(QType::NS), nsset, vector>(), vector>(), false, boost::none, validationState); // and stuff in the cache + s_RC->doWipeCache(g_rootdnsname, false, QType::NS); + s_RC->replace(time(0), g_rootdnsname, QType(QType::NS), nsset, vector>(), vector>(), false, boost::none, validationState); // and stuff in the cache } @@ -132,7 +130,7 @@ void primeRootNSZones(bool dnssecmode) // so make a local copy set copy(t_rootNSZones); for (const auto & qname: copy) { - t_RC->doWipeCache(qname, false, QType::NS); + s_RC->doWipeCache(qname, false, QType::NS); vector ret; sr.beginResolve(qname, QType(QType::NS), QClass::IN, ret); } diff --git a/pdns/syncres.cc b/pdns/syncres.cc index 095a2b1bbf..05bd59e965 100644 --- a/pdns/syncres.cc +++ b/pdns/syncres.cc @@ -951,16 +951,16 @@ vector SyncRes::getAddrs(const DNSName &qname, unsigned int depth, } } } - } else { - // We have some IPv4 records, don't bother with going out to get IPv6, but do consult the cache - // Once IPv6 adoption matters, this needs to be revisited - res_t cset; - if (t_RC->get(d_now.tv_sec, qname, QType(QType::AAAA), false, &cset, d_cacheRemote) > 0) { - for (const auto &i : cset) { - if (i.d_ttl > (unsigned int)d_now.tv_sec ) { - if (auto rec = getRR(i)) { - ret.push_back(rec->getCA(53)); - } + } + } else { + // We have some IPv4 records, don't bother with going out to get IPv6, but do consult the cache + // Once IPv6 adoption matters, this needs to be revisited + res_t cset; + if (s_RC->get(d_now.tv_sec, qname, QType(QType::AAAA), false, &cset, d_cacheRemote) > 0) { + for (const auto &i : cset) { + if (i.d_ttl > (unsigned int)d_now.tv_sec ) { + if (auto rec = getRR(i)) { + ret.push_back(rec->getCA(53)); } } } @@ -1031,7 +1031,7 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType& qtype, vecto vector ns; *flawedNSSet = false; - if(t_RC->get(d_now.tv_sec, subdomain, QType(QType::NS), false, &ns, d_cacheRemote) > 0) { + if(s_RC->get(d_now.tv_sec, subdomain, QType(QType::NS), false, &ns, d_cacheRemote) > 0) { bestns.reserve(ns.size()); for(auto k=ns.cbegin();k!=ns.cend(); ++k) { @@ -1040,7 +1040,7 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType& qtype, vecto const DNSRecord& dr=*k; auto nrr = getRR(dr); - if(nrr && (!nrr->getNS().isPartOf(subdomain) || t_RC->get(d_now.tv_sec, nrr->getNS(), s_doIPv6 ? QType(QType::ADDR) : QType(QType::A), + if(nrr && (!nrr->getNS().isPartOf(subdomain) || s_RC->get(d_now.tv_sec, nrr->getNS(), s_doIPv6 ? QType(QType::ADDR) : QType(QType::A), false, doLog() ? &aset : 0, d_cacheRemote) > 5)) { bestns.push_back(dr); LOG(prefix< '"<getNS()<<"'"<updateValidationStatus(d_now.tv_sec, qname, qt, d_cacheRemote, aa, newState, s_maxbogusttl + d_now.tv_sec); + s_RC->updateValidationStatus(d_now.tv_sec, qname, qt, d_cacheRemote, aa, newState, s_maxbogusttl + d_now.tv_sec); } else { - t_RC->updateValidationStatus(d_now.tv_sec, qname, qt, d_cacheRemote, aa, newState, boost::none); + s_RC->updateValidationStatus(d_now.tv_sec, qname, qt, d_cacheRemote, aa, newState, boost::none); } } @@ -1187,7 +1187,7 @@ bool SyncRes::doCNAMECacheCheck(const DNSName &qname, const QType &qtype, vector LOG(prefix<get(d_now.tv_sec, qname, QType(QType::CNAME), !wasForwardRecurse && d_requireAuthData, &cset, d_cacheRemote, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &state, &wasAuth) > 0) { + if (s_RC->get(d_now.tv_sec, qname, QType(QType::CNAME), !wasForwardRecurse && d_requireAuthData, &cset, d_cacheRemote, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &state, &wasAuth) > 0) { foundName = qname; foundQT = QType(QType::CNAME); } @@ -1204,7 +1204,7 @@ bool SyncRes::doCNAMECacheCheck(const DNSName &qname, const QType &qtype, vector if (dnameName == qname && qtype != QType::DNAME) { // The client does not want a DNAME, but we've reached the QNAME already. So there is no match break; } - if (t_RC->get(d_now.tv_sec, dnameName, QType(QType::DNAME), !wasForwardRecurse && d_requireAuthData, &cset, d_cacheRemote, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &state, &wasAuth) > 0) { + if (s_RC->get(d_now.tv_sec, dnameName, QType(QType::DNAME), !wasForwardRecurse && d_requireAuthData, &cset, d_cacheRemote, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &state, &wasAuth) > 0) { foundName = dnameName; foundQT = QType(QType::DNAME); break; @@ -1540,7 +1540,7 @@ bool SyncRes::doCacheCheck(const DNSName &qname, const DNSName& authname, bool w uint32_t ttl=0; uint32_t capTTL = std::numeric_limits::max(); bool wasCachedAuth; - if(t_RC->get(d_now.tv_sec, sqname, sqt, !wasForwardRecurse && d_requireAuthData, &cset, d_cacheRemote, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &cachedState, &wasCachedAuth) > 0) { + if(s_RC->get(d_now.tv_sec, sqname, sqt, !wasForwardRecurse && d_requireAuthData, &cset, d_cacheRemote, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &cachedState, &wasCachedAuth) > 0) { LOG(prefix<replace(d_now.tv_sec, i->first.name, QType(i->first.type), i->second.records, i->second.signatures, authorityRecs, i->first.type == QType::DS ? true : isAA, i->first.place == DNSResourceRecord::ANSWER ? ednsmask : boost::none, recordState); + s_RC->replace(d_now.tv_sec, i->first.name, QType(i->first.type), i->second.records, i->second.signatures, authorityRecs, i->first.type == QType::DS ? true : isAA, i->first.place == DNSResourceRecord::ANSWER ? ednsmask : boost::none, recordState); } } @@ -3457,7 +3457,7 @@ int SyncRes::doResolveAt(NsSet &nameservers, DNSName auth, bool flawedNSSet, con if(!auth.isRoot() && flawedNSSet) { LOG(prefix<doAgeCache(d_now.tv_sec, auth, QType::NS, 10)) + if(s_RC->doAgeCache(d_now.tv_sec, auth, QType::NS, 10)) g_stats.nsSetInvalidations++; } return -1; diff --git a/pdns/syncres.hh b/pdns/syncres.hh index 3b8cc14e73..ae9187a906 100644 --- a/pdns/syncres.hh +++ b/pdns/syncres.hh @@ -965,7 +965,7 @@ struct PacketIDBirthdayCompare: public std::binary_function t_RC; +extern std::unique_ptr s_RC; extern thread_local std::unique_ptr t_packetCache; typedef MTasker MT_t; MT_t* getMT();