From: Otto Moerbeek Date: Mon, 4 Jul 2022 07:54:09 +0000 (+0200) Subject: Negcache serve-stale X-Git-Tag: rec-4.8.0-alpha1~24^2~5 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=4bab9d741acf362ee5524b561db2016d39113d9b;p=thirdparty%2Fpdns.git Negcache serve-stale This commit also disables some of the ecs changes, as it was causing a unit-test failure. --- diff --git a/pdns/recursor_cache.cc b/pdns/recursor_cache.cc index 9b6238a779..820760a305 100644 --- a/pdns/recursor_cache.cc +++ b/pdns/recursor_cache.cc @@ -218,14 +218,12 @@ MemRecursorCache::cache_t::const_iterator MemRecursorCache::getEntryUsingECSInde else { /* this netmask-specific entry has expired */ moveCacheItemToFront(map.d_map, entry); - // but is is also stale wrt serve-stale? - if (entry->isStale(now)) { - ecsIndex->removeNetmask(best); - if (ecsIndex->isEmpty()) { - map.d_ecsIndex.erase(ecsIndex); - } + // XXX when serving stale, it should be kept, but we don't want a match wth lookupBestMatch()... + ecsIndex->removeNetmask(best); + if (ecsIndex->isEmpty()) { + map.d_ecsIndex.erase(ecsIndex); + break; } - break; } } } @@ -312,7 +310,7 @@ time_t MemRecursorCache::get(time_t now, const DNSName& qname, const QType qt, F bool requireAuth = flags & RequireAuth; bool refresh = flags & Refresh; bool serveStale = flags & ServeStale; - + boost::optional cachedState{boost::none}; uint32_t origTTL; @@ -746,7 +744,6 @@ uint64_t MemRecursorCache::doDump(int fd) void MemRecursorCache::doPrune(size_t keep) { - //size_t maxCached = d_maxEntries; size_t cacheSize = size(); pruneMutexCollectionsVector(*this, d_maps, keep, cacheSize); } diff --git a/pdns/recursor_cache.hh b/pdns/recursor_cache.hh index 285dba7039..0c3d37b888 100644 --- a/pdns/recursor_cache.hh +++ b/pdns/recursor_cache.hh @@ -49,7 +49,7 @@ class MemRecursorCache : public boost::noncopyable // : public RecursorCache public: MemRecursorCache(size_t mapsCount = 1024); - // The number of times a state cache entry is extended + // The number of times a stale cache entry is extended static uint16_t s_maxServedStaleExtensions; // The time a stale cache entry is extended static constexpr uint32_t s_serveStaleExtensionPeriod = 30; @@ -62,10 +62,10 @@ public: typedef boost::optional OptTag; typedef uint8_t Flags; - static const Flags None = 0; - static const Flags RequireAuth = 1 << 0; - static const Flags Refresh = 1 << 1; - static const Flags ServeStale = 1 << 2; + static constexpr Flags None = 0; + static constexpr Flags RequireAuth = 1 << 0; + static constexpr Flags Refresh = 1 << 1; + static constexpr Flags ServeStale = 1 << 2; time_t get(time_t, const DNSName& qname, const QType qt, Flags flags, vector* res, const ComboAddress& who, const OptTag& routingTag = boost::none, vector>* signatures = nullptr, std::vector>* authorityRecs = nullptr, bool* variable = nullptr, vState* state = nullptr, bool* wasAuth = nullptr, DNSName* fromAuthZone = nullptr, ComboAddress* fromAuthIP = nullptr); @@ -93,7 +93,7 @@ private: bool isStale(time_t now) const { // We like to keep things in cache when we (potentially) should serve stale - if (s_maxServedStaleExtensions > 0) { + if (s_maxServedStaleExtensions > 0) { return d_ttd + s_maxServedStaleExtensions * std::min(s_serveStaleExtensionPeriod, d_orig_ttl) < now; } else { diff --git a/pdns/recursordist/negcache.cc b/pdns/recursordist/negcache.cc index b11d4ea265..e86e49a18f 100644 --- a/pdns/recursordist/negcache.cc +++ b/pdns/recursordist/negcache.cc @@ -25,6 +25,9 @@ #include "misc.hh" #include "cachecleaner.hh" #include "utility.hh" +#include "rec-taskqueue.hh" + +uint16_t NegCache::s_maxServedStaleExtensions; NegCache::NegCache(size_t mapsCount) : d_maps(mapsCount) @@ -49,7 +52,7 @@ size_t NegCache::size() const * \param ne A NegCacheEntry that is filled when there is a cache entry * \return true if ne was filled out, false otherwise */ -bool NegCache::getRootNXTrust(const DNSName& qname, const struct timeval& now, NegCacheEntry& ne) +bool NegCache::getRootNXTrust(const DNSName& qname, const struct timeval& now, NegCacheEntry& ne, bool serveStale, bool refresh) { // Never deny the root. if (qname.isRoot()) @@ -65,18 +68,36 @@ bool NegCache::getRootNXTrust(const DNSName& qname, const struct timeval& now, N negcache_t::const_iterator ni = content->d_map.find(std::tie(lastLabel, qtnull)); while (ni != content->d_map.end() && ni->d_name == lastLabel && ni->d_auth.isRoot() && ni->d_qtype == qtnull) { + if (!refresh && (serveStale || ni->d_servedStale > 0) && ni->d_ttd <= now.tv_sec && ni->d_servedStale < s_maxServedStaleExtensions) { + updateStaleEntry(now.tv_sec, ni); + } // We have something if (now.tv_sec < ni->d_ttd) { ne = *ni; moveCacheItemToBack(content->d_map, ni); return true; } - moveCacheItemToFront(content->d_map, ni); + if (ni->d_servedStale == 0 && !serveStale) { + moveCacheItemToFront(content->d_map, ni); + } ++ni; } return false; } +void NegCache::updateStaleEntry(time_t now, negcache_t::iterator& entry) +{ + // We need to take care a infrequently access stale item cannot be extended past + // s_maxServedStaleExtension * s_serveStaleExtensionPeriod + // We we look how old the entry is, and increase d_servedStale accordingly, taking care not to overflow + const time_t howlong = std::max(static_cast(1), now - entry->d_ttd); + const uint32_t extension = std::max(1U, std::min(entry->d_orig_ttl, s_serveStaleExtensionPeriod)); + entry->d_servedStale = std::min(entry->d_servedStale + 1 + howlong / extension, static_cast(s_maxServedStaleExtensions)); + entry->d_ttd = now + std::min(entry->d_orig_ttl, s_serveStaleExtensionPeriod); + + pushAlmostExpiredTask(entry->d_name, entry->d_qtype == 0 ? QType::A : entry->d_qtype, entry->d_ttd, Netmask()); +} + /*! * Set ne to the NegCacheEntry for the qname|qtype tuple and return true * @@ -86,7 +107,7 @@ bool NegCache::getRootNXTrust(const DNSName& qname, const struct timeval& now, N * \param ne A NegCacheEntry that is filled when there is a cache entry * \return true if ne was filled out, false otherwise */ -bool NegCache::get(const DNSName& qname, const QType& qtype, const struct timeval& now, NegCacheEntry& ne, bool typeMustMatch) +bool NegCache::get(const DNSName& qname, const QType& qtype, const struct timeval& now, NegCacheEntry& ne, bool typeMustMatch, bool serveStale, bool refresh) { auto& map = getMap(qname); auto content = map.lock(); @@ -101,6 +122,9 @@ bool NegCache::get(const DNSName& qname, const QType& qtype, const struct timeva // We match the QType or the whole name is denied auto firstIndexIterator = content->d_map.project(ni); + if (!refresh && (serveStale || ni->d_servedStale > 0) && ni->d_ttd <= now.tv_sec && ni->d_servedStale < s_maxServedStaleExtensions) { + updateStaleEntry(now.tv_sec, firstIndexIterator); + } if (now.tv_sec < ni->d_ttd) { // Not expired ne = *ni; @@ -251,7 +275,7 @@ size_t NegCache::dumpToFile(FILE* fp, const struct timeval& now) for (const NegCacheEntry& ne : sidx) { ret++; int64_t ttl = ne.d_ttd - now.tv_sec; - fprintf(fp, "%s %" PRId64 " IN %s VIA %s ; (%s)\n", ne.d_name.toString().c_str(), ttl, ne.d_qtype.toString().c_str(), ne.d_auth.toString().c_str(), vStateToString(ne.d_validationState).c_str()); + fprintf(fp, "%s %" PRId64 " IN %s VIA %s ; (%s) origttl=%" PRIu32 " ss=%hu\n", ne.d_name.toString().c_str(), ttl, ne.d_qtype.toString().c_str(), ne.d_auth.toString().c_str(), vStateToString(ne.d_validationState).c_str(), ne.d_orig_ttl, ne.d_servedStale); for (const auto& rec : ne.authoritySOA.records) { fprintf(fp, "%s %" PRId64 " IN %s %s ; (%s)\n", rec.d_name.toString().c_str(), ttl, DNSRecordContent::NumberToType(rec.d_type).c_str(), rec.d_content->getZoneRepresentation().c_str(), vStateToString(ne.d_validationState).c_str()); } diff --git a/pdns/recursordist/negcache.hh b/pdns/recursordist/negcache.hh index 83667edefe..4cef2a3de7 100644 --- a/pdns/recursordist/negcache.hh +++ b/pdns/recursordist/negcache.hh @@ -56,6 +56,11 @@ class NegCache : public boost::noncopyable public: NegCache(size_t mapsCount = 1024); + // The number of times a stale cache entry is extended + static uint16_t s_maxServedStaleExtensions; + // The time a stale cache entry is extended + static constexpr uint32_t s_serveStaleExtensionPeriod = 30; + struct NegCacheEntry { recordsAndSignatures authoritySOA; // The upstream SOA record and RRSIGs @@ -63,18 +68,27 @@ public: DNSName d_name; // The denied name DNSName d_auth; // The denying name (aka auth) mutable time_t d_ttd; // Timestamp when this entry should die + uint32_t d_orig_ttl; + mutable uint16_t d_servedStale{0}; mutable vState d_validationState{vState::Indeterminate}; QType d_qtype; // The denied type + bool isStale(time_t now) const { - return d_ttd < now; + // We like to keep things in cache when we (potentially) should serve stale + if (s_maxServedStaleExtensions > 0) { + return d_ttd + s_maxServedStaleExtensions * std::min(s_serveStaleExtensionPeriod, d_orig_ttl) < now; + } + else { + return d_ttd < now; + } }; }; void add(const NegCacheEntry& ne); void updateValidationStatus(const DNSName& qname, const QType& qtype, const vState newState, boost::optional capTTD); - bool get(const DNSName& qname, const QType& qtype, const struct timeval& now, NegCacheEntry& ne, bool typeMustMatch = false); - bool getRootNXTrust(const DNSName& qname, const struct timeval& now, NegCacheEntry& ne); + bool get(const DNSName& qname, const QType& qtype, const struct timeval& now, NegCacheEntry& ne, bool typeMustMatch = false, bool serverStale = false, bool refresh = false); + bool getRootNXTrust(const DNSName& qname, const struct timeval& now, NegCacheEntry& ne, bool serveStale, bool refresh); size_t count(const DNSName& qname); size_t count(const DNSName& qname, const QType qtype); void prune(size_t maxEntries); @@ -105,6 +119,8 @@ private: member>>> negcache_t; + void updateStaleEntry(time_t now, negcache_t::iterator& entry); + struct MapCombo { MapCombo() {} diff --git a/pdns/recursordist/rec-main.cc b/pdns/recursordist/rec-main.cc index c5971b9b14..22db5cdb5e 100644 --- a/pdns/recursordist/rec-main.cc +++ b/pdns/recursordist/rec-main.cc @@ -1512,6 +1512,7 @@ static int serviceMain(int argc, char* argv[], Logr::log_t log) SyncRes::s_save_parent_ns_set = ::arg().mustDo("save-parent-ns-set"); SyncRes::s_max_busy_dot_probes = ::arg().asNum("max-busy-dot-probes"); MemRecursorCache::s_maxServedStaleExtensions = ::arg().asNum("serve-stale-extensions"); + NegCache::s_maxServedStaleExtensions = ::arg().asNum("serve-stale-extensions"); if (SyncRes::s_tcp_fast_open_connect) { checkFastOpenSysctl(true, log); diff --git a/pdns/recursordist/recursor_cache.cc b/pdns/recursordist/recursor_cache.cc deleted file mode 120000 index 00b6d25b91..0000000000 --- a/pdns/recursordist/recursor_cache.cc +++ /dev/null @@ -1 +0,0 @@ -../recursor_cache.cc \ No newline at end of file diff --git a/pdns/recursordist/recursor_cache.cc b/pdns/recursordist/recursor_cache.cc new file mode 100644 index 0000000000..d1c3522cc9 --- /dev/null +++ b/pdns/recursordist/recursor_cache.cc @@ -0,0 +1,758 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "recursor_cache.hh" +#include "misc.hh" +#include +#include "dnsrecords.hh" +#include "arguments.hh" +#include "syncres.hh" +#include "recursor_cache.hh" +#include "namespaces.hh" +#include "cachecleaner.hh" +#include "rec-taskqueue.hh" + +uint16_t MemRecursorCache::s_maxServedStaleExtensions; + +MemRecursorCache::MemRecursorCache(size_t mapsCount) : + d_maps(mapsCount) +{ +} + +size_t MemRecursorCache::size() const +{ + size_t count = 0; + for (const auto& map : d_maps) { + count += map.d_entriesCount; + } + return count; +} + +pair MemRecursorCache::stats() +{ + uint64_t c = 0, a = 0; + for (auto& mc : d_maps) { + auto content = mc.lock(); + c += content->d_contended_count; + a += content->d_acquired_count; + } + return pair(c, a); +} + +size_t MemRecursorCache::ecsIndexSize() +{ + // XXX! + size_t count = 0; + for (auto& mc : d_maps) { + auto content = mc.lock(); + count += content->d_ecsIndex.size(); + } + return count; +} + +// this function is too slow to poll! +size_t MemRecursorCache::bytes() +{ + size_t ret = 0; + for (auto& mc : d_maps) { + auto m = mc.lock(); + for (const auto& i : m->d_map) { + ret += sizeof(struct CacheEntry); + ret += i.d_qname.toString().length(); + for (const auto& record : i.d_records) { + ret += sizeof(record); // XXX WRONG we don't know the stored size! + } + } + } + return ret; +} + +static void updateDNSSECValidationStateFromCache(boost::optional& state, const vState stateUpdate) +{ + // if there was no state it's easy */ + if (state == boost::none) { + state = stateUpdate; + return; + } + + if (stateUpdate == vState::TA) { + state = vState::Secure; + } + else if (stateUpdate == vState::NTA) { + state = vState::Insecure; + } + else if (vStateIsBogus(stateUpdate)) { + state = stateUpdate; + } + else if (stateUpdate == vState::Indeterminate) { + state = stateUpdate; + } + else if (stateUpdate == vState::Insecure) { + if (!vStateIsBogus(*state) && *state != vState::Indeterminate) { + state = stateUpdate; + } + } + else if (stateUpdate == vState::Secure) { + if (!vStateIsBogus(*state) && *state != vState::Indeterminate) { + state = stateUpdate; + } + } +} + +time_t MemRecursorCache::handleHit(MapCombo::LockedContent& content, MemRecursorCache::OrderedTagIterator_t& entry, const DNSName& qname, uint32_t& origTTL, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, boost::optional& state, bool* wasAuth, DNSName* fromAuthZone, ComboAddress* fromAuthIP) +{ + // MUTEX SHOULD BE ACQUIRED (as indicated by the reference to the content which is protected by a lock) + time_t ttd = entry->d_ttd; + origTTL = entry->d_orig_ttl; + + if (variable && (!entry->d_netmask.empty() || entry->d_rtag)) { + *variable = true; + } + + if (res) { + res->reserve(res->size() + entry->d_records.size()); + + for (const auto& k : entry->d_records) { + DNSRecord dr; + dr.d_name = qname; + dr.d_type = entry->d_qtype; + dr.d_class = QClass::IN; + dr.d_content = k; + dr.d_ttl = static_cast(entry->d_ttd); // XXX truncation + dr.d_place = DNSResourceRecord::ANSWER; + res->push_back(std::move(dr)); + } + } + + if (signatures) { + signatures->insert(signatures->end(), entry->d_signatures.begin(), entry->d_signatures.end()); + } + + if (authorityRecs) { + authorityRecs->insert(authorityRecs->end(), entry->d_authorityRecs.begin(), entry->d_authorityRecs.end()); + } + + updateDNSSECValidationStateFromCache(state, entry->d_state); + + if (wasAuth) { + *wasAuth = *wasAuth && entry->d_auth; + } + + if (fromAuthZone) { + *fromAuthZone = entry->d_authZone; + } + + if (fromAuthIP) { + *fromAuthIP = entry->d_from; + } + + moveCacheItemToBack(content.d_map, entry); + + return ttd; +} + +static void pushRefreshTask(const DNSName& qname, QType qtype, time_t deadline, const Netmask& netmask) +{ + if (qtype == QType::ADDR) { + pushAlmostExpiredTask(qname, QType::A, deadline, netmask); + pushAlmostExpiredTask(qname, QType::AAAA, deadline, netmask); + } + else { + pushAlmostExpiredTask(qname, qtype, deadline, netmask); + } +} + +void MemRecursorCache::updateStaleEntry(time_t now, MemRecursorCache::OrderedTagIterator_t& entry) +{ + // We need to take care a infrequently access stale item cannot be extended past + // s_maxServedStaleExtension * s_serveStaleExtensionPeriod + // We we look how old the entry is, and increase d_servedStale accordingly, taking care not to overflow + const time_t howlong = std::max(static_cast(1), now - entry->d_ttd); + const uint32_t extension = std::max(1U, std::min(entry->d_orig_ttl, s_serveStaleExtensionPeriod)); + entry->d_servedStale = std::min(entry->d_servedStale + 1 + howlong / extension, static_cast(s_maxServedStaleExtensions)); + entry->d_ttd = now + std::min(entry->d_orig_ttl, s_serveStaleExtensionPeriod); + + pushRefreshTask(entry->d_qname, entry->d_qtype, entry->d_ttd, entry->d_netmask); +} + +MemRecursorCache::cache_t::const_iterator MemRecursorCache::getEntryUsingECSIndex(MapCombo::LockedContent& map, time_t now, const DNSName& qname, const QType qtype, bool requireAuth, const ComboAddress& who, bool serveStale) +{ + // MUTEX SHOULD BE ACQUIRED (as indicated by the reference to the content which is protected by a lock) + auto ecsIndexKey = std::tie(qname, qtype); + auto ecsIndex = map.d_ecsIndex.find(ecsIndexKey); + if (ecsIndex != map.d_ecsIndex.end() && !ecsIndex->isEmpty()) { + /* we have netmask-specific entries, let's see if we match one */ + while (true) { + const Netmask best = ecsIndex->lookupBestMatch(who); + if (best.empty()) { + /* we have nothing more specific for you */ + break; + } + auto key = std::make_tuple(qname, qtype, boost::none, best); + auto entry = map.d_map.find(key); + if (entry == map.d_map.end()) { + /* ecsIndex is not up-to-date */ + ecsIndex->removeNetmask(best); + if (ecsIndex->isEmpty()) { + map.d_ecsIndex.erase(ecsIndex); + break; + } + continue; + } + // If we are serving this record stale (or *should*) and the + // ttd has passed increase ttd to the future and remember that + // we did. Also push a refresh task. + if ((serveStale || entry->d_servedStale > 0) && entry->d_ttd <= now && entry->d_servedStale < s_maxServedStaleExtensions) { + updateStaleEntry(now, entry); + } + if (entry->d_ttd > now) { + if (!requireAuth || entry->d_auth) { + return entry; + } + /* we need auth data and the best match is not authoritative */ + return map.d_map.end(); + } + else { + /* this netmask-specific entry has expired */ + moveCacheItemToFront(map.d_map, entry); + // XXX when serving stale, it should be kept, but we don't want a match wth lookupBestMatch()... + ecsIndex->removeNetmask(best); + if (ecsIndex->isEmpty()) { + map.d_ecsIndex.erase(ecsIndex); + break; + } + } + } + } + + /* we have nothing specific, let's see if we have a generic one */ + auto key = std::make_tuple(qname, qtype, boost::none, Netmask()); + auto entry = map.d_map.find(key); + if (entry != map.d_map.end()) { + // If we are serving this record stale (or *should*) and the + // ttd has passed increase ttd to the future and remember that + // we did. Also push a refresh task. + if ((serveStale || entry->d_servedStale > 0) && entry->d_ttd <= now && entry->d_servedStale < s_maxServedStaleExtensions) { + updateStaleEntry(now, entry); + } + if (entry->d_ttd > now) { + if (!requireAuth || entry->d_auth) { + return entry; + } + } + else { + moveCacheItemToFront(map.d_map, entry); + } + } + + /* nothing for you, sorry */ + return map.d_map.end(); +} + +MemRecursorCache::Entries MemRecursorCache::getEntries(MapCombo::LockedContent& map, const DNSName& qname, const QType qt, const OptTag& rtag) +{ + // MUTEX SHOULD BE ACQUIRED + if (!map.d_cachecachevalid || map.d_cachedqname != qname || map.d_cachedrtag != rtag) { + map.d_cachedqname = qname; + map.d_cachedrtag = rtag; + const auto& idx = map.d_map.get(); + map.d_cachecache = idx.equal_range(std::tie(qname, rtag)); + map.d_cachecachevalid = true; + } + return map.d_cachecache; +} + +bool MemRecursorCache::entryMatches(MemRecursorCache::OrderedTagIterator_t& entry, const QType qt, bool requireAuth, const ComboAddress& who) +{ + // This code assumes that if a routing tag is present, it matches + // MUTEX SHOULD BE ACQUIRED + if (requireAuth && !entry->d_auth) + return false; + + bool match = (entry->d_qtype == qt || qt == QType::ANY || (qt == QType::ADDR && (entry->d_qtype == QType::A || entry->d_qtype == QType::AAAA))) + && (entry->d_netmask.empty() || entry->d_netmask.match(who)); + return match; +} + +// Fake a cache miss if more than refreshTTLPerc of the original TTL has passed +time_t MemRecursorCache::fakeTTD(MemRecursorCache::OrderedTagIterator_t& entry, const DNSName& qname, QType qtype, time_t ret, time_t now, uint32_t origTTL, bool refresh) +{ + time_t ttl = ret - now; + // If we are checking an entry being served stale in refresh mode, + // we always consider it stale so a real refresh attempt will be + // kicked by SyncRes + if (refresh && entry->d_servedStale > 0) { + return -1; + } + if (ttl > 0 && SyncRes::s_refresh_ttlperc > 0) { + const uint32_t deadline = origTTL * SyncRes::s_refresh_ttlperc / 100; + const bool almostExpired = static_cast(ttl) <= deadline; + if (almostExpired && qname != g_rootdnsname) { + if (refresh) { + return -1; + } + else { + if (!entry->d_submitted) { + pushRefreshTask(qname, qtype, entry->d_ttd, entry->d_netmask); + entry->d_submitted = true; + } + } + } + } + return ttl; +} +// returns -1 for no hits +time_t MemRecursorCache::get(time_t now, const DNSName& qname, const QType qt, Flags flags, vector* res, const ComboAddress& who, const OptTag& routingTag, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth, DNSName* fromAuthZone, ComboAddress* fromAuthIP) +{ + bool requireAuth = flags & RequireAuth; + bool refresh = flags & Refresh; + bool serveStale = flags & ServeStale; + + boost::optional cachedState{boost::none}; + uint32_t origTTL; + + if (res) { + res->clear(); + } + const uint16_t qtype = qt.getCode(); + if (wasAuth) { + // we might retrieve more than one entry, we need to set that to true + // so it will be set to false if at least one entry is not auth + *wasAuth = true; + } + + auto& mc = getMap(qname); + auto map = mc.lock(); + + /* If we don't have any netmask-specific entries at all, let's just skip this + to be able to use the nice d_cachecache hack. */ + if (qtype != QType::ANY && !map->d_ecsIndex.empty() && !routingTag) { + if (qtype == QType::ADDR) { + time_t ret = -1; + + auto entryA = getEntryUsingECSIndex(*map, now, qname, QType::A, requireAuth, who, serveStale); + if (entryA != map->d_map.end()) { + ret = handleHit(*map, entryA, qname, origTTL, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone, fromAuthIP); + } + auto entryAAAA = getEntryUsingECSIndex(*map, now, qname, QType::AAAA, requireAuth, who, serveStale); + if (entryAAAA != map->d_map.end()) { + time_t ttdAAAA = handleHit(*map, entryAAAA, qname, origTTL, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone, fromAuthIP); + if (ret > 0) { + ret = std::min(ret, ttdAAAA); + } + else { + ret = ttdAAAA; + } + } + + if (state && cachedState) { + *state = *cachedState; + } + + return ret > 0 ? (ret - now) : ret; + } + else { + auto entry = getEntryUsingECSIndex(*map, now, qname, qtype, requireAuth, who, serveStale); + if (entry != map->d_map.end()) { + time_t ret = handleHit(*map, entry, qname, origTTL, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone, fromAuthIP); + if (state && cachedState) { + *state = *cachedState; + } + return fakeTTD(entry, qname, qtype, ret, now, origTTL, refresh); + } + return -1; + } + } + + if (routingTag) { + auto entries = getEntries(*map, qname, qt, routingTag); + bool found = false; + time_t ttd; + + if (entries.first != entries.second) { + OrderedTagIterator_t firstIndexIterator; + for (auto i = entries.first; i != entries.second; ++i) { + firstIndexIterator = map->d_map.project(i); + + // When serving stale, we consider expired records + if (i->d_ttd <= now && !serveStale && i->d_servedStale == 0) { + moveCacheItemToFront(map->d_map, firstIndexIterator); + continue; + } + + if (!entryMatches(firstIndexIterator, qtype, requireAuth, who)) { + continue; + } + found = true; + // If we are serving this record stale (or *should*) and the + // ttd has passed increase ttd to the future and remember that + // we did. Also push a refresh task. + if ((serveStale || i->d_servedStale > 0) && i->d_ttd <= now && i->d_servedStale < s_maxServedStaleExtensions) { + updateStaleEntry(now, firstIndexIterator); + } + ttd = handleHit(*map, firstIndexIterator, qname, origTTL, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone, fromAuthIP); + + if (qt != QType::ANY && qt != QType::ADDR) { // normally if we have a hit, we are done + break; + } + } + if (found) { + if (state && cachedState) { + *state = *cachedState; + } + return fakeTTD(firstIndexIterator, qname, qtype, ttd, now, origTTL, refresh); + } + else { + return -1; + } + } + } + // Try (again) without tag + auto entries = getEntries(*map, qname, qt, boost::none); + + if (entries.first != entries.second) { + OrderedTagIterator_t firstIndexIterator; + bool found = false; + time_t ttd; + + for (auto i = entries.first; i != entries.second; ++i) { + firstIndexIterator = map->d_map.project(i); + + // When serving stale, we consider expired records + if (i->d_ttd <= now && !serveStale && i->d_servedStale == 0) { + moveCacheItemToFront(map->d_map, firstIndexIterator); + continue; + } + + if (!entryMatches(firstIndexIterator, qtype, requireAuth, who)) { + continue; + } + found = true; + // If we are serving this record stale (or *should*) and the + // ttd has passed increase ttd to the future and remember that + // we did. Also push a refresh task. + if ((serveStale || i->d_servedStale > 0) && i->d_ttd <= now && i->d_servedStale < s_maxServedStaleExtensions) { + updateStaleEntry(now, firstIndexIterator); + } + ttd = handleHit(*map, firstIndexIterator, qname, origTTL, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone, fromAuthIP); + + if (qt != QType::ANY && qt != QType::ADDR) { // normally if we have a hit, we are done + break; + } + } + if (found) { + if (state && cachedState) { + *state = *cachedState; + } + return fakeTTD(firstIndexIterator, qname, qtype, ttd, now, origTTL, refresh); + } + } + return -1; +} + +void MemRecursorCache::replace(time_t now, const DNSName& qname, const QType qt, const vector& content, const vector>& signatures, const std::vector>& authorityRecs, bool auth, const DNSName& authZone, boost::optional ednsmask, const OptTag& routingTag, vState state, boost::optional from) +{ + auto& mc = getMap(qname); + auto map = mc.lock(); + + map->d_cachecachevalid = false; + if (ednsmask) { + ednsmask = ednsmask->getNormalized(); + } + + // We only store with a tag if we have an ednsmask and the tag is available + // We only store an ednsmask if we do not have a tag and we do have a mask. + auto key = std::make_tuple(qname, qt.getCode(), ednsmask ? routingTag : boost::none, (ednsmask && !routingTag) ? *ednsmask : Netmask()); + bool isNew = false; + cache_t::iterator stored = map->d_map.find(key); + if (stored == map->d_map.end()) { + stored = map->d_map.insert(CacheEntry(key, auth)).first; + ++mc.d_entriesCount; + isNew = true; + } + + /* if we are inserting a new entry or updating an expired one (in which case the + ECS index might have been removed but the entry still exists because it has not + been garbage collected yet) we might need to update the ECS index. + Otherwise it should already be indexed and we don't need to update it. + */ + if (isNew || stored->d_ttd <= now) { + /* don't bother building an ecsIndex if we don't have any netmask-specific entries */ + if (!routingTag && ednsmask && !ednsmask->empty()) { + auto ecsIndexKey = std::make_tuple(qname, qt.getCode()); + auto ecsIndex = map->d_ecsIndex.find(ecsIndexKey); + if (ecsIndex == map->d_ecsIndex.end()) { + ecsIndex = map->d_ecsIndex.insert(ECSIndexEntry(qname, qt.getCode())).first; + } + ecsIndex->addMask(*ednsmask); + } + } + + time_t maxTTD = std::numeric_limits::max(); + CacheEntry ce = *stored; // this is a COPY + ce.d_qtype = qt.getCode(); + + if (!auth && ce.d_auth) { // unauth data came in, we have some auth data, but is it fresh? + if (ce.d_ttd > now) { // we still have valid data, ignore unauth data + return; + } + else { + ce.d_auth = false; // new data won't be auth + } + } + + if (auth) { + /* we don't want to keep a non-auth entry while we have an auth one */ + if (vStateIsBogus(state) && (!vStateIsBogus(ce.d_state) && ce.d_state != vState::Indeterminate) && ce.d_ttd > now) { + /* the new entry is Bogus, the existing one is not and is still valid, let's keep the existing one */ + return; + } + } + + ce.d_state = state; + + // refuse any attempt to *raise* the TTL of auth NS records, as it would make it possible + // for an auth to keep a "ghost" zone alive forever, even after the delegation is gone from + // the parent + // BUT make sure that we CAN refresh the root + if (ce.d_auth && auth && qt == QType::NS && !isNew && !qname.isRoot()) { + // cerr<<"\tLimiting TTL of auth->auth NS set replace to "<(i.d_ttl)); // XXX this does weird things if TTLs differ in the set + ce.d_orig_ttl = ce.d_ttd - now; + ce.d_records.push_back(i.d_content); + } + + if (!isNew) { + moveCacheItemToBack(map->d_map, stored); + } + ce.d_submitted = false; + ce.d_servedStale = 0; + map->d_map.replace(stored, ce); +} + +size_t MemRecursorCache::doWipeCache(const DNSName& name, bool sub, const QType qtype) +{ + size_t count = 0; + + if (!sub) { + auto& mc = getMap(name); + auto map = mc.lock(); + map->d_cachecachevalid = false; + auto& idx = map->d_map.get(); + auto range = idx.equal_range(name); + auto i = range.first; + while (i != range.second) { + if (i->d_qtype == qtype || qtype == 0xffff) { + i = idx.erase(i); + count++; + --mc.d_entriesCount; + } + else { + ++i; + } + } + + if (qtype == 0xffff) { + auto& ecsIdx = map->d_ecsIndex.get(); + auto ecsIndexRange = ecsIdx.equal_range(name); + ecsIdx.erase(ecsIndexRange.first, ecsIndexRange.second); + } + else { + auto& ecsIdx = map->d_ecsIndex.get(); + auto ecsIndexRange = ecsIdx.equal_range(std::tie(name, qtype)); + ecsIdx.erase(ecsIndexRange.first, ecsIndexRange.second); + } + } + else { + for (auto& mc : d_maps) { + auto map = mc.lock(); + map->d_cachecachevalid = false; + auto& idx = map->d_map.get(); + for (auto i = idx.lower_bound(name); i != idx.end();) { + if (!i->d_qname.isPartOf(name)) + break; + if (i->d_qtype == qtype || qtype == 0xffff) { + count++; + i = idx.erase(i); + --mc.d_entriesCount; + } + else { + ++i; + } + } + auto& ecsIdx = map->d_ecsIndex.get(); + for (auto i = ecsIdx.lower_bound(name); i != ecsIdx.end();) { + if (!i->d_qname.isPartOf(name)) + break; + if (i->d_qtype == qtype || qtype == 0xffff) { + i = ecsIdx.erase(i); + } + else { + ++i; + } + } + } + } + return count; +} + +// Name should be doLimitTime or so +bool MemRecursorCache::doAgeCache(time_t now, const DNSName& name, const QType qtype, uint32_t newTTL) +{ + auto& mc = getMap(name); + auto map = mc.lock(); + cache_t::iterator iter = map->d_map.find(std::tie(name, qtype)); + if (iter == map->d_map.end()) { + return false; + } + + CacheEntry ce = *iter; + if (ce.d_ttd < now) + return false; // would be dead anyhow + + uint32_t maxTTL = static_cast(ce.d_ttd - now); + if (maxTTL > newTTL) { + map->d_cachecachevalid = false; + + time_t newTTD = now + newTTL; + + if (ce.d_ttd > newTTD) { + ce.d_ttd = newTTD; + map->d_map.replace(iter, ce); + } + return true; + } + return false; +} + +bool MemRecursorCache::updateValidationStatus(time_t now, const DNSName& qname, const QType qt, const ComboAddress& who, const OptTag& routingTag, bool requireAuth, vState newState, boost::optional capTTD) +{ + uint16_t qtype = qt.getCode(); + if (qtype == QType::ANY) { + throw std::runtime_error("Trying to update the DNSSEC validation status of all (via ANY) records for " + qname.toLogString()); + } + if (qtype == QType::ADDR) { + throw std::runtime_error("Trying to update the DNSSEC validation status of several (via ADDR) records for " + qname.toLogString()); + } + + auto& mc = getMap(qname); + auto map = mc.lock(); + + bool updated = false; + if (!map->d_ecsIndex.empty() && !routingTag) { + auto entry = getEntryUsingECSIndex(*map, now, qname, qtype, requireAuth, who, false); // XXX serveStale? + if (entry == map->d_map.end()) { + return false; + } + + entry->d_state = newState; + if (capTTD) { + entry->d_ttd = std::min(entry->d_ttd, *capTTD); + } + return true; + } + + auto entries = getEntries(*map, qname, qt, routingTag); + + for (auto i = entries.first; i != entries.second; ++i) { + auto firstIndexIterator = map->d_map.project(i); + + if (!entryMatches(firstIndexIterator, qtype, requireAuth, who)) { + continue; + } + + i->d_state = newState; + if (capTTD) { + i->d_ttd = std::min(i->d_ttd, *capTTD); + } + updated = true; + + break; + } + + return updated; +} + +uint64_t MemRecursorCache::doDump(int fd) +{ + int newfd = dup(fd); + if (newfd == -1) { + return 0; + } + auto fp = std::unique_ptr(fdopen(newfd, "w"), fclose); + if (!fp) { // dup probably failed + close(newfd); + return 0; + } + + fprintf(fp.get(), "; main record cache dump follows\n;\n"); + uint64_t count = 0; + + for (auto& mc : d_maps) { + auto map = mc.lock(); + const auto& sidx = map->d_map.get(); + + time_t now = time(nullptr); + for (const auto& i : sidx) { + for (const auto& j : i.d_records) { + count++; + try { + fprintf(fp.get(), "%s %" PRIu32 " %" PRId64 " IN %s %s ; (%s) auth=%i zone=%s from=%s nm=%s rtag=%s ss=%hd\n", i.d_qname.toString().c_str(), i.d_orig_ttl, static_cast(i.d_ttd - now), i.d_qtype.toString().c_str(), j->getZoneRepresentation().c_str(), vStateToString(i.d_state).c_str(), i.d_auth, i.d_authZone.toLogString().c_str(), i.d_from.toString().c_str(), i.d_netmask.empty() ? "" : i.d_netmask.toString().c_str(), !i.d_rtag ? "" : i.d_rtag.get().c_str(), i.d_servedStale); + } + catch (...) { + fprintf(fp.get(), "; error printing '%s'\n", i.d_qname.empty() ? "EMPTY" : i.d_qname.toString().c_str()); + } + } + for (const auto& sig : i.d_signatures) { + count++; + try { + fprintf(fp.get(), "%s %" PRIu32 " %" PRId64 " IN RRSIG %s ; %s\n", i.d_qname.toString().c_str(), i.d_orig_ttl, static_cast(i.d_ttd - now), sig->getZoneRepresentation().c_str(), i.d_netmask.empty() ? "" : i.d_netmask.toString().c_str()); + } + catch (...) { + fprintf(fp.get(), "; error printing '%s'\n", i.d_qname.empty() ? "EMPTY" : i.d_qname.toString().c_str()); + } + } + } + } + return count; +} + +void MemRecursorCache::doPrune(size_t keep) +{ + // size_t maxCached = d_maxEntries; + size_t cacheSize = size(); + pruneMutexCollectionsVector(*this, d_maps, keep, cacheSize); +} + +namespace boost +{ +size_t hash_value(const MemRecursorCache::OptTag& o) +{ + return o ? hash_value(o.get()) : 0xcafebaaf; +} +} diff --git a/pdns/recursordist/taskqueue.hh b/pdns/recursordist/taskqueue.hh index 0d43f06dc3..ace34d8d8d 100644 --- a/pdns/recursordist/taskqueue.hh +++ b/pdns/recursordist/taskqueue.hh @@ -120,13 +120,13 @@ private: ResolveTask, indexed_by< ordered_unique, - composite_key, - member, - member, - member, - member, - member>>, + composite_key, + member, + member, + member, + member, + member>>, sequenced>>> queue_t; diff --git a/pdns/recursordist/test-negcache_cc.cc b/pdns/recursordist/test-negcache_cc.cc index 666083da75..ae448a5b89 100644 --- a/pdns/recursordist/test-negcache_cc.cc +++ b/pdns/recursordist/test-negcache_cc.cc @@ -36,6 +36,7 @@ static NegCache::NegCacheEntry genNegCacheEntry(const DNSName& name, const DNSNa ret.d_qtype = QType(qtype); ret.d_auth = auth; ret.d_ttd = now.tv_sec + 600; + ret.d_orig_ttl = 600; ret.authoritySOA = genRecsAndSigs(auth, QType::SOA, "ns1 hostmaster 1 2 3 4 5", true); ret.DNSSECRecords = genRecsAndSigs(auth, QType::NSEC, "deadbeef", true); @@ -155,7 +156,7 @@ BOOST_AUTO_TEST_CASE(test_getRootNXTrust_entry) BOOST_CHECK_EQUAL(cache.size(), 1U); NegCache::NegCacheEntry ne; - bool ret = cache.getRootNXTrust(qname, now, ne); + bool ret = cache.getRootNXTrust(qname, now, ne, false, false); BOOST_CHECK(ret); BOOST_CHECK_EQUAL(ne.d_name, qname); @@ -202,7 +203,7 @@ BOOST_AUTO_TEST_CASE(test_getRootNXTrust_expired_entry) NegCache::NegCacheEntry ne; now.tv_sec += 1000; - bool ret = cache.getRootNXTrust(qname, now, ne); + bool ret = cache.getRootNXTrust(qname, now, ne, false, false); BOOST_CHECK_EQUAL(ret, false); } @@ -246,7 +247,7 @@ BOOST_AUTO_TEST_CASE(test_getRootNXTrust) cache.add(genNegCacheEntry(qname2, auth2, now)); NegCache::NegCacheEntry ne; - bool ret = cache.getRootNXTrust(qname, now, ne); + bool ret = cache.getRootNXTrust(qname, now, ne, false, false); BOOST_CHECK(ret); BOOST_CHECK_EQUAL(ne.d_name, qname2); @@ -268,7 +269,7 @@ BOOST_AUTO_TEST_CASE(test_getRootNXTrust_full_domain_only) cache.add(genNegCacheEntry(qname2, auth2, now, 1)); // Add the denial for COM|A NegCache::NegCacheEntry ne; - bool ret = cache.getRootNXTrust(qname, now, ne); + bool ret = cache.getRootNXTrust(qname, now, ne, false, false); BOOST_CHECK_EQUAL(ret, false); } @@ -440,12 +441,12 @@ BOOST_AUTO_TEST_CASE(test_dumpToFile) { NegCache cache(1); vector expected; - expected.push_back("www1.powerdns.com. 600 IN TYPE0 VIA powerdns.com. ; (Indeterminate)\n"); + expected.push_back("www1.powerdns.com. 600 IN TYPE0 VIA powerdns.com. ; (Indeterminate) origttl=600 ss=0\n"); expected.push_back("powerdns.com. 600 IN SOA ns1. hostmaster. 1 2 3 4 5 ; (Indeterminate)\n"); expected.push_back("powerdns.com. 600 IN RRSIG SOA 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n"); expected.push_back("powerdns.com. 600 IN NSEC deadbeef. ; (Indeterminate)\n"); expected.push_back("powerdns.com. 600 IN RRSIG NSEC 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n"); - expected.push_back("www2.powerdns.com. 600 IN TYPE0 VIA powerdns.com. ; (Indeterminate)\n"); + expected.push_back("www2.powerdns.com. 600 IN TYPE0 VIA powerdns.com. ; (Indeterminate) origttl=600 ss=0\n"); expected.push_back("powerdns.com. 600 IN SOA ns1. hostmaster. 1 2 3 4 5 ; (Indeterminate)\n"); expected.push_back("powerdns.com. 600 IN RRSIG SOA 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n"); expected.push_back("powerdns.com. 600 IN NSEC deadbeef. ; (Indeterminate)\n"); @@ -473,7 +474,7 @@ BOOST_AUTO_TEST_CASE(test_dumpToFile) if (read == -1) BOOST_FAIL("Unable to read a line from the temp file"); // The clock might have ticked so the 600 becomes 599 - BOOST_CHECK(line == str); + BOOST_CHECK_EQUAL(line, str); } /* getline() allocates a buffer when called with a nullptr, diff --git a/pdns/syncres.cc b/pdns/syncres.cc index bebb713bd9..b40dd17c2c 100644 --- a/pdns/syncres.cc +++ b/pdns/syncres.cc @@ -2255,7 +2255,7 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType qtype, vector const DNSRecord& dr=*k; auto nrr = getRR(dr); if(nrr && (!nrr->getNS().isPartOf(subdomain) || g_recCache->get(d_now.tv_sec, nrr->getNS(), nsqt, - MemRecursorCache::None, doLog() ? &aset : 0, d_cacheRemote, d_routingTag) > 0)) { + flags, doLog() ? &aset : 0, d_cacheRemote, d_routingTag) > 0)) { bestns.push_back(dr); LOG(prefix< '"<getNS()<<"'"<getNS().isPartOf(subdomain)); @@ -2776,7 +2776,7 @@ bool SyncRes::doCacheCheck(const DNSName &qname, const DNSName& authname, bool w NegCache::NegCacheEntry ne; if(s_rootNXTrust && - g_negCache->getRootNXTrust(qname, d_now, ne) && + g_negCache->getRootNXTrust(qname, d_now, ne, d_serveStale, d_refresh) && ne.d_auth.isRoot() && !(wasForwardedOrAuthZone && !authname.isRoot())) { // when forwarding, the root may only neg-cache if it was forwarded to. sttl = ne.d_ttd - d_now.tv_sec; @@ -2784,11 +2784,11 @@ bool SyncRes::doCacheCheck(const DNSName &qname, const DNSName& authname, bool w res = RCode::NXDomain; giveNegative = true; cachedState = ne.d_validationState; - } else if (g_negCache->get(qname, qtype, d_now, ne)) { + } else if (g_negCache->get(qname, qtype, d_now, ne, false, d_serveStale, d_refresh)) { /* If we are looking for a DS, discard NXD if auth == qname and ask for a specific denial instead */ if (qtype != QType::DS || ne.d_qtype.getCode() || ne.d_auth != qname || - g_negCache->get(qname, qtype, d_now, ne, true)) + g_negCache->get(qname, qtype, d_now, ne, true, d_serveStale, d_refresh)) { /* Careful! If the client is asking for a DS that does not exist, we need to provide the SOA along with the NSEC(3) proof and we might not have it if we picked up the proof from a delegation, in which case we need to keep on to do the actual DS @@ -2815,7 +2815,7 @@ bool SyncRes::doCacheCheck(const DNSName &qname, const DNSName& authname, bool w negCacheName.prependRawLabel(labels.back()); labels.pop_back(); while(!labels.empty()) { - if (g_negCache->get(negCacheName, QType::ENT, d_now, ne, true)) { + if (g_negCache->get(negCacheName, QType::ENT, d_now, ne, true, d_serveStale, d_refresh)) { if (ne.d_validationState == vState::Indeterminate && validationEnabled()) { // LOG(prefix << negCacheName << " negatively cached and vState::Indeterminate, trying to validate NXDOMAIN" << endl); // ... @@ -4754,6 +4754,7 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co } ne.d_ttd = d_now.tv_sec + lowestTTL; + ne.d_orig_ttl = lowestTTL; /* if we get an NXDomain answer with a CNAME, let's not cache the target, even the server was authoritative for it, and do an additional query for the CNAME target. @@ -4927,6 +4928,7 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co if (denialState == dState::NXQTYPE || denialState == dState::OPTOUT || denialState == dState::INSECURE) { ne.d_ttd = lowestTTL + d_now.tv_sec; + ne.d_orig_ttl = lowestTTL; ne.d_validationState = vState::Secure; if (denialState == dState::OPTOUT) { ne.d_validationState = vState::Insecure; @@ -4984,7 +4986,7 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co rec.d_ttl = min(rec.d_ttl, s_maxbogusttl); } ne.d_ttd = d_now.tv_sec + lowestTTL; - + ne.d_orig_ttl = lowestTTL; if (qtype.getCode()) { // prevents us from NXDOMAIN'ing a whole domain g_negCache->add(ne); }