From: Otto Moerbeek Date: Mon, 26 Oct 2020 10:36:39 +0000 (+0100) Subject: Introduce an auto cache refresh mechanism. X-Git-Tag: rec-4.5.0-alpha1~4^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fpull%2F9699%2Fhead;p=thirdparty%2Fpdns.git Introduce an auto cache refresh mechanism. If cache records are consulted and seen as "almost-expired" schedule a task to refetch that record. Default off, enable using refresh-on-ttl-perc. --- diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc index b449b96a18..c9fb3f254f 100644 --- a/pdns/pdns_recursor.cc +++ b/pdns/pdns_recursor.cc @@ -99,6 +99,7 @@ #include "query-local-address.hh" #include "rec-snmp.hh" +#include "rec-taskqueue.hh" #ifdef HAVE_SYSTEMD #include @@ -2175,6 +2176,8 @@ static void startDoResolve(void *p) g_log<getMaxStackUsage(), g_stats.maxMThreadStackUsage); } @@ -3356,6 +3359,9 @@ static void doStats(void) auto rc_stats = g_recCache->stats(); double r = rc_stats.second == 0 ? 0.0 : (100.0 * rc_stats.first / rc_stats.second); uint64_t negCacheSize = g_negCache->size(); + auto taskPushes = getTaskPushes(); + auto taskExpired = getTaskExpired(); + auto taskSize = getTaskSize(); if(g_stats.qcounter && (cacheHits + cacheMisses) && SyncRes::s_queries && SyncRes::s_outqueries) { g_log< >& operator+=(vector d_get32bitpointers; @@ -1231,6 +1233,10 @@ void registerAllStats() addGetStat("nod-lookups-dropped-oversize", &g_stats.nodLookupsDroppedOversize); + addGetStat("taskqueue-pushed", []() { return getTaskPushes(); }); + addGetStat("taskqueue-expired", []() { return getTaskExpired(); }); + addGetStat("taskqueue-size", []() { return getTaskSize(); }); + /* make sure that the ECS stats are properly initialized */ SyncRes::clearECSStats(); for (size_t idx = 0; idx < SyncRes::s_ecsResponsesBySubnetSize4.size(); idx++) { @@ -1664,7 +1670,6 @@ static string clearDontThrottleNetmasks(T begin, T end) { return ret + "\n"; } - string RecursorControlParser::getAnswer(const string& question, RecursorControlParser::func_t** command) { *command=nop; diff --git a/pdns/recpacketcache.cc b/pdns/recpacketcache.cc index bbccffd26d..047c18311e 100644 --- a/pdns/recpacketcache.cc +++ b/pdns/recpacketcache.cc @@ -8,12 +8,15 @@ #include "cachecleaner.hh" #include "dns.hh" #include "namespaces.hh" +#include "rec-taskqueue.hh" RecursorPacketCache::RecursorPacketCache() { d_hits = d_misses = 0; } +unsigned int RecursorPacketCache::s_refresh_ttlperc{0}; + int RecursorPacketCache::doWipePacketCache(const DNSName& name, uint16_t qtype, bool subtree) { int count=0; @@ -41,7 +44,7 @@ int RecursorPacketCache::doWipePacketCache(const DNSName& name, uint16_t qtype, bool RecursorPacketCache::qrMatch(const packetCache_t::index::type::iterator& iter, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass) { - // this ignores checking on the EDNS subnet flags! + // this ignores checking on the EDNS subnet flags! if (qname != iter->d_name || iter->d_type != qtype || iter->d_class != qclass) { return false; } @@ -60,10 +63,20 @@ bool RecursorPacketCache::checkResponseMatches(std::paird_ttd) { // it is right, it is fresh! *age = static_cast(now - iter->d_creation); + // we know ttl is > 0 + uint32_t ttl = static_cast(iter->d_ttd - now); + if (s_refresh_ttlperc > 0 && !iter->d_submitted) { + const uint32_t deadline = iter->getOrigTTL() * s_refresh_ttlperc / 100; + const bool almostExpired = ttl <= deadline; + if (almostExpired) { + iter->d_submitted = true; + pushTask(qname, qtype, iter->d_ttd); + } + } *responsePacket = iter->d_packet; responsePacket->replace(0, 2, queryPacket.c_str(), 2); *valState = iter->d_vstate; - + const size_t wirelength = qname.wirelength(); if (responsePacket->size() > (sizeof(dnsheader) + wirelength)) { responsePacket->replace(sizeof(dnsheader), wirelength, queryPacket, sizeof(dnsheader), wirelength); @@ -83,7 +96,7 @@ bool RecursorPacketCache::checkResponseMatches(std::pair(d_packetCache, iter); + moveCacheItemToFront(d_packetCache, iter); d_misses++; break; } @@ -158,14 +171,14 @@ void RecursorPacketCache::insertResponsePacket(unsigned int tag, uint32_t qhash, iter->d_ttd = now + ttl; iter->d_creation = now; iter->d_vstate = valState; - + iter->d_submitted = false; if (pbdata) { iter->d_pbdata = std::move(*pbdata); } break; } - + if(iter == range.second) { // nothing to refresh struct Entry e(qname, std::move(responsePacket), std::move(query)); e.d_qhash = qhash; @@ -175,7 +188,7 @@ void RecursorPacketCache::insertResponsePacket(unsigned int tag, uint32_t qhash, e.d_creation = now; e.d_tag = tag; e.d_vstate = valState; - + e.d_submitted = false; if (pbdata) { e.d_pbdata = std::move(*pbdata); } diff --git a/pdns/recpacketcache.hh b/pdns/recpacketcache.hh index aae60d9549..6cc62244e7 100644 --- a/pdns/recpacketcache.hh +++ b/pdns/recpacketcache.hh @@ -49,6 +49,8 @@ using namespace ::boost::multi_index; class RecursorPacketCache: public PacketCache { public: + static unsigned int s_refresh_ttlperc; + struct PBData { std::string d_message; std::string d_response; @@ -92,12 +94,17 @@ private: uint16_t d_type; uint16_t d_class; mutable vState d_vstate; + mutable bool d_submitted; // whether this entry has been queued for refetch inline bool operator<(const struct Entry& rhs) const; time_t getTTD() const { return d_ttd; } + + uint32_t getOrigTTL() const { + return d_ttd - d_creation; + } }; struct SequencedTag{}; diff --git a/pdns/recursor_cache.cc b/pdns/recursor_cache.cc index 8a798a60a1..bc63003ffe 100644 --- a/pdns/recursor_cache.cc +++ b/pdns/recursor_cache.cc @@ -13,6 +13,7 @@ #include "recursor_cache.hh" #include "namespaces.hh" #include "cachecleaner.hh" +#include "rec-taskqueue.hh" MemRecursorCache::MemRecursorCache(size_t mapsCount) : d_maps(mapsCount) { @@ -111,16 +112,16 @@ static void updateDNSSECValidationStateFromCache(boost::optional& state, } } -int32_t MemRecursorCache::handleHit(MapCombo& map, MemRecursorCache::OrderedTagIterator_t& entry, const DNSName& qname, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, boost::optional& state, bool* wasAuth, DNSName* fromAuthZone) +int32_t MemRecursorCache::handleHit(MapCombo& map, MemRecursorCache::OrderedTagIterator_t& entry, const DNSName& qname, uint32_t& origTTL, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, boost::optional& state, bool* wasAuth, DNSName* fromAuthZone) { // MUTEX SHOULD BE ACQUIRED int32_t ttd = entry->d_ttd; + origTTL = entry->d_orig_ttl; if (variable && (!entry->d_netmask.empty() || entry->d_rtag)) { *variable = true; } - // cerr<<"Looking at "<d_records.size()<<" records for this name"<reserve(res->size() + entry->d_records.size()); @@ -245,16 +246,36 @@ bool MemRecursorCache::entryMatches(MemRecursorCache::OrderedTagIterator_t& entr bool match = (entry->d_qtype == qt || qt == QType::ANY || (qt == QType::ADDR && (entry->d_qtype == QType::A || entry->d_qtype == QType::AAAA))) && (entry->d_netmask.empty() || entry->d_netmask.match(who)); - //cerr << match << "N " << qt << ':' << entry->d_qtype << ' ' << entry->d_netmask.toString() << ':' << who.toString() << endl; return match; } +// Fake a cache miss if more than refreshTTLPerc of the original TTL has passed +int32_t MemRecursorCache::fakeTTD(MemRecursorCache::OrderedTagIterator_t& entry, const DNSName& qname, uint16_t qtype, int32_t ret, time_t now, uint32_t origTTL, bool refresh) +{ + int32_t ttl = static_cast(ret - now); + if (ttl > 0 && SyncRes::s_refresh_ttlperc > 0) { + const uint32_t deadline = origTTL * SyncRes::s_refresh_ttlperc / 100; + const bool almostExpired = static_cast(ttl) <= deadline; + if (almostExpired) { + if (refresh) { + return -1; + } else { + if (!entry->d_submitted) { + pushTask(qname, qtype, entry->d_ttd); + entry->d_submitted = true; + } + } + } + } + return ttl; +} // returns -1 for no hits -int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, bool requireAuth, vector* res, const ComboAddress& who, const OptTag& routingTag, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth, DNSName* fromAuthZone) +int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, bool requireAuth, vector* res, const ComboAddress& who, bool refresh, const OptTag& routingTag, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth, DNSName* fromAuthZone) { boost::optional cachedState{boost::none}; time_t ttd=0; - // cerr<<"looking up "<< qname<<"|"+qt.getName()<<"\n"; + uint32_t origTTL; + if(res) { res->clear(); } @@ -276,11 +297,11 @@ int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, auto entryA = getEntryUsingECSIndex(map, now, qname, QType::A, requireAuth, who); if (entryA != map.d_map.end()) { - ret = handleHit(map, entryA, qname, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone); + ret = handleHit(map, entryA, qname, origTTL, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone); } auto entryAAAA = getEntryUsingECSIndex(map, now, qname, QType::AAAA, requireAuth, who); if (entryAAAA != map.d_map.end()) { - int32_t ttdAAAA = handleHit(map, entryAAAA, qname, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone); + int32_t ttdAAAA = handleHit(map, entryAAAA, qname, origTTL, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone); if (ret > 0) { ret = std::min(ret, ttdAAAA); } else { @@ -297,11 +318,11 @@ int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, else { auto entry = getEntryUsingECSIndex(map, now, qname, qtype, requireAuth, who); if (entry != map.d_map.end()) { - int32_t ret = handleHit(map, entry, qname, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone); + int32_t ret = handleHit(map, entry, qname, origTTL, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone); if (state && cachedState) { *state = *cachedState; } - return static_cast(ret-now); + return fakeTTD(entry, qname, qtype, ret, now, origTTL, refresh); } return -1; } @@ -311,9 +332,10 @@ int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, auto entries = getEntries(map, qname, qt, routingTag); if (entries.first != entries.second) { + OrderedTagIterator_t firstIndexIterator; for (auto i=entries.first; i != entries.second; ++i) { - auto firstIndexIterator = map.d_map.project(i); + firstIndexIterator = map.d_map.project(i); if (i->d_ttd <= now) { moveCacheItemToFront(map.d_map, firstIndexIterator); continue; @@ -323,7 +345,7 @@ int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, continue; } - ttd = handleHit(map, firstIndexIterator, qname, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone); + ttd = handleHit(map, firstIndexIterator, qname, origTTL, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone); if (qt.getCode() != QType::ANY && qt.getCode() != QType::ADDR) { // normally if we have a hit, we are done break; @@ -332,16 +354,17 @@ int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, if (state && cachedState) { *state = *cachedState; } - return static_cast(ttd-now); + return fakeTTD(firstIndexIterator, qname, qtype, ttd, now, origTTL, refresh); } } // Try (again) without tag auto entries = getEntries(map, qname, qt, boost::none); if (entries.first != entries.second) { + OrderedTagIterator_t firstIndexIterator; for (auto i=entries.first; i != entries.second; ++i) { - auto firstIndexIterator = map.d_map.project(i); + firstIndexIterator = map.d_map.project(i); if (i->d_ttd <= now) { moveCacheItemToFront(map.d_map, firstIndexIterator); continue; @@ -351,7 +374,7 @@ int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, continue; } - ttd = handleHit(map, firstIndexIterator, qname, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone); + ttd = handleHit(map, firstIndexIterator, qname, origTTL, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone); if (qt.getCode() != QType::ANY && qt.getCode() != QType::ADDR) { // normally if we have a hit, we are done break; @@ -360,7 +383,7 @@ int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, if (state && cachedState) { *state = *cachedState; } - return static_cast(ttd-now); + return fakeTTD(firstIndexIterator, qname, qtype, ttd, now, origTTL, refresh); } return -1; } @@ -371,7 +394,6 @@ void MemRecursorCache::replace(time_t now, const DNSName &qname, const QType& qt const lock l(map); map.d_cachecachevalid = false; - // cerr<<"Replacing "<toString() : "everyone") << endl; if (ednsmask) { ednsmask = ednsmask->getNormalized(); } @@ -408,13 +430,8 @@ void MemRecursorCache::replace(time_t now, const DNSName &qname, const QType& qt CacheEntry ce=*stored; // this is a COPY ce.d_qtype=qt.getCode(); - // cerr<<"asked to store "<< (qname.empty() ? "EMPTY" : qname.toString()) <<"|"+qt.getName()<<" -> '"; - // cerr<<(content.empty() ? string("EMPTY CONTENT") : content.begin()->d_content->getZoneRepresentation())<<"', auth="<toString() : "none") < now) { // we still have valid data, ignore unauth data - // cerr<<"\tStill hold valid auth data, and the new data is unauth, return\n"; return; } else { @@ -457,14 +474,15 @@ void MemRecursorCache::replace(time_t now, const DNSName &qname, const QType& qt for (const auto& i : content) { /* Yes, we have altered the d_ttl value by adding time(nullptr) to it prior to calling this function, so the TTL actually holds a TTD. */ - ce.d_ttd=min(maxTTD, static_cast(i.d_ttl)); // XXX this does weird things if TTLs differ in the set - //cerr<<"To store: "<getZoneRepresentation()<<" with ttl/ttd "<(i.d_ttl)); // XXX this does weird things if TTLs differ in the set + ce.d_orig_ttl = ce.d_ttd - now; ce.d_records.push_back(i.d_content); } if (!isNew) { moveCacheItemToBack(map.d_map, stored); } + ce.d_submitted = false; map.d_map.replace(stored, ce); } @@ -632,7 +650,7 @@ uint64_t MemRecursorCache::doDump(int fd) for (const auto& j : i.d_records) { count++; try { - fprintf(fp.get(), "%s %" PRId64 " IN %s %s ; (%s) auth=%i zone=%s from=%s %s %s\n", i.d_qname.toString().c_str(), static_cast(i.d_ttd - now), DNSRecordContent::NumberToType(i.d_qtype).c_str(), j->getZoneRepresentation().c_str(), vStateToString(i.d_state).c_str(), i.d_auth, i.d_authZone.toLogString().c_str(), i.d_from.toString().c_str(), i.d_netmask.empty() ? "" : i.d_netmask.toString().c_str(), !i.d_rtag ? "" : i.d_rtag.get().c_str()); + fprintf(fp.get(), "%s %" PRIu32 " %" PRId64 " IN %s %s ; (%s) auth=%i zone=%s from=%s %s %s\n", i.d_qname.toString().c_str(), i.d_orig_ttl, static_cast(i.d_ttd - now), DNSRecordContent::NumberToType(i.d_qtype).c_str(), j->getZoneRepresentation().c_str(), vStateToString(i.d_state).c_str(), i.d_auth, i.d_authZone.toLogString().c_str(), i.d_from.toString().c_str(), i.d_netmask.empty() ? "" : i.d_netmask.toString().c_str(), !i.d_rtag ? "" : i.d_rtag.get().c_str()); } catch(...) { fprintf(fp.get(), "; error printing '%s'\n", i.d_qname.empty() ? "EMPTY" : i.d_qname.toString().c_str()); @@ -641,7 +659,7 @@ uint64_t MemRecursorCache::doDump(int fd) for (const auto &sig : i.d_signatures) { count++; try { - fprintf(fp.get(), "%s %" PRId64 " IN RRSIG %s ; %s\n", i.d_qname.toString().c_str(), static_cast(i.d_ttd - now), sig->getZoneRepresentation().c_str(), i.d_netmask.empty() ? "" : i.d_netmask.toString().c_str()); + fprintf(fp.get(), "%s %" PRIu32 " %" PRId64 " IN RRSIG %s ; %s\n", i.d_qname.toString().c_str(), i.d_orig_ttl, static_cast(i.d_ttd - now), sig->getZoneRepresentation().c_str(), i.d_netmask.empty() ? "" : i.d_netmask.toString().c_str()); } catch(...) { fprintf(fp.get(), "; error printing '%s'\n", i.d_qname.empty() ? "EMPTY" : i.d_qname.toString().c_str()); diff --git a/pdns/recursor_cache.hh b/pdns/recursor_cache.hh index ed81e8cefb..f9ab3601ae 100644 --- a/pdns/recursor_cache.hh +++ b/pdns/recursor_cache.hh @@ -57,7 +57,7 @@ public: typedef boost::optional OptTag; - int32_t get(time_t, const DNSName &qname, const QType& qt, bool requireAuth, vector* res, const ComboAddress& who, const OptTag& routingTag = boost::none, vector>* signatures=nullptr, std::vector>* authorityRecs=nullptr, bool* variable=nullptr, vState* state=nullptr, bool* wasAuth=nullptr, DNSName* fromAuthZone=nullptr); +int32_t get(time_t, const DNSName &qname, const QType& qt, bool requireAuth, vector* res, const ComboAddress& who, bool refresh = false, const OptTag& routingTag = boost::none, vector>* signatures=nullptr, std::vector>* authorityRecs=nullptr, bool* variable=nullptr, vState* state=nullptr, bool* wasAuth=nullptr, DNSName* fromAuthZone=nullptr); void replace(time_t, const DNSName &qname, const QType& qt, const vector& content, const vector>& signatures, const std::vector>& authorityRecs, bool auth, const DNSName& authZone, boost::optional ednsmask=boost::none, const OptTag& routingTag = boost::none, vState state=vState::Indeterminate, boost::optional from=boost::none); @@ -95,8 +95,10 @@ private: OptTag d_rtag; mutable vState d_state; mutable time_t d_ttd; + uint32_t d_orig_ttl; uint16_t d_qtype; bool d_auth; + mutable bool d_submitted; // whether this entry has been queued for refetch }; /* The ECS Index (d_ecsIndex) keeps track of whether there is any ECS-specific @@ -229,10 +231,13 @@ private: return d_maps[qname.hash() % d_maps.size()]; } + static int32_t fakeTTD(OrderedTagIterator_t& entry, const DNSName& qname, uint16_t qtype, int32_t ret, time_t now, uint32_t origTTL, bool refresh); + bool entryMatches(OrderedTagIterator_t& entry, uint16_t qt, bool requireAuth, const ComboAddress& who); Entries getEntries(MapCombo& map, const DNSName &qname, const QType& qt, const OptTag& rtag); cache_t::const_iterator getEntryUsingECSIndex(MapCombo& map, time_t now, const DNSName &qname, uint16_t qtype, bool requireAuth, const ComboAddress& who); - int32_t handleHit(MapCombo& map, OrderedTagIterator_t& entry, const DNSName& qname, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, boost::optional& state, bool* wasAuth, DNSName* authZone); + + int32_t handleHit(MapCombo& map, OrderedTagIterator_t& entry, const DNSName& qname, uint32_t& origTTL, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, boost::optional& state, bool* wasAuth, DNSName* authZone); public: struct lock { diff --git a/pdns/recursordist/Makefile.am b/pdns/recursordist/Makefile.am index 725cf5c3cb..7476cd6287 100644 --- a/pdns/recursordist/Makefile.am +++ b/pdns/recursordist/Makefile.am @@ -152,6 +152,7 @@ pdns_recursor_SOURCES = \ rec-lua-conf.hh rec-lua-conf.cc \ rec-protozero.cc rec-protozero.hh \ rec-snmp.hh rec-snmp.cc \ + rec-taskqueue.cc rec-taskqueue.hh \ rec_channel.cc rec_channel.hh rec_metrics.hh \ rec_channel_rec.cc \ recpacketcache.cc recpacketcache.hh \ @@ -175,6 +176,7 @@ pdns_recursor_SOURCES = \ stable-bloom.hh \ svc-records.cc svc-records.hh \ syncres.cc syncres.hh \ + taskqueue.cc taskqueue.hh \ threadname.hh threadname.cc \ tsigverifier.cc tsigverifier.hh \ ueberbackend.hh \ diff --git a/pdns/recursordist/docs/settings.rst b/pdns/recursordist/docs/settings.rst index f140ed9f03..5c2b7bc4ff 100644 --- a/pdns/recursordist/docs/settings.rst +++ b/pdns/recursordist/docs/settings.rst @@ -1414,6 +1414,20 @@ contention as reported by ``record-cache-contented/record-cache-acquired``, you can try to enlarge this value or run with fewer threads. +.. _setting-refresh-on-ttl-perc: + +``refresh-on-ttl-perc`` +----------------------- +.. versionadded: 4.5.0 + +- Integer +- Default: 0 + +Sets the "refresh almost expired" percentage of the record cache. Whenever a record is fetched from the packet or record cache +and only ``refresh-on-ttl-perc`` percent or less of its original TTL is left, a task is queued to refetch the name/type combination to +update the record cache. In most cases this causes future queries to always see a non-expired record cache entry. +A typical value is 10. If the value is zero, this functionality is disabled. + .. _setting-reuseport: ``reuseport`` diff --git a/pdns/recursordist/rec-taskqueue.cc b/pdns/recursordist/rec-taskqueue.cc new file mode 100644 index 0000000000..6b33aba0c2 --- /dev/null +++ b/pdns/recursordist/rec-taskqueue.cc @@ -0,0 +1,51 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include "rec-taskqueue.hh" +#include "taskqueue.hh" +#include "syncres.hh" + +static thread_local pdns::TaskQueue t_taskQueue; + +void runTaskOnce(bool logErrors) +{ + t_taskQueue.runOnce(logErrors); +} + +void pushTask(const DNSName& qname, uint16_t qtype, time_t deadline) +{ + t_taskQueue.push({qname, qtype, deadline, true}); +} + +uint64_t getTaskPushes() +{ + return broadcastAccFunction([] { return t_taskQueue.getPushes(); }); +} + +uint64_t getTaskExpired() +{ + return broadcastAccFunction([] { return t_taskQueue.getExpired(); }); +} + +uint64_t getTaskSize() +{ + return broadcastAccFunction([] { return t_taskQueue.getSize(); }); +} diff --git a/pdns/recursordist/rec-taskqueue.hh b/pdns/recursordist/rec-taskqueue.hh new file mode 100644 index 0000000000..0bcaf70ab3 --- /dev/null +++ b/pdns/recursordist/rec-taskqueue.hh @@ -0,0 +1,30 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include "dnsname.hh" + +void runTaskOnce(bool logErrors); +void pushTask(const DNSName& qname, uint16_t qtype, time_t deadline); +uint64_t getTaskPushes(); +uint64_t getTaskExpired(); +uint64_t getTaskSize(); diff --git a/pdns/recursordist/taskqueue.cc b/pdns/recursordist/taskqueue.cc new file mode 100644 index 0000000000..847ed88b37 --- /dev/null +++ b/pdns/recursordist/taskqueue.cc @@ -0,0 +1,124 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "taskqueue.hh" + +#include "logger.hh" +#include "syncres.hh" + +namespace pdns +{ + +bool TaskQueue::empty() const +{ + return d_queue.empty(); +} + +size_t TaskQueue::size() const +{ + return d_queue.size(); +} + +void TaskQueue::push(const ResolveTask&& task) +{ + // Insertion fails if it's already there, no problem since we're already scheduled + // and the deadline would remain the same anyway. + auto result = d_queue.insert(std::move(task)); + if (result.second) { + d_pushes++; + } +} + +ResolveTask TaskQueue::pop() +{ + ResolveTask ret = d_queue.get().front(); + d_queue.get().pop_front(); + return ret; +} + +bool TaskQueue::runOnce(bool logErrors) +{ + if (d_queue.empty()) { + return false; + } + ResolveTask task = pop(); + struct timeval now; + gettimeofday(&now, 0); + if (task.d_deadline >= now.tv_sec) { + SyncRes sr(now); + vector ret; + sr.setRefreshAlmostExpired(task.d_refreshMode); + try { + g_log << Logger::Debug << "TaskQueue: resolving " << task.d_qname.toString() << '|' << QType(task.d_qtype).getName() << endl; + sr.beginResolve(task.d_qname, QType(task.d_qtype), QClass::IN, ret); + } + catch (const std::exception& e) { + g_log << Logger::Error << "Exception while running the background task queue: " << e.what() << endl; + } + catch (const PDNSException& e) { + g_log << Logger::Notice << "Exception while running the background task queue: " << e.reason << endl; + } + catch (const ImmediateServFailException& e) { + if (logErrors) { + g_log << Logger::Notice << "Exception while running the background task queue: " << e.reason << endl; + } + } + catch (const PolicyHitException& e) { + if (logErrors) { + g_log << Logger::Notice << "Policy hit while running the background task queue" << endl; + } + } + catch (...) { + g_log << Logger::Error << "Exception while running the background task queue" << endl; + } + } + else { + // Deadline passed + g_log << Logger::Debug << "TaskQueue: deadline for " << task.d_qname.toString() << '|' << QType(task.d_qtype).getName() << " passed" << endl; + d_expired++; + } + return true; +} + +void TaskQueue::runAll(bool logErrors) +{ + while (runOnce(logErrors)) { + /* empty */ + } +} + +uint64_t* TaskQueue::getPushes() const +{ + return new uint64_t(d_pushes); +} + +uint64_t* TaskQueue::getExpired() const +{ + return new uint64_t(d_expired); +} + +uint64_t* TaskQueue::getSize() const +{ + return new uint64_t(size()); +} + +} /* namespace pdns */ diff --git a/pdns/recursordist/taskqueue.hh b/pdns/recursordist/taskqueue.hh new file mode 100644 index 0000000000..0fe61428f9 --- /dev/null +++ b/pdns/recursordist/taskqueue.hh @@ -0,0 +1,87 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "dnsname.hh" +#include "qtype.hh" + +using namespace ::boost::multi_index; + +namespace pdns +{ + +struct ResolveTask +{ + DNSName d_qname; + uint16_t d_qtype; + time_t d_deadline; + bool d_refreshMode; // Whether to run this task in regular mode (false) or in the mode that refreshes almost expired tasks +}; + +struct HashTag +{ +}; +struct SequencedTag +{ +}; + +typedef multi_index_container< + ResolveTask, + indexed_by< + hashed_unique, + composite_key, + member, + member>>, + sequenced>>> + queue_t; + +class TaskQueue +{ +public: + bool empty() const; + size_t size() const; + void push(const ResolveTask&& task); + ResolveTask pop(); + bool runOnce(bool logErrors); // Run one task if the queue is not empty + void runAll(bool logErrors); + uint64_t* getPushes() const; + uint64_t* getExpired() const; + uint64_t* getSize() const; + +private: + queue_t d_queue; + uint64_t d_pushes{0}; + uint64_t d_expired{0}; +}; + +} diff --git a/pdns/recursordist/test-recursorcache_cc.cc b/pdns/recursordist/test-recursorcache_cc.cc index 799240bd63..52602e2181 100644 --- a/pdns/recursordist/test-recursorcache_cc.cc +++ b/pdns/recursordist/test-recursorcache_cc.cc @@ -70,7 +70,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheSimple) int64_t expected = counter - delcounter; for (; delcounter < counter; ++delcounter) { - if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(delcounter)), QType(QType::A), false, &retrieved, who) > 0) { + if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(delcounter)), QType(QType::A), false, &retrieved, who, false) > 0) { matches++; BOOST_REQUIRE_EQUAL(retrieved.size(), records.size()); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr0Content.toString()); @@ -281,7 +281,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheSimple) BOOST_CHECK_EQUAL(MRC.size(), 1U); vState retrievedState = vState::Indeterminate; bool wasAuth = false; - BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("127.0.0.1"), boost::none, nullptr, nullptr, nullptr, &retrievedState, &wasAuth), (ttd - now)); + BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("127.0.0.1"), false, boost::none, nullptr, nullptr, nullptr, &retrievedState, &wasAuth), (ttd - now)); BOOST_CHECK_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(vStateToString(retrievedState), vStateToString(vState::Secure)); BOOST_CHECK_EQUAL(wasAuth, true); @@ -290,7 +290,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheSimple) BOOST_CHECK_EQUAL(MRC.size(), 1U); retrievedState = vState::Indeterminate; wasAuth = false; - BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("127.0.0.1"), boost::none, nullptr, nullptr, nullptr, &retrievedState, &wasAuth), (ttd - now)); + BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("127.0.0.1"), false, boost::none, nullptr, nullptr, nullptr, &retrievedState, &wasAuth), (ttd - now)); BOOST_CHECK_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(vStateToString(retrievedState), vStateToString(vState::Secure)); BOOST_CHECK_EQUAL(wasAuth, true); @@ -457,11 +457,11 @@ BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingExpiredEntries) /* the remaining entry should be power2, but to get it we need to go back in the past a bit */ - BOOST_CHECK_EQUAL(MRC.get(ttd - 1, power2, QType(dr2.d_type), false, &retrieved, who, boost::none, nullptr), 1); + BOOST_CHECK_EQUAL(MRC.get(ttd - 1, power2, QType(dr2.d_type), false, &retrieved, who, 0, boost::none, nullptr), 1); BOOST_REQUIRE_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr2Content.toString()); /* check that power1 is gone */ - BOOST_CHECK_EQUAL(MRC.get(ttd - 1, power1, QType(dr1.d_type), false, &retrieved, who, boost::none, nullptr), -1); + BOOST_CHECK_EQUAL(MRC.get(ttd - 1, power1, QType(dr1.d_type), false, &retrieved, who, 0, boost::none, nullptr), -1); /* clear everything up */ MRC.doWipeCache(DNSName("."), true); @@ -478,7 +478,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingExpiredEntries) BOOST_CHECK_EQUAL(MRC.size(), 2U); /* trigger a miss (expired) for power2 */ - BOOST_CHECK_EQUAL(MRC.get(now, power2, QType(dr2.d_type), false, &retrieved, who, boost::none, nullptr), -now); + BOOST_CHECK_EQUAL(MRC.get(now, power2, QType(dr2.d_type), false, &retrieved, who, 0, boost::none, nullptr), -now); /* power2 should have been moved to the front of the expunge queue, and should this time be removed first */ @@ -488,11 +488,11 @@ BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingExpiredEntries) /* the remaining entry should be power1, but to get it we need to go back in the past a bit */ - BOOST_CHECK_EQUAL(MRC.get(ttd - 1, power1, QType(dr1.d_type), false, &retrieved, who, boost::none, nullptr), 1); + BOOST_CHECK_EQUAL(MRC.get(ttd - 1, power1, QType(dr1.d_type), false, &retrieved, who, 0, boost::none, nullptr), 1); BOOST_REQUIRE_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr1Content.toString()); /* check that power2 is gone */ - BOOST_CHECK_EQUAL(MRC.get(ttd - 1, power2, QType(dr2.d_type), false, &retrieved, who, boost::none, nullptr), -1); + BOOST_CHECK_EQUAL(MRC.get(ttd - 1, power2, QType(dr2.d_type), false, &retrieved, who, 0, boost::none, nullptr), -1); } BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingValidEntries) @@ -547,11 +547,11 @@ BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingValidEntries) BOOST_CHECK_EQUAL(MRC.size(), 1U); /* the remaining entry should be power2 */ - BOOST_CHECK_EQUAL(MRC.get(now, power2, QType(dr2.d_type), false, &retrieved, who, boost::none, nullptr), ttd - now); + BOOST_CHECK_EQUAL(MRC.get(now, power2, QType(dr2.d_type), false, &retrieved, who, 0, boost::none, nullptr), ttd - now); BOOST_REQUIRE_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr2Content.toString()); /* check that power1 is gone */ - BOOST_CHECK_EQUAL(MRC.get(now, power1, QType(dr1.d_type), false, &retrieved, who, boost::none, nullptr), -1); + BOOST_CHECK_EQUAL(MRC.get(now, power1, QType(dr1.d_type), false, &retrieved, who, 0, boost::none, nullptr), -1); /* clear everything up */ MRC.doWipeCache(DNSName("."), true); @@ -581,11 +581,11 @@ BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingValidEntries) BOOST_CHECK_EQUAL(MRC.size(), 1U); /* the remaining entry should be power1 */ - BOOST_CHECK_EQUAL(MRC.get(now, power1, QType(dr1.d_type), false, &retrieved, who, boost::none, nullptr), ttd - now); + BOOST_CHECK_EQUAL(MRC.get(now, power1, QType(dr1.d_type), false, &retrieved, who, 0, boost::none, nullptr), ttd - now); BOOST_REQUIRE_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr1Content.toString()); /* check that power2 is gone */ - BOOST_CHECK_EQUAL(MRC.get(now, power2, QType(dr2.d_type), false, &retrieved, who, boost::none, nullptr), -1); + BOOST_CHECK_EQUAL(MRC.get(now, power2, QType(dr2.d_type), false, &retrieved, who, 0, boost::none, nullptr), -1); /* clear everything up */ MRC.doWipeCache(DNSName("."), true); @@ -602,7 +602,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingValidEntries) BOOST_CHECK_EQUAL(MRC.size(), 2U); /* get a hit for power1 */ - BOOST_CHECK_EQUAL(MRC.get(now, power1, QType(dr1.d_type), false, &retrieved, who, boost::none, nullptr), ttd - now); + BOOST_CHECK_EQUAL(MRC.get(now, power1, QType(dr1.d_type), false, &retrieved, who, 0, boost::none, nullptr), ttd - now); BOOST_REQUIRE_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr1Content.toString()); @@ -613,11 +613,11 @@ BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingValidEntries) BOOST_CHECK_EQUAL(MRC.size(), 1U); /* the remaining entry should be power1 */ - BOOST_CHECK_EQUAL(MRC.get(now, power1, QType(dr1.d_type), false, &retrieved, who, boost::none, nullptr), ttd - now); + BOOST_CHECK_EQUAL(MRC.get(now, power1, QType(dr1.d_type), false, &retrieved, who, 0, boost::none, nullptr), ttd - now); BOOST_REQUIRE_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr1Content.toString()); /* check that power2 is gone */ - BOOST_CHECK_EQUAL(MRC.get(now, power2, QType(dr2.d_type), false, &retrieved, who, boost::none, nullptr), -1); + BOOST_CHECK_EQUAL(MRC.get(now, power2, QType(dr2.d_type), false, &retrieved, who, 0, boost::none, nullptr), -1); MRC.doPrune(0); BOOST_CHECK_EQUAL(MRC.size(), 0U); @@ -653,7 +653,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCache_ExpungingValidEntries) for (size_t i = 0; i <= 255; i++) { ComboAddress whoLoop("192.0.2." + std::to_string(i)); - auto ret = MRC.get(now, power1, QType(QType::A), false, &retrieved, whoLoop); + auto ret = MRC.get(now, power1, QType(QType::A), false, &retrieved, whoLoop, 0); if (ret > 0) { BOOST_REQUIRE_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), whoLoop.toString()); @@ -985,7 +985,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheTagged) int64_t expected = counter; for (counter = 0; counter < 110; counter++) { - if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(counter)), QType(QType::A), false, &retrieved, nobody, boost::none) > 0) { + if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(counter)), QType(QType::A), false, &retrieved, nobody, 0, boost::none) > 0) { matches++; BOOST_CHECK_EQUAL(retrieved.size(), rset0.size()); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr0Content.toString()); @@ -995,7 +995,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheTagged) matches = 0; for (counter = 0; counter < 110; ++counter) { - if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(counter)), QType(QType::A), false, &retrieved, who, string("mytagB")) > 0) { + if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(counter)), QType(QType::A), false, &retrieved, who, 0, string("mytagB")) > 0) { matches++; BOOST_CHECK_EQUAL(retrieved.size(), rset0.size()); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr0Content.toString()); @@ -1005,7 +1005,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheTagged) matches = 0; for (counter = 0; counter < 110; counter++) { - if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(counter)), QType(QType::A), false, &retrieved, who, string("mytagX")) > 0) { + if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(counter)), QType(QType::A), false, &retrieved, who, 0, string("mytagX")) > 0) { matches++; BOOST_CHECK_EQUAL(retrieved.size(), rset0.size()); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr0Content.toString()); @@ -1022,7 +1022,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheTagged) matches = 0; for (counter = 0; counter < 110; counter++) { - if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(counter)), QType(QType::A), false, &retrieved, nobody, boost::none) > 0) { + if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(counter)), QType(QType::A), false, &retrieved, nobody, 0, boost::none) > 0) { matches++; BOOST_CHECK_EQUAL(retrieved.size(), rset0.size()); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr0Content.toString()); @@ -1032,7 +1032,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheTagged) matches = 0; for (counter = 0; counter < 110; ++counter) { - if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(counter)), QType(QType::A), false, &retrieved, nobody, string("mytagA")) > 0) { + if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(counter)), QType(QType::A), false, &retrieved, nobody, 0, string("mytagA")) > 0) { matches++; if (counter < 50) { BOOST_CHECK_EQUAL(retrieved.size(), rset0tagged.size()); @@ -1048,7 +1048,7 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheTagged) matches = 0; for (counter = 0; counter < 110; counter++) { - if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(counter)), QType(QType::A), false, &retrieved, nobody, string("mytagX")) > 0) { + if (MRC.get(now, DNSName("hello ") + DNSName(std::to_string(counter)), QType(QType::A), false, &retrieved, nobody, 0, string("mytagX")) > 0) { matches++; BOOST_CHECK_EQUAL(retrieved.size(), rset0.size()); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr0Content.toString()); @@ -1097,12 +1097,12 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheTagged) BOOST_CHECK_EQUAL(MRC.size(), 1U); // tagged specific should be returned for a matching tag - BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.2.2"), string("mytag")), (ttd - now)); + BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.2.2"), 0, string("mytag")), (ttd - now)); BOOST_REQUIRE_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr1Content.toString()); // tag specific should not be returned for a different tag - BOOST_CHECK_LT(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.2.2"), string("othertag")), 0); + BOOST_CHECK_LT(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.2.2"), 0, string("othertag")), 0); BOOST_CHECK_EQUAL(retrieved.size(), 0U); // insert a new entry without tag @@ -1110,16 +1110,16 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheTagged) BOOST_CHECK_EQUAL(MRC.size(), 2U); // tagged specific should be returned for a matching tag - BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.2.2"), string("mytag")), (ttd - now)); + BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.2.2"), 0, string("mytag")), (ttd - now)); BOOST_REQUIRE_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr1Content.toString()); // if no tag given nothing should be retrieved if address doesn't match - BOOST_CHECK_LT(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("127.0.0.1"), boost::none), 0); + BOOST_CHECK_LT(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("127.0.0.1"), 0, boost::none), 0); BOOST_REQUIRE_EQUAL(retrieved.size(), 0U); // if no tag given and no-non-tagged entries matches nothing should be returned - BOOST_CHECK_LT(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.2.2"), boost::none), 0); + BOOST_CHECK_LT(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.2.2"), 0, boost::none), 0); BOOST_REQUIRE_EQUAL(retrieved.size(), 0U); // Insert untagged entry with no netmask @@ -1127,21 +1127,21 @@ BOOST_AUTO_TEST_CASE(test_RecursorCacheTagged) BOOST_CHECK_EQUAL(MRC.size(), 3U); // Retrieval with no address and no tag should get that one - BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress(), boost::none), (ttd - now)); + BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress(), 0, boost::none), (ttd - now)); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr3Content.toString()); // If no tag given match non-tagged entry - BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.2.2"), boost::none), (ttd - now)); + BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.2.2"), 0, boost::none), (ttd - now)); BOOST_REQUIRE_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr3Content.toString()); // If no tag given we should be able to retrieve the netmask specific record - BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.3.1"), boost::none), (ttd - now)); + BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.3.1"), 0, boost::none), (ttd - now)); BOOST_REQUIRE_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr2Content.toString()); // tagged specific should still be returned for a matching tag, address is not used - BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.2.2"), string("mytag")), (ttd - now)); + BOOST_CHECK_EQUAL(MRC.get(now, power, QType(QType::A), false, &retrieved, ComboAddress("192.0.2.2"), 0, string("mytag")), (ttd - now)); BOOST_REQUIRE_EQUAL(retrieved.size(), 1U); BOOST_CHECK_EQUAL(getRR(retrieved.at(0))->getCA().toString(), dr1Content.toString()); diff --git a/pdns/recursordist/test-syncres_cc.cc b/pdns/recursordist/test-syncres_cc.cc index 457ad652dd..ce1d165b1d 100644 --- a/pdns/recursordist/test-syncres_cc.cc +++ b/pdns/recursordist/test-syncres_cc.cc @@ -4,6 +4,7 @@ #include "base32.hh" #include "lua-recursor4.hh" #include "root-dnssec.hh" +#include "rec-taskqueue.hh" #include "test-syncres_cc.hh" RecursorStats g_stats; @@ -530,3 +531,7 @@ LWResult::Result basicRecordsForQnameMinimization(LWResult* res, const DNSName& } return LWResult::Result::Timeout; } + +void pushTask(const DNSName& qname, uint16_t qtype, time_t deadline) +{ +} diff --git a/pdns/recursordist/test-syncres_cc3.cc b/pdns/recursordist/test-syncres_cc3.cc index 711b6ea535..e950dd05fc 100644 --- a/pdns/recursordist/test-syncres_cc3.cc +++ b/pdns/recursordist/test-syncres_cc3.cc @@ -317,7 +317,7 @@ BOOST_AUTO_TEST_CASE(test_answer_no_aa) const ComboAddress who; vector cached; vector> signatures; - BOOST_REQUIRE_EQUAL(g_recCache->get(now, target, QType(QType::A), false, &cached, who, boost::none, &signatures), -1); + BOOST_REQUIRE_EQUAL(g_recCache->get(now, target, QType(QType::A), false, &cached, who, 0, boost::none, &signatures), -1); } BOOST_AUTO_TEST_CASE(test_special_types) diff --git a/pdns/recursordist/test-syncres_cc7.cc b/pdns/recursordist/test-syncres_cc7.cc index d1341945e0..4641f96b7f 100644 --- a/pdns/recursordist/test-syncres_cc7.cc +++ b/pdns/recursordist/test-syncres_cc7.cc @@ -1583,7 +1583,7 @@ BOOST_AUTO_TEST_CASE(test_dnssec_secure_to_insecure_cut_with_cname_at_apex) BOOST_CHECK_EQUAL(queriesCount, 11U); /* now we remove the denial of powerdns.com DS from the cache and ask www2 */ - BOOST_REQUIRE_EQUAL(g_negCache->wipe(target, false), 1); + BOOST_REQUIRE_EQUAL(g_negCache->wipe(target, false), 1U); ret.clear(); res = sr->beginResolve(DNSName("www2.powerdns.com."), QType(QType::A), QClass::IN, ret); BOOST_CHECK_EQUAL(res, RCode::NoError); @@ -1706,7 +1706,7 @@ BOOST_AUTO_TEST_CASE(test_dnssec_cname_inside_secure_zone) BOOST_CHECK_EQUAL(queriesCount, 10U); /* now we remove the denial of powerdns.com DS from the cache and ask www2 */ - BOOST_REQUIRE_EQUAL(g_negCache->wipe(target, false), 1); + BOOST_REQUIRE_EQUAL(g_negCache->wipe(target, false), 1U); ret.clear(); res = sr->beginResolve(DNSName("www2.powerdns.com."), QType(QType::A), QClass::IN, ret); BOOST_CHECK_EQUAL(res, RCode::NoError); diff --git a/pdns/recursordist/test-syncres_cc8.cc b/pdns/recursordist/test-syncres_cc8.cc index 0c8de63b86..1dd464c5e7 100644 --- a/pdns/recursordist/test-syncres_cc8.cc +++ b/pdns/recursordist/test-syncres_cc8.cc @@ -822,7 +822,7 @@ BOOST_AUTO_TEST_CASE(test_dnssec_rrsig_cache_validity) const ComboAddress who; vector cached; vector> signatures; - BOOST_REQUIRE_EQUAL(g_recCache->get(tnow, target, QType(QType::A), true, &cached, who, boost::none, &signatures), 1); + BOOST_REQUIRE_EQUAL(g_recCache->get(tnow, target, QType(QType::A), true, &cached, who, 0, boost::none, &signatures), 1); BOOST_REQUIRE_EQUAL(cached.size(), 1U); BOOST_REQUIRE_EQUAL(signatures.size(), 1U); BOOST_CHECK_EQUAL((cached[0].d_ttl - tnow), 1); diff --git a/pdns/recursordist/test-syncres_cc9.cc b/pdns/recursordist/test-syncres_cc9.cc index 20e96a273f..29874e7181 100644 --- a/pdns/recursordist/test-syncres_cc9.cc +++ b/pdns/recursordist/test-syncres_cc9.cc @@ -937,7 +937,7 @@ BOOST_AUTO_TEST_CASE(test_cname_plus_authority_ns_ttl) vector cached; bool wasAuth = false; - auto ttl = g_recCache->get(now, DNSName("powerdns.com."), QType(QType::NS), false, &cached, who, boost::none, nullptr, nullptr, nullptr, nullptr, &wasAuth); + auto ttl = g_recCache->get(now, DNSName("powerdns.com."), QType(QType::NS), false, &cached, who, 0, boost::none, nullptr, nullptr, nullptr, nullptr, &wasAuth); BOOST_REQUIRE_GE(ttl, 1); BOOST_REQUIRE_LE(ttl, 42); BOOST_CHECK_EQUAL(cached.size(), 1U); @@ -946,7 +946,7 @@ BOOST_AUTO_TEST_CASE(test_cname_plus_authority_ns_ttl) cached.clear(); /* Also check that the the part in additional is still not auth */ - BOOST_REQUIRE_GE(g_recCache->get(now, DNSName("a.gtld-servers.net."), QType(QType::A), false, &cached, who, boost::none, nullptr, nullptr, nullptr, nullptr, &wasAuth), -1); + BOOST_REQUIRE_GE(g_recCache->get(now, DNSName("a.gtld-servers.net."), QType(QType::A), false, &cached, who, 0, boost::none, nullptr, nullptr, nullptr, nullptr, &wasAuth), -1); BOOST_CHECK_EQUAL(cached.size(), 1U); BOOST_CHECK_EQUAL(wasAuth, false); } @@ -1016,7 +1016,7 @@ BOOST_AUTO_TEST_CASE(test_bogus_does_not_replace_secure_in_the_cache) vector cached; bool wasAuth = false; vState retrievedState = vState::Insecure; - BOOST_CHECK_GT(g_recCache->get(now, DNSName("powerdns.com."), QType(QType::SOA), true, &cached, who, boost::none, nullptr, nullptr, nullptr, &retrievedState, &wasAuth), 0); + BOOST_CHECK_GT(g_recCache->get(now, DNSName("powerdns.com."), QType(QType::SOA), true, &cached, who, false, boost::none, nullptr, nullptr, nullptr, &retrievedState, &wasAuth), 0); BOOST_CHECK_EQUAL(vStateToString(retrievedState), vStateToString(vState::Secure)); BOOST_CHECK_EQUAL(wasAuth, true); @@ -1026,7 +1026,7 @@ BOOST_AUTO_TEST_CASE(test_bogus_does_not_replace_secure_in_the_cache) BOOST_REQUIRE_EQUAL(ret.size(), 2U); cached.clear(); - BOOST_CHECK_GT(g_recCache->get(now, DNSName("powerdns.com."), QType(QType::SOA), true, &cached, who, boost::none, nullptr, nullptr, nullptr, &retrievedState, &wasAuth), 0); + BOOST_CHECK_GT(g_recCache->get(now, DNSName("powerdns.com."), QType(QType::SOA), true, &cached, who, false, boost::none, nullptr, nullptr, nullptr, &retrievedState, &wasAuth), 0); BOOST_CHECK_EQUAL(vStateToString(retrievedState), vStateToString(vState::Secure)); BOOST_CHECK_EQUAL(wasAuth, true); } diff --git a/pdns/syncres.cc b/pdns/syncres.cc index f05a3204f9..16eb0ca0f4 100644 --- a/pdns/syncres.cc +++ b/pdns/syncres.cc @@ -92,6 +92,7 @@ bool SyncRes::s_rootNXTrust; bool SyncRes::s_noEDNS; bool SyncRes::s_qnameminimization; SyncRes::HardenNXD SyncRes::s_hardenNXD; +unsigned int SyncRes::s_refresh_ttlperc; #define LOG(x) if(d_lm == Log) { g_log < SyncRes::getAddrs(const DNSName &qname, unsigned int depth, // We have some IPv4 records, don't bother with going out to get IPv6, but do consult the cache // Once IPv6 adoption matters, this needs to be revisited res_t cset; - if (g_recCache->get(d_now.tv_sec, qname, QType(QType::AAAA), false, &cset, d_cacheRemote, d_routingTag) > 0) { + if (g_recCache->get(d_now.tv_sec, qname, QType(QType::AAAA), false, &cset, d_cacheRemote, d_refresh, d_routingTag) > 0) { for (const auto &i : cset) { if (i.d_ttl > (unsigned int)d_now.tv_sec ) { if (auto rec = getRR(i)) { @@ -1175,7 +1176,7 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType& qtype, vecto vector ns; *flawedNSSet = false; - if(g_recCache->get(d_now.tv_sec, subdomain, QType(QType::NS), false, &ns, d_cacheRemote, d_routingTag) > 0) { + if(g_recCache->get(d_now.tv_sec, subdomain, QType(QType::NS), false, &ns, d_cacheRemote, d_refresh, d_routingTag) > 0) { bestns.reserve(ns.size()); for(auto k=ns.cbegin();k!=ns.cend(); ++k) { @@ -1191,7 +1192,7 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType& qtype, vecto const DNSRecord& dr=*k; auto nrr = getRR(dr); if(nrr && (!nrr->getNS().isPartOf(subdomain) || g_recCache->get(d_now.tv_sec, nrr->getNS(), nsqt, - false, doLog() ? &aset : 0, d_cacheRemote, d_routingTag) > 5)) { + false, doLog() ? &aset : 0, d_cacheRemote, d_refresh, d_routingTag) > 5)) { bestns.push_back(dr); LOG(prefix< '"<getNS()<<"'"<getNS().isPartOf(subdomain)); @@ -1393,7 +1394,7 @@ bool SyncRes::doCNAMECacheCheck(const DNSName &qname, const QType &qtype, vector LOG(prefix<get(d_now.tv_sec, qname, QType(QType::CNAME), !wasForwardRecurse && d_requireAuthData, &cset, d_cacheRemote, d_routingTag, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &state, &wasAuth, &authZone) > 0) { + if (g_recCache->get(d_now.tv_sec, qname, QType(QType::CNAME), !wasForwardRecurse && d_requireAuthData, &cset, d_cacheRemote, d_refresh, d_routingTag, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &state, &wasAuth, &authZone) > 0) { foundName = qname; foundQT = QType(QType::CNAME); } @@ -1410,7 +1411,7 @@ bool SyncRes::doCNAMECacheCheck(const DNSName &qname, const QType &qtype, vector if (dnameName == qname && qtype != QType::DNAME) { // The client does not want a DNAME, but we've reached the QNAME already. So there is no match break; } - if (g_recCache->get(d_now.tv_sec, dnameName, QType(QType::DNAME), !wasForwardRecurse && d_requireAuthData, &cset, d_cacheRemote, d_routingTag, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &state, &wasAuth, &authZone) > 0) { + if (g_recCache->get(d_now.tv_sec, dnameName, QType(QType::DNAME), !wasForwardRecurse && d_requireAuthData, &cset, d_cacheRemote, d_refresh, d_routingTag, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &state, &wasAuth, &authZone) > 0) { foundName = dnameName; foundQT = QType(QType::DNAME); break; @@ -1799,7 +1800,7 @@ bool SyncRes::doCacheCheck(const DNSName &qname, const DNSName& authname, bool w uint32_t capTTL = std::numeric_limits::max(); bool wasCachedAuth; - if(g_recCache->get(d_now.tv_sec, sqname, sqt, !wasForwardRecurse && d_requireAuthData, &cset, d_cacheRemote, d_routingTag, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &cachedState, &wasCachedAuth) > 0) { + if(g_recCache->get(d_now.tv_sec, sqname, sqt, !wasForwardRecurse && d_requireAuthData, &cset, d_cacheRemote, d_refresh, d_routingTag, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &cachedState, &wasCachedAuth) > 0) { LOG(prefix< d_discardedPolicies; DNSFilterEngine::Policy d_appliedPolicy; @@ -892,6 +900,7 @@ private: bool d_qNameMinimization{false}; bool d_queryReceivedOverTCP{false}; bool d_followCNAME{true}; + bool d_refresh{false}; LogMode d_lm; }; diff --git a/regression-tests/recursor-test b/regression-tests/recursor-test index 280d578a02..656190495f 100755 --- a/regression-tests/recursor-test +++ b/regression-tests/recursor-test @@ -31,7 +31,7 @@ rm -f recursor.pid pdns_recursor.pid system CPU seconds%S wallclock seconds%e %% CPU used%P -' ${RECURSOR} --daemon=no --local-port=$port --socket-dir=./ --trace=$TRACE --config-dir=. --max-mthreads=$mthreads --query-local-address="0.0.0.0${QLA6}" --threads=$threads --record-cache-shards=$shards --disable-packetcache > recursor.log 2>&1 & +' ${RECURSOR} --daemon=no --local-port=$port --socket-dir=./ --trace=$TRACE --config-dir=. --max-mthreads=$mthreads --query-local-address="0.0.0.0${QLA6}" --threads=$threads --record-cache-shards=$shards --disable-packetcache --refresh-on-ttl-perc=10 > recursor.log 2>&1 & sleep 3 # warm up the cache