From: Pieter Lexis Date: Wed, 5 Apr 2017 14:49:29 +0000 (+0200) Subject: rec: Implement the negative cache as a class X-Git-Tag: rec-4.1.0-alpha1~166^2~9 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=39ce10b25237d9a1a30953df76521d9fc0c40ea7;p=thirdparty%2Fpdns.git rec: Implement the negative cache as a class --- diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc index 478bc4a2ab..8db2797292 100644 --- a/pdns/pdns_recursor.cc +++ b/pdns/pdns_recursor.cc @@ -2064,7 +2064,7 @@ static void houseKeeping(void *) t_RC->doPrune(); // this function is local to a thread, so fine anyhow t_packetCache->doPruneTo(::arg().asNum("max-packetcache-entries") / g_numWorkerThreads); - pruneCollection(t_sstorage->negcache, ::arg().asNum("max-cache-entries") / (g_numWorkerThreads * 10), 200); + t_sstorage->negcache.prune(::arg().asNum("max-cache-entries") / (g_numWorkerThreads * 10)); if(!((cleanCounter++)%40)) { // this is a full scan! time_t limit=now.tv_sec-300; diff --git a/pdns/rec_channel_rec.cc b/pdns/rec_channel_rec.cc index fa40d1d03e..42fb89f54a 100644 --- a/pdns/rec_channel_rec.cc +++ b/pdns/rec_channel_rec.cc @@ -11,6 +11,7 @@ #include "misc.hh" #include "recursor_cache.hh" #include "syncres.hh" +#include "negcache.hh" #include #include #include @@ -175,26 +176,17 @@ string doGetParameter(T begin, T end) } -static uint64_t dumpNegCache(SyncRes::negcache_t& negcache, int fd) +static uint64_t dumpNegCache(NegCache& negcache, int fd) { FILE* fp=fdopen(dup(fd), "w"); if(!fp) { // dup probably failed return 0; } + uint64_t ret; fprintf(fp, "; negcache dump from thread follows\n;\n"); - time_t now = time(0); - - typedef SyncRes::negcache_t::nth_index<1>::type sequence_t; - sequence_t& sidx=negcache.get<1>(); - - uint64_t count=0; - for(const NegCacheEntry& neg : sidx) - { - ++count; - fprintf(fp, "%s IN %s %d VIA %s\n", neg.d_name.toString().c_str(), neg.d_qtype.getName().c_str(), (unsigned int) (neg.d_ttd - now), neg.d_qname.toString().c_str()); - } + ret = negcache.dumpToFile(fp); fclose(fp); - return count; + return ret; } static uint64_t* pleaseDump(int fd) @@ -282,24 +274,11 @@ uint64_t* pleaseWipePacketCache(const DNSName& canon, bool subtree) uint64_t* pleaseWipeAndCountNegCache(const DNSName& canon, bool subtree) { - if(!subtree) { - uint64_t res = t_sstorage->negcache.count(tie(canon)); - auto range=t_sstorage->negcache.equal_range(tie(canon)); - t_sstorage->negcache.erase(range.first, range.second); - return new uint64_t(res); - } - else { - unsigned int erased=0; - for(auto iter = t_sstorage->negcache.lower_bound(tie(canon)); iter != t_sstorage->negcache.end(); ) { - if(!iter->d_qname.isPartOf(canon)) - break; - t_sstorage->negcache.erase(iter++); - erased++; - } - return new uint64_t(erased); - } + uint64_t ret = t_sstorage->negcache.wipe(canon, subtree); + return new uint64_t(ret); } + template string doWipeCache(T begin, T end) { diff --git a/pdns/recursordist/Makefile.am b/pdns/recursordist/Makefile.am index 72a0c59899..9344db7612 100644 --- a/pdns/recursordist/Makefile.am +++ b/pdns/recursordist/Makefile.am @@ -110,6 +110,7 @@ pdns_recursor_SOURCES = \ mtasker.hh \ mtasker_context.cc mtasker_context.hh \ namespaces.hh \ + negcache.hh negcache.cc \ nsecrecords.cc \ opensslsigners.cc opensslsigners.hh \ packetcache.hh \ @@ -196,6 +197,7 @@ testrunner_SOURCES = \ iputils.cc iputils.hh \ logger.cc logger.hh \ misc.cc misc.hh \ + negcache.hh negcache.cc \ namespaces.hh \ nsecrecords.cc \ pdnsexception.hh \ diff --git a/pdns/recursordist/negcache.cc b/pdns/recursordist/negcache.cc new file mode 100644 index 0000000000..4588d671e6 --- /dev/null +++ b/pdns/recursordist/negcache.cc @@ -0,0 +1,155 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include "negcache.hh" +#include "misc.hh" +#include "cachecleaner.hh" + +/*! + * Set ne to the NegCacheEntry for the last label in qname and return true + * + * \param qname The name to look up (only the last label is used) + * \param now A timeval with the current time, to check if an entry is expired + * \param ne A NegCacheEntry that is filled when there is a cache entry + * \return true if ne was filled out, false otherwise + */ +bool NegCache::getRootNXTrust(const DNSName& qname, const struct timeval& now, NegCacheEntry& ne) { + // An 'ENT' QType entry, used as "whole name" in the neg-cache context. + static const QType qtnull(0); + pair range; + DNSName lastLabel = qname.getLastLabel(); + range.first = d_negcache.find(tie(lastLabel, qtnull)); + + if (range.first != d_negcache.end() && + range.first->d_auth.isRoot()) { + if ((uint32_t)now.tv_sec < range.first->d_ttd) { + ne = *range.first; + moveCacheItemToBack(d_negcache, range.first); + return true; + } + moveCacheItemToFront(d_negcache, range.first); + } + return false; +} + +/*! + * Set ne to the NegCacheEntry for the qname|qtype tuple and return true + * + * \param qname The name to look up + * \param qtype The qtype to look up + * \param now A timeval with the current time, to check if an entry is expired + * \param ne A NegCacheEntry that is filled when there is a cache entry + * \return true if ne was filled out, false otherwise + */ +bool NegCache::get(const DNSName& qname, const QType& qtype, const struct timeval& now, NegCacheEntry& ne) { + auto range = d_negcache.equal_range(tie(qname)); + negcache_t::iterator ni = range.first; + + while (ni != range.second) { + // We have an entry + if (ni->d_qtype.getCode() == 0 || ni->d_qtype == qtype) { + // We match the QType or the whole name is denied + if((uint32_t) now.tv_sec < ni->d_ttd) { + // Not expired + ne = *ni; + moveCacheItemToBack(d_negcache, ni); + return true; + } + // expired + moveCacheItemToFront(d_negcache, ni); + } + ni++; + } + return false; +} + +/*! + * Places ne into the negative cache, possibly overriding an existing entry. + * + * \param ne The NegCacheEntry to add to the cache + */ +void NegCache::add(const NegCacheEntry& ne) { + replacing_insert(d_negcache, ne); +} + +/*! + * Returns the amount of entries in the cache + */ +uint64_t NegCache::count(const DNSName& qname) const { + return d_negcache.count(tie(qname)); +} + +/*! + * Remove all entries for name from the cache. If subtree is true, wipe all names + * underneath it. + * + * \param name The DNSName of the entries to wipe + * \param subtree Should all entries under name be removed? + */ +uint64_t NegCache::wipe(const DNSName& name, bool subtree) { + uint64_t ret(0); + if (subtree) { + for (auto i = d_negcache.lower_bound(tie(name)); i != d_negcache.end();) { + if(!i->d_name.isPartOf(name)) + break; + i = d_negcache.erase(i); + ret++; + } + return ret; + } + + ret = count(name); + auto range = d_negcache.equal_range(tie(name)); + d_negcache.erase(range.first, range.second); + return ret; +} + +/*! + * Clear the negative cache + */ +void NegCache::clear() { + d_negcache.clear(); +} + +/*! + * Perform some cleanup in the cache, removing stale entries + * + * \param maxEntries The maximum number of entries that may exist in the cache. + */ +void NegCache::prune(unsigned int maxEntries) { + pruneCollection(d_negcache, maxEntries, 200); +} + +/*! + * Writes the whole negative cache to fp + * + * \param fp A pointer to an open FILE object + */ +uint64_t NegCache::dumpToFile(FILE* fp) { + uint64_t ret(0); + time_t now = time(0); + negcache_sequence_t& sidx = d_negcache.get<1>(); + for(const NegCacheEntry& ne : sidx) { + ret++; + fprintf(fp, "%s %d IN %s VIA %s\n", ne.d_name.toString().c_str(), (unsigned int) (ne.d_ttd - now), ne.d_qtype.getName().c_str(), ne.d_auth.toString().c_str()); + } + return ret; +} diff --git a/pdns/recursordist/negcache.hh b/pdns/recursordist/negcache.hh new file mode 100644 index 0000000000..6f0b4cf9fe --- /dev/null +++ b/pdns/recursordist/negcache.hh @@ -0,0 +1,94 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include +#include "dnsparser.hh" +#include "dnsname.hh" +#include "dns.hh" + +using namespace ::boost::multi_index; + +/* FIXME should become part of the normal cache (I think) and shoudl become more like + * struct { + * vector records; + * vector signatures; + * } recsig_t; + * + * typedef vector recordsAndSignatures; + */ +typedef struct { + vector records; + vector signatures; +} recordsAndSignatures; + +class NegCache : public boost::noncopyable { + public: + struct NegCacheEntry { + DNSName d_name; // The denied name + QType d_qtype; // The denied type + DNSName d_auth; // The denying name (aka auth) + uint32_t d_ttd; // Timestamp when this entry should die + recordsAndSignatures authoritySOA; // The upstream SOA record and RRSIGs + recordsAndSignatures DNSSECRecords; // The upstream NSEC(3) and RRSIGs + uint32_t getTTD() const { + return d_ttd; + }; + }; + + void add(const NegCacheEntry& ne); + bool get(const DNSName& qname, const QType& qtype, const struct timeval& now, NegCacheEntry& ne); + bool getRootNXTrust(const DNSName& qname, const struct timeval& now, NegCacheEntry& ne); + uint64_t count(const DNSName& qname) const; + void prune(unsigned int maxEntries); + void clear(); + uint64_t dumpToFile(FILE* fd); + uint64_t wipe(const DNSName& name, bool subtree = false); + + uint64_t size() { + return d_negcache.size(); + }; + + private: + typedef boost::multi_index_container < + NegCacheEntry, + indexed_by < + ordered_unique < + composite_key < + NegCacheEntry, + member, + member + >, + composite_key_compare < + CanonDNSNameCompare, std::less + > + >, + sequenced<> + > + > negcache_t; + + // Required for the cachecleaner + typedef negcache_t::nth_index<1>::type negcache_sequence_t; + + // Stores the negative cache entries + negcache_t d_negcache; +}; diff --git a/pdns/syncres.cc b/pdns/syncres.cc index f092e5cbe2..a22600955d 100644 --- a/pdns/syncres.cc +++ b/pdns/syncres.cc @@ -802,6 +802,14 @@ static const DNSName getLastLabel(const DNSName& qname) return ret; } +static inline void addTTLModifiedRecords(const vector& records, const uint32_t ttl, vector& ret) { + for (const auto& rec : records) { + DNSRecord r(rec); + r.d_ttl = ttl; + ret.push_back(r); + } +} + bool SyncRes::doCacheCheck(const DNSName &qname, const QType &qtype, vector&ret, unsigned int depth, int &res) { @@ -819,65 +827,46 @@ bool SyncRes::doCacheCheck(const DNSName &qname, const QType &qtype, vector "< range; - QType qtnull(0); - DNSName authname(qname); bool wasForwardedOrAuth = (getBestAuthZone(&authname) != t_sstorage->domainmap->end()); + NegCache::NegCacheEntry ne; if(s_rootNXTrust && - (range.first=t_sstorage->negcache.find(tie(getLastLabel(qname), qtnull))) != t_sstorage->negcache.end() && - !(wasForwardedOrAuth && !authname.isRoot()) && // when forwarding, the root may only neg-cache if it was forwarded to. - range.first->d_qname.isRoot() && (uint32_t)d_now.tv_sec < range.first->d_ttd) { - sttl=range.first->d_ttd - d_now.tv_sec; - - LOG(prefix<d_name<<"' & '"<d_qname<<"' for another "<negcache.getRootNXTrust(qname, d_now, ne) && + ne.d_auth.isRoot() && + !(wasForwardedOrAuth && !authname.isRoot())) { // when forwarding, the root may only neg-cache if it was forwarded to. + sttl = ne.d_ttd - d_now.tv_sec; + LOG(prefix<d_qname; - sqt=QType::SOA; - moveCacheItemToBack(t_sstorage->negcache, range.first); - - giveNegative=true; + giveNegative = true; } - else { - range=t_sstorage->negcache.equal_range(tie(qname)); - negcache_t::iterator ni; - for(ni=range.first; ni != range.second; ni++) { - // we have something - if(!(wasForwardedOrAuth && ni->d_qname != authname) && // Only the authname nameserver can neg cache entries - (ni->d_qtype.getCode() == 0 || ni->d_qtype == qtype)) { - res=0; - if((uint32_t)d_now.tv_sec < ni->d_ttd) { - sttl=ni->d_ttd - d_now.tv_sec; - if(ni->d_qtype.getCode()) { - LOG(prefix<d_qname<<"' for another "<d_qname<<"' for another "<d_qname; - sqt=QType::SOA; - if(d_doDNSSEC) { - for(const auto& p : ni->d_dnssecProof) { - for(const auto& rec: p.second.records) - ret.push_back(rec); - for(const auto& rec: p.second.signatures) - ret.push_back(rec); - } - } - moveCacheItemToBack(t_sstorage->negcache, ni); - break; - } - else { - LOG(prefix<negcache, ni); - } - } + else if (t_sstorage->negcache.get(qname, qtype, d_now, ne) && + !(wasForwardedOrAuth && ne.d_auth != authname)) { // Only the authname nameserver can neg cache entries + res = 0; + sttl = ne.d_ttd - d_now.tv_sec; + giveNegative = true; + if(ne.d_qtype.getCode()) { + LOG(prefix< cset; bool found=false, expired=false; vector> signatures; @@ -889,10 +878,6 @@ bool SyncRes::doCacheCheck(const DNSName &qname, const QType &qtype, vectord_ttl>(unsigned int) d_now.tv_sec) { DNSRecord dr=*j; ttl = (dr.d_ttl-=d_now.tv_sec); - if(giveNegative) { - dr.d_place=DNSResourceRecord::AUTHORITY; - dr.d_ttl=sttl; - } ret.push_back(dr); LOG("[ttl="<& records, const set& types) -{ - recsig_t ret; +/* Fills the authoritySOA and DNSSECRecords fields from ne with those found in the records + * + * \param records The records to parse for the authority SOA and NSEC(3) records + * \param ne The NegCacheEntry to be filled out (will not be cleared, only appended to + */ +static void harvestNXRecords(const vector& records, NegCache::NegCacheEntry& ne) { + static const set nsecTypes = {QType::NSEC, QType::NSEC3}; for(const auto& rec : records) { + if(rec.d_place != DNSResourceRecord::AUTHORITY) + // RFC 4035 section 3.1.3. indicates that NSEC records MUST be placed in + // the AUTHORITY section. Section 3.1.1 indicates that that RRSIGs for + // records MUST be in the same section as the records they cover. + // Hence, we ignore all records outside of the AUTHORITY section. + continue; + if(rec.d_type == QType::RRSIG) { - auto rrs=getRR(rec); - if(rrs && types.count(rrs->d_type)) - ret[make_pair(rec.d_name, rrs->d_type)].signatures.push_back(rec); + auto rrsig = getRR(rec); + if(rrsig) { + if(rrsig->d_type == QType::SOA) { + ne.authoritySOA.signatures.push_back(rec); + } + if(nsecTypes.count(rrsig->d_type)) { + ne.DNSSECRecords.signatures.push_back(rec); + } + } + continue; + } + if(rec.d_type == QType::SOA) { + ne.authoritySOA.records.push_back(rec); + continue; + } + if(nsecTypes.count(rec.d_type)) { + ne.DNSSECRecords.records.push_back(rec); + continue; } - else if(types.count(rec.d_type)) - ret[make_pair(rec.d_name, rec.d_type)].records.push_back(rec); } - return ret; } +// TODO remove after processRecords is fixed! +// Adds the RRSIG for the SOA and the NSEC(3) + RRSIGs to ret static void addNXNSECS(vector&ret, const vector& records) { - auto csp = harvestRecords(records, {QType::NSEC, QType::NSEC3, QType::SOA}); - for(const auto& c : csp) { - if(c.first.second == QType::NSEC || c.first.second == QType::NSEC3 || c.first.second == QType::SOA) { - if(c.first.second !=QType::SOA) { - for(const auto& rec : c.second.records) - ret.push_back(rec); - } - for(const auto& rec : c.second.signatures) - ret.push_back(rec); - } - } + NegCache::NegCacheEntry ne; + harvestNXRecords(records, ne); + ret.insert(ret.end(), ne.authoritySOA.signatures.begin(), ne.authoritySOA.signatures.end()); + ret.insert(ret.end(), ne.DNSSECRecords.records.begin(), ne.DNSSECRecords.records.end()); + ret.insert(ret.end(), ne.DNSSECRecords.signatures.begin(), ne.DNSSECRecords.signatures.end()); } bool SyncRes::nameserversBlockedByRPZ(const DNSFilterEngine& dfe, const NsSet& nameservers) @@ -1224,17 +1227,18 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co if(newtarget.empty()) // only add a SOA if we're not going anywhere after this ret.push_back(rec); if(!wasVariable()) { - NegCacheEntry ne; - - ne.d_qname=rec.d_name; - ne.d_ttd=d_now.tv_sec + rec.d_ttl; - ne.d_name=qname; - ne.d_qtype=QType(0); // this encodes 'whole record' - ne.d_dnssecProof = harvestRecords(lwr.d_records, {QType::NSEC, QType::NSEC3}); - replacing_insert(t_sstorage->negcache, ne); + NegCache::NegCacheEntry ne; + + ne.d_name = rec.d_name; + ne.d_ttd = d_now.tv_sec + rec.d_ttl; + ne.d_name = qname; + ne.d_qtype = QType(0); // this encodes 'whole record' + ne.d_auth = auth; + harvestNXRecords(lwr.d_records, ne); + t_sstorage->negcache.add(ne); if(s_rootNXTrust && auth.isRoot()) { // We should check if it was forwarded here, see issue #5107 ne.d_name = getLastLabel(ne.d_name); - replacing_insert(t_sstorage->negcache, ne); + t_sstorage->negcache.add(ne); } } @@ -1290,14 +1294,14 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co rec.d_ttl = min(s_maxnegttl, rec.d_ttl); ret.push_back(rec); if(!wasVariable()) { - NegCacheEntry ne; - ne.d_qname=rec.d_name; - ne.d_ttd=d_now.tv_sec + rec.d_ttl; - ne.d_name=qname; - ne.d_qtype=qtype; - ne.d_dnssecProof = harvestRecords(lwr.d_records, {QType::NSEC, QType::NSEC3}); + NegCache::NegCacheEntry ne; + ne.d_auth = rec.d_name; + ne.d_ttd = d_now.tv_sec + rec.d_ttl; + ne.d_name = qname; + ne.d_qtype = qtype; + harvestNXRecords(lwr.d_records, ne); if(qtype.getCode()) { // prevents us from blacking out a whole domain - replacing_insert(t_sstorage->negcache, ne); + t_sstorage->negcache.add(ne); } } negindic=true; diff --git a/pdns/syncres.hh b/pdns/syncres.hh index f557094890..e0d110178d 100644 --- a/pdns/syncres.hh +++ b/pdns/syncres.hh @@ -48,6 +48,7 @@ #include "validate.hh" #include "ednssubnet.hh" #include "filterpo.hh" +#include "negcache.hh" #ifdef HAVE_CONFIG_H #include "config.h" @@ -62,26 +63,6 @@ void primeHints(void); class RecursorLua4; -struct BothRecordsAndSignatures -{ - vector records; - vector signatures; -}; -typedef map, BothRecordsAndSignatures> recsig_t; - -struct NegCacheEntry -{ - DNSName d_name; - QType d_qtype; - DNSName d_qname; - uint32_t d_ttd; - uint32_t getTTD() const - { - return d_ttd; - } - recsig_t d_dnssecProof; -}; - typedef map< DNSName, pair< @@ -433,21 +414,6 @@ public: unsigned int d_totUsec; ComboAddress d_requestor; - typedef multi_index_container < - NegCacheEntry, - indexed_by < - ordered_unique< - composite_key< - NegCacheEntry, - member, - member - >, - composite_key_compare > - >, - sequenced<> - > - > negcache_t; - //! This represents a number of decaying Ewmas, used to store performance per nameserver-name. /** Modelled to work mostly like the underlying DecayingEwma. After you've called get, d_best is filled out with the best address for this collection */ @@ -551,13 +517,13 @@ public: static string s_serverID; struct StaticStorage { - negcache_t negcache; nsspeeds_t nsSpeeds; ednsstatus_t ednsstatus; throttle_t throttle; fails_t fails; domainmap_t* domainmap; map dnssecmap; + NegCache negcache; }; private: