From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Fri, 19 Jun 2020 16:45:37 +0000 (+0200)
Subject: rec: Use a separate cache for aggressive NSEC to keep things simple
X-Git-Tag: dnsdist-1.6.0-alpha2~12^2~33
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=25f5783a8aa35e7c4be9a4b745f6d417846ac476;p=thirdparty%2Fpdns.git

rec: Use a separate cache for aggressive NSEC to keep things simple
---

diff --git a/pdns/lock.hh b/pdns/lock.hh
index 5cbc4404d7..98f386dc3d 100644
--- a/pdns/lock.hh
+++ b/pdns/lock.hh
@@ -36,7 +36,6 @@ public:
   }
 
   ~ReadWriteLock() {
-    /* might have been moved */
     pthread_rwlock_destroy(&d_lock);
   }
 
diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc
index db1deb5db0..17755c5bd6 100644
--- a/pdns/pdns_recursor.cc
+++ b/pdns/pdns_recursor.cc
@@ -68,6 +68,7 @@
 #include "malloctrace.hh"
 #endif
 #include <netinet/tcp.h>
+#include "aggressive_nsec.hh"
 #include "capabilities.hh"
 #include "dnsparser.hh"
 #include "dnswriter.hh"
@@ -4740,6 +4741,15 @@ static int serviceMain(int argc, char*argv[])
 
   s_addExtendedResolutionDNSErrors = ::arg().mustDo("extended-resolution-errors");
 
+  if (::arg().mustDo("aggressive-nsec")) {
+    if (g_dnssecmode == DNSSECMode::ValidateAll || g_dnssecmode == DNSSECMode::ValidateForLog) {
+      g_aggressiveNSECCache = make_unique<AggressiveNSECCache>();
+    }
+    else {
+      g_log<<Logger::Warning<<"Aggressive NSEC/NSEC3 caching is enabled but DNSSEC validation is not set to 'validate' or 'log-fail', ignoring"<<endl;
+    }
+  }
+
   {
     SuffixMatchNode dontThrottleNames;
     vector<string> parts;
@@ -5506,6 +5516,8 @@ int main(int argc, char **argv)
 
     ::arg().setSwitch("extended-resolution-errors", "If set, send an EDNS Extended Error extension on resolution failures, like DNSSEC validation errors")="no";
 
+    ::arg().setSwitch("aggressive-nsec", "If set, and DNSSEC validation is enabled, the recursor will look at cached NSEC and NSEC3 records to generate negative answers, as defined in rfc8198")="no";
+
     ::arg().setCmd("help","Provide a helpful message");
     ::arg().setCmd("version","Print version string");
     ::arg().setCmd("config","Output blank configuration");
diff --git a/pdns/recursor_cache.cc b/pdns/recursor_cache.cc
index 28e3832f5b..c7c2414ebe 100644
--- a/pdns/recursor_cache.cc
+++ b/pdns/recursor_cache.cc
@@ -440,7 +440,7 @@ void MemRecursorCache::replace(time_t now, const DNSName &qname, const QType qt,
   time_t maxTTD=std::numeric_limits<time_t>::max();
   CacheEntry ce=*stored; // this is a COPY
   ce.d_qtype=qt.getCode();
-  
+
   if(!auth && ce.d_auth) {  // unauth data came in, we have some auth data, but is it fresh?
     if(ce.d_ttd > now) { // we still have valid data, ignore unauth data
       return;
diff --git a/pdns/recursordist/Makefile.am b/pdns/recursordist/Makefile.am
index 2bc4043565..a45f8294bf 100644
--- a/pdns/recursordist/Makefile.am
+++ b/pdns/recursordist/Makefile.am
@@ -94,6 +94,7 @@ check-local:
 endif
 
 pdns_recursor_SOURCES = \
+	aggressive_nsec.cc aggressive_nsec.hh \
 	arguments.cc \
 	ascii.hh \
 	axfr-retriever.hh axfr-retriever.cc \
@@ -226,6 +227,7 @@ pdns_recursor_LDFLAGS += \
 endif
 
 testrunner_SOURCES = \
+	aggressive_nsec.cc aggressive_nsec.hh \
 	arguments.cc \
 	axfr-retriever.hh axfr-retriever.cc \
 	base32.cc \
diff --git a/pdns/recursordist/aggressive_nsec.cc b/pdns/recursordist/aggressive_nsec.cc
new file mode 100644
index 0000000000..b0bba44f34
--- /dev/null
+++ b/pdns/recursordist/aggressive_nsec.cc
@@ -0,0 +1,578 @@
+/*
+ * This file is part of PowerDNS or dnsdist.
+ * Copyright -- PowerDNS.COM B.V. and its contributors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * In addition, for the avoidance of any doubt, permission is granted to
+ * link this program with OpenSSL and to (re)distribute the binaries
+ * produced as the result of such linking.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include "aggressive_nsec.hh"
+#include "recursor_cache.hh"
+#include "validate.hh"
+
+std::unique_ptr<AggressiveNSECCache> g_aggressiveNSECCache{nullptr};
+
+/* this is defined in syncres.hh and we are not importing that here */
+extern std::unique_ptr<MemRecursorCache> g_recCache;
+
+std::shared_ptr<AggressiveNSECCache::ZoneEntry> AggressiveNSECCache::getBestZone(const DNSName& zone)
+{
+  std::shared_ptr<AggressiveNSECCache::ZoneEntry> entry{nullptr};
+  {
+    ReadLock rl(d_lock);
+    auto got = d_zones.lookup(zone);
+    if (got) {
+      return *got;
+    }
+  }
+  return entry;
+}
+
+std::shared_ptr<AggressiveNSECCache::ZoneEntry> AggressiveNSECCache::getZone(const DNSName& zone)
+{
+  std::shared_ptr<AggressiveNSECCache::ZoneEntry> entry{nullptr};
+  {
+    ReadLock rl(d_lock);
+    auto got = d_zones.lookup(zone);
+    if (got && *got && (*got)->d_zone == zone) {
+      return *got;
+    }
+  }
+
+  entry = std::make_shared<ZoneEntry>();
+  entry->d_zone = zone;
+
+  {
+    WriteLock wl(d_lock);
+    /* it might have been inserted in the mean time */
+    auto got = d_zones.lookup(zone);
+    if (got && *got && (*got)->d_zone == zone) {
+      return *got;
+    }
+    d_zones.add(zone, std::shared_ptr<ZoneEntry>(entry));
+    return entry;
+  }
+}
+
+void AggressiveNSECCache::insertNSEC(const DNSName& zone, const DNSName& owner, const DNSRecord& record, const std::vector<std::shared_ptr<RRSIGRecordContent>>& signatures, bool nsec3)
+{
+  std::shared_ptr<AggressiveNSECCache::ZoneEntry> entry = getZone(zone);
+  {
+    std::lock_guard<std::mutex> lock(entry->d_lock);
+    if (nsec3 && !entry->d_nsec3) {
+      entry->d_entries.clear();
+      entry->d_nsec3 = true;
+    }
+
+    DNSName next;
+    if (!nsec3) {
+      auto content = getRR<NSECRecordContent>(record);
+      if (!content) {
+        throw std::runtime_error("Error getting the content from a NSEC record");
+      }
+      next = content->d_next;
+    }
+    else {
+      auto content = getRR<NSEC3RecordContent>(record);
+      if (!content) {
+        throw std::runtime_error("Error getting the content from a NSEC3 record");
+      }
+      next = DNSName(content->d_nexthash) + zone;
+      entry->d_iterations = content->d_iterations;
+      entry->d_salt = content->d_salt;
+    }
+
+    /* the TTL is already a TTD by now */
+    entry->d_entries.insert({record.d_content, signatures, owner, std::move(next), record.d_ttl});
+  }
+}
+
+bool AggressiveNSECCache::getNSECBefore(time_t now, std::shared_ptr<AggressiveNSECCache::ZoneEntry>& zoneEntry, const DNSName& name, ZoneEntry::CacheEntry& entry) {
+
+  std::lock_guard<std::mutex> lock(zoneEntry->d_lock);
+  if (zoneEntry->d_entries.empty()) {
+    return false;
+  }
+
+  auto& idx = zoneEntry->d_entries.get<ZoneEntry::OrderedTag>();
+  auto it = idx.lower_bound(name);
+  bool end = false;
+
+  while (!end && (it == idx.end() || (it->d_owner != name && !it->d_owner.canonCompare(name))))
+  {
+    if (it == idx.end()) {
+      cerr<<"GOT END"<<endl;
+    }
+    else {
+      cerr<<"got "<<it->d_owner<<endl;
+    }
+
+    if (it == idx.begin()) {
+      // can't go further, but perhaps we wrapped?
+      it = idx.end();
+      it--;
+      if (!it->d_owner.canonCompare(name) && it->d_next.canonCompare(name)) {
+        break;
+      }
+      end = true;
+      break;
+    }
+    else {
+      it--;
+       cerr<<"looping with "<<it->d_owner<<endl;
+    }
+  }
+
+  if (end) {
+     cerr<<"nothing left"<<endl;
+    return false;
+  }
+
+  cerr<<"considering "<<it->d_owner<<" "<<it->d_next<<endl;
+
+  if (it->d_ttd <= now) {
+     cerr<<"not using it"<<endl;
+    return false;
+  }
+
+  entry = *it;
+  return true;
+}
+
+bool AggressiveNSECCache::getDenial(time_t now, const DNSName& name, const QType& type, std::vector<DNSRecord>&ret, int& res, const ComboAddress& who, const boost::optional<std::string>& routingTag, bool doDNSSEC)
+{
+  auto zoneEntry = getBestZone(name);
+  if (!zoneEntry) {
+    cerr<<"zone info not found"<<endl;
+    return false;
+  }
+
+  vState cachedState;
+  std::vector<DNSRecord> soaSet;
+  std::vector<std::shared_ptr<RRSIGRecordContent>> soaSignatures;
+  if (g_recCache->get(now, zoneEntry->d_zone, QType::SOA, true, &soaSet, who, false, routingTag, doDNSSEC ? &soaSignatures : nullptr, nullptr, nullptr, &cachedState) <= 0 || cachedState != vState::Secure) {
+    cerr<<"could not find SOA"<<endl;
+    return false;
+  }
+
+  if (zoneEntry->d_nsec3) {
+    cerr<<"nsec 3"<<endl;
+    return false;
+    //return doAggressiveNSEC3Cache(prefix, qname, qtype, zone, salt, iterations, ret, res, state);
+  }
+
+  ZoneEntry::CacheEntry entry;
+  ZoneEntry::CacheEntry wcEntry;
+  bool covered = false;
+  bool needWildcard = false;
+
+  cerr<<"looking for nsec before "<<name<<endl;
+  if (!getNSECBefore(now, zoneEntry, name, entry)) {
+    cerr<<"nothing found in aggressive cache either"<<endl;
+    return false;
+  }
+
+  auto content = std::dynamic_pointer_cast<NSECRecordContent>(entry.d_record);
+  if (!content) {
+    return false;
+  }
+
+  cerr<<"nsecFound "<<entry.d_owner<<endl;
+  auto denial = matchesNSEC(name, type.getCode(), entry.d_owner, content, entry.d_signatures);
+  if (denial == dState::NXQTYPE) {
+    covered = true;
+    res = RCode::NoError;
+  }
+  else if (denial == dState::NXDOMAIN) {
+    if (name.countLabels() > 1) {
+      DNSName wc(name);
+      wc.chopOff();
+      wc = g_wildcarddnsname + wc;
+
+      cerr<<"looking for nsec before "<<wc<<endl;
+      if (!getNSECBefore(now, zoneEntry, wc, wcEntry)) {
+         cerr<<"nothing found in aggressive cache for Wildcard"<<endl;
+        return false;
+      }
+
+      cerr<<"wc nsec found "<<wcEntry.d_owner<<endl;
+      if (wcEntry.d_owner == entry.d_owner) {
+        covered = true;
+        res = RCode::NXDomain;
+      }
+      else {
+        if (wcEntry.d_owner == wc) {
+#warning FIXME: if the wc does exist but the type does not, it is actually quite simple
+          /* too complicated for now */
+          return false;
+        }
+        else if (isCoveredByNSEC(wc, wcEntry.d_owner, wcEntry.d_next)) {
+          cerr<<"next is "<<wcEntry.d_next<<endl;
+          covered = true;
+          res = RCode::NXDomain;
+          needWildcard = true;
+        }
+      }
+    }
+  }
+
+  if (!covered) {
+    return false;
+  }
+
+  uint32_t ttl=0;
+
+  ret.reserve(ret.size() + soaSet.size() + soaSignatures.size() + /* NSEC */ 1 + entry.d_signatures.size() + (needWildcard ? (/* NSEC */ 1 + wcEntry.d_signatures.size()) : 0));
+
+  for (auto& record : soaSet) {
+    if (record.d_class != QClass::IN) {
+      continue;
+    }
+
+    record.d_ttl -= now;
+    ttl = record.d_ttl;
+    ret.push_back(std::move(record));
+  }
+
+  if (doDNSSEC) {
+    for (auto& signature : soaSignatures) {
+      DNSRecord dr;
+      dr.d_type = QType::RRSIG;
+      dr.d_name = zoneEntry->d_zone;
+      dr.d_ttl = ttl;
+      dr.d_content = std::move(signature);
+      dr.d_place = DNSResourceRecord::AUTHORITY;
+      dr.d_class = QClass::IN;
+      ret.push_back(std::move(dr));
+    }
+  }
+
+  DNSRecord nsecRec;
+  nsecRec.d_type = QType::NSEC;
+  nsecRec.d_name = entry.d_owner;
+  nsecRec.d_ttl = entry.d_ttd - now;
+  ttl = nsecRec.d_ttl;
+  nsecRec.d_content = std::move(entry.d_record);
+  nsecRec.d_place = DNSResourceRecord::AUTHORITY;
+  nsecRec.d_class = QClass::IN;
+  ret.push_back(std::move(nsecRec));
+
+  if (doDNSSEC) {
+    for (auto& signature : entry.d_signatures) {
+      DNSRecord dr;
+      dr.d_type = QType::RRSIG;
+      dr.d_name = entry.d_owner;
+      dr.d_ttl = ttl;
+      dr.d_content = std::move(signature);
+      dr.d_place = DNSResourceRecord::AUTHORITY;
+      dr.d_class = QClass::IN;
+      ret.push_back(std::move(dr));
+    }
+  }
+
+  if (needWildcard) {
+    DNSRecord wcNsecRec;
+    wcNsecRec.d_type = QType::NSEC;
+    wcNsecRec.d_name = wcEntry.d_owner;
+    wcNsecRec.d_ttl = wcEntry.d_ttd - now;
+    ttl = wcNsecRec.d_ttl;
+    wcNsecRec.d_content = std::move(wcEntry.d_record);
+    wcNsecRec.d_place = DNSResourceRecord::AUTHORITY;
+    wcNsecRec.d_class = QClass::IN;
+    ret.push_back(std::move(wcNsecRec));
+
+    if (doDNSSEC) {
+      for (auto& signature : wcEntry.d_signatures) {
+        DNSRecord dr;
+        dr.d_type = QType::RRSIG;
+        dr.d_name = wcEntry.d_owner;
+        dr.d_ttl = ttl;
+        dr.d_content = std::move(signature);
+        dr.d_place = DNSResourceRecord::AUTHORITY;
+        dr.d_class = QClass::IN;
+        ret.push_back(std::move(dr));
+      }
+    }
+  }
+
+  return true;
+}
+
+#if 0
+
+bool SyncRes::doAggressiveNSEC3Cache(const std::string& prefix, const DNSName& qname, const QType& qtype, const DNSName& zone, const std::string& salt, uint16_t iterations, vector<DNSRecord>&ret, int& res, vState& state)
+{
+#warning FIXME: nsec3
+  cerr<<"nsec3, sorry"<<endl;
+  if (g_maxNSEC3Iterations && iterations > g_maxNSEC3Iterations) {
+    return false;
+  }
+
+  vector<DNSRecord> cset;
+  vector<std::shared_ptr<RRSIGRecordContent>> signatures;
+  vState cachedState;
+
+  auto qnameHash = toBase32Hex(hashQNameWithSalt(salt, iterations, qname));
+
+  cerr<<"looking for nsec3 "<<(DNSName(qnameHash) + zone)<<endl;
+  DNSName nsec3Found;
+  if (g_recCache->get(d_now.tv_sec, DNSName(qnameHash) + zone, QType::NSEC3, true, &cset, d_cacheRemote, false, d_routingTag, d_doDNSSEC ? &signatures : nullptr, nullptr, nullptr, &cachedState, nullptr, &zone) > 0) {
+    cerr<<"found direct match "<<qnameHash<<endl;
+    if (cachedState == vState::Secure) {
+      cerr<<"but not secure"<<endl;
+      return false;
+    }
+
+    return false;
+  }
+
+  cerr<<"no direct match, looking for closest encloser"<<endl;
+  DNSName closestEncloser(qname);
+  bool found = false;
+  while (!found && closestEncloser.chopOff()) {
+    auto closestHash = toBase32Hex(hashQNameWithSalt(salt, iterations, closestEncloser));
+    cerr<<"looking for nsec3 "<<(DNSName(closestHash) + zone)<<endl;
+
+    if (g_recCache->get(d_now.tv_sec, DNSName(closestHash) + zone, QType::NSEC3, true, &cset, d_cacheRemote, false, d_routingTag, d_doDNSSEC ? &signatures : nullptr, nullptr, nullptr, &cachedState, nullptr, &zone) > 0) {
+      cerr<<"found direct match for closest encloser "<<closestHash<<endl;
+      found = true;
+      break;
+    }
+  }
+
+  if (!found) {
+    cerr<<"nothing found in aggressive cache either"<<endl;
+    return false;
+  }
+
+  unsigned int labelIdx = qname.countLabels() - closestEncloser.countLabels();
+  if (labelIdx < 1) {
+    return false;
+  }
+
+  DNSName nsecFound;
+  DNSName nextCloser(closestEncloser);
+  nextCloser.prependRawLabel(qname.getRawLabel(labelIdx - 1));
+  auto nextCloserHash = toBase32Hex(hashQNameWithSalt(salt, iterations, nextCloser));
+  cerr<<"looking for a NSEC3 covering the next closer "<<nextCloser<<": "<<nextCloserHash<<endl;
+
+  if (!g_recCache->getNSECBefore(d_now.tv_sec, zone, DNSName(nextCloserHash) + zone, QType::NSEC3, nsecFound, cset, signatures, cachedState)) {
+    cerr<<"nothing found for the next closer in aggressive cache"<<endl;
+    return false;
+  }
+
+  DNSName wildcard(g_wildcarddnsname + closestEncloser);
+  auto wcHash = toBase32Hex(hashQNameWithSalt(salt, iterations, wildcard));
+  cerr<<"looking for a NSEC3 covering the wildcard "<<wildcard<<": "<<wcHash<<endl;
+
+  if (!g_recCache->getNSECBefore(d_now.tv_sec, zone, DNSName(wcHash) + zone, QType::NSEC3, nsecFound, cset, signatures, cachedState)) {
+    cerr<<"nothing found for the wildcard in aggressive cache"<<endl;
+    return false;
+  }
+
+  return false;
+}
+
+bool SyncRes::doAggressiveNSECCacheCheck(const std::string& prefix, const DNSName& qname, const QType& qtype, vector<DNSRecord>&ret, int& res, vState& state)
+{
+  if (!g_aggressiveNSECCache) {
+    cerr<<"no aggressive NSEC"<<endl;
+    return false;
+  }
+
+  DNSName zone(qname);
+  std::string salt;
+  uint16_t iterations = 0;
+  bool nsec3 = false;
+  if (!g_aggressiveNSECCache->getBestZoneInfo(zone, nsec3, salt, iterations)) {
+    cerr<<"zone info not found"<<endl;
+    return false;
+  }
+
+  vState cachedState;
+  std::vector<DNSRecord> soaSet;
+  std::vector<std::shared_ptr<RRSIGRecordContent>> soaSignatures;
+  if (g_recCache->get(d_now.tv_sec, zone, QType::SOA, true, &soaSet, d_cacheRemote, false, d_routingTag, d_doDNSSEC ? &soaSignatures : nullptr, nullptr, nullptr, &cachedState) <= 0 || cachedState != vState::Secure) {
+    cerr<<"could not find SOA"<<endl;
+    return false;
+  }
+
+  vector<DNSRecord> cset;
+  vector<std::shared_ptr<RRSIGRecordContent>> signatures;
+
+  if (nsec3) {
+    return doAggressiveNSEC3Cache(prefix, qname, qtype, zone, salt, iterations, ret, res, state);
+  }
+
+  DNSName nsecFound;
+  std::vector<DNSRecord> wcSet;
+  std::vector<std::shared_ptr<RRSIGRecordContent>> wcSignatures;
+
+   cerr<<"looking for nsec before "<<qname<<endl;
+   if (!g_recCache->getNSECBefore(d_now.tv_sec, zone, qname, QType::NSEC, nsecFound, cset, signatures, cachedState)) {
+     cerr<<"nothing found in aggressive cache either"<<endl;
+    return false;
+  }
+
+   cerr<<"nsecFound "<<nsecFound<<endl;
+  if (cset.empty() || cachedState != vState::Secure) {
+    return false;
+  }
+
+  bool covered = false;
+   cerr<<"Got "<<cset.size()<<" records"<<endl;
+  const auto& nsecRecord = cset.at(0);
+  auto content = getRR<NSECRecordContent>(nsecRecord);
+  if (!content) {
+    return false;
+  }
+
+   cerr<<"next is "<<content->d_next<<endl;
+  auto denial = matchesNSEC(qname, qtype.getCode(), nsecRecord, signatures);
+  if (denial == dState::NXQTYPE) {
+    covered = true;
+    res = RCode::NoError;
+  }
+  else if (denial == dState::NXDOMAIN) {
+    if (qname.countLabels() > 1) {
+      DNSName wc = qname;
+      wc.chopOff();
+      wc = g_wildcarddnsname + wc;
+
+       cerr<<"looking for nsec before "<<wc<<endl;
+      DNSName wcNSEC;
+      if (!g_recCache->getNSECBefore(d_now.tv_sec, zone, wc, QType::NSEC, wcNSEC, wcSet, wcSignatures, cachedState)) {
+         cerr<<"nothing found in aggressive cache for Wildcard"<<endl;
+        return false;
+      }
+
+      if (wcSet.empty() || cachedState != vState::Secure) {
+        cerr<<"nothing usable"<<endl;
+        return false;
+      }
+
+      cerr<<"wc nsec found "<<wcNSEC<<endl;
+      if (wcNSEC == nsecFound) {
+        wcSet.clear();
+        wcSignatures.clear();
+        covered = true;
+        res = RCode::NXDomain;
+      }
+      else {
+        const auto& wcNsecRecord = wcSet.at(0);
+        auto wcContent = getRR<NSECRecordContent>(wcNsecRecord);
+        if (!wcContent) {
+          return false;
+        }
+
+        if (wcNSEC == wc) {
+          /* too complicated for now */
+          return false;
+        }
+        else if (isCoveredByNSEC(wc, wcNsecRecord.d_name, wcContent->d_next)) {
+          cerr<<"next is "<<wcContent->d_next<<endl;
+          covered = true;
+          res = RCode::NXDomain;
+        }
+      }
+    }
+  }
+
+  if (!covered) {
+    return false;
+  }
+
+  LOG(prefix<<qname<<": Found aggressive NSEC cache hit for "<<qtype.getName()<<endl);
+
+  uint32_t ttl=0;
+  uint32_t capTTL = std::numeric_limits<uint32_t>::max();
+
+  ret.reserve(ret.size() + soaSet.size() + soaSignatures.size() + cset.size() + signatures.size() + wcSet.size() + wcSignatures.size());
+
+  for (auto& record : soaSet) {
+    if (record.d_class != QClass::IN) {
+      continue;
+    }
+
+    record.d_ttl -= d_now.tv_sec;
+    record.d_ttl = std::min(record.d_ttl, capTTL);
+    ttl = record.d_ttl;
+    ret.push_back(std::move(record));
+  }
+
+  for (auto& signature : soaSignatures) {
+    DNSRecord dr;
+    dr.d_type = QType::RRSIG;
+    dr.d_name = zone;
+    dr.d_ttl = ttl;
+    dr.d_content = std::move(signature);
+    dr.d_place = DNSResourceRecord::ANSWER;
+    dr.d_class = QClass::IN;
+    ret.push_back(std::move(dr));
+  }
+
+  for (auto& record : cset) {
+    if (record.d_class != QClass::IN) {
+      continue;
+    }
+
+    record.d_ttl -= d_now.tv_sec;
+    record.d_ttl = std::min(record.d_ttl, capTTL);
+    ttl = record.d_ttl;
+    ret.push_back(std::move(record));
+  }
+
+  for (auto& signature : signatures) {
+    DNSRecord dr;
+    dr.d_type = QType::RRSIG;
+    dr.d_name = qname;
+    dr.d_ttl = ttl;
+    dr.d_content = std::move(signature);
+    dr.d_place = DNSResourceRecord::ANSWER;
+    dr.d_class = QClass::IN;
+    ret.push_back(std::move(dr));
+  }
+
+  for (auto& record : wcSet) {
+    if (record.d_class != QClass::IN) {
+      continue;
+    }
+
+    record.d_ttl -= d_now.tv_sec;
+    record.d_ttl = std::min(record.d_ttl, capTTL);
+    ttl = record.d_ttl;
+    ret.push_back(std::move(record));
+  }
+
+  for (auto& signature : wcSignatures) {
+    DNSRecord dr;
+    dr.d_type = QType::RRSIG;
+    dr.d_name = qname;
+    dr.d_ttl = ttl;
+    dr.d_content = std::move(signature);
+    dr.d_place = DNSResourceRecord::ANSWER;
+    dr.d_class = QClass::IN;
+    ret.push_back(std::move(dr));
+  }
+
+  /* we would have given up otherwise */
+  state = vState::Secure;
+
+  return true;
+}
+
+#endif
diff --git a/pdns/recursordist/aggressive_nsec.hh b/pdns/recursordist/aggressive_nsec.hh
new file mode 100644
index 0000000000..5c8d8d1934
--- /dev/null
+++ b/pdns/recursordist/aggressive_nsec.hh
@@ -0,0 +1,103 @@
+/*
+ * This file is part of PowerDNS or dnsdist.
+ * Copyright -- PowerDNS.COM B.V. and its contributors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * In addition, for the avoidance of any doubt, permission is granted to
+ * link this program with OpenSSL and to (re)distribute the binaries
+ * produced as the result of such linking.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+#pragma once
+
+#include <mutex>
+
+#include <boost/utility.hpp>
+#include <boost/multi_index_container.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/hashed_index.hpp>
+#include <boost/multi_index/key_extractors.hpp>
+#include <boost/multi_index/sequenced_index.hpp>
+
+#include "dnsname.hh"
+#include "dnsrecords.hh"
+#include "lock.hh"
+
+class AggressiveNSECCache
+{
+public:
+  void insertNSEC(const DNSName& zone, const DNSName& owner, const DNSRecord& record, const std::vector<std::shared_ptr<RRSIGRecordContent>>& signatures, bool nsec3);
+  bool getDenial(time_t, const DNSName& name, const QType& type, std::vector<DNSRecord>& ret, int& res, const ComboAddress& who, const boost::optional<std::string>& routingTag, bool doDNSSEC);
+
+  //bool getBestZoneInfo(DNSName& lookup, bool& nsec3, std::string& salt, uint16_t& iterations);
+  //void removeZoneInfo(const DNSName& zone);
+
+private:
+
+  struct ZoneEntry
+  {
+    ZoneEntry()
+    {
+    }
+
+    ZoneEntry(const DNSName& zone, const std::string& salt, uint16_t iterations, bool nsec3): d_zone(zone), d_salt(salt), d_iterations(iterations), d_nsec3(nsec3)
+    {
+    }
+
+    struct HashedTag {};
+    struct SequencedTag {};
+    struct OrderedTag {};
+
+    struct CacheEntry
+    {
+      std::shared_ptr<DNSRecordContent> d_record;
+      std::vector<std::shared_ptr<RRSIGRecordContent>> d_signatures;
+
+      DNSName d_owner;
+      DNSName d_next;
+      time_t d_ttd;
+    };
+
+    typedef multi_index_container<
+      CacheEntry,
+      indexed_by <
+        ordered_unique<tag<OrderedTag>,
+                       member<CacheEntry,DNSName,&CacheEntry::d_owner>,
+                       CanonDNSNameCompare
+                       >,
+        sequenced<tag<SequencedTag> >,
+        hashed_non_unique<tag<HashedTag>,
+                          member<CacheEntry,DNSName,&CacheEntry::d_owner>
+                          >
+        >
+      > cache_t;
+
+    cache_t d_entries;
+    DNSName d_zone;
+    std::string d_salt;
+    std::mutex d_lock;
+    uint16_t d_iterations{0};
+    bool d_nsec3{false};
+  };
+
+  std::shared_ptr<ZoneEntry> getZone(const DNSName& zone);
+  std::shared_ptr<ZoneEntry> getBestZone(const DNSName& zone);
+  bool getNSECBefore(time_t now, std::shared_ptr<ZoneEntry>& zoneEntry, const DNSName& name, ZoneEntry::CacheEntry& entry);
+
+  SuffixMatchTree<std::shared_ptr<ZoneEntry>> d_zones;
+  ReadWriteLock d_lock;
+};
+
+
+extern std::unique_ptr<AggressiveNSECCache> g_aggressiveNSECCache;
diff --git a/pdns/syncres.cc b/pdns/syncres.cc
index 2a4345dc47..18a8e81bb3 100644
--- a/pdns/syncres.cc
+++ b/pdns/syncres.cc
@@ -24,6 +24,7 @@
 #endif
 
 #include "arguments.hh"
+#include "aggressive_nsec.hh"
 #include "cachecleaner.hh"
 #include "dns_random.hh"
 #include "dnsparser.hh"
@@ -1942,6 +1943,18 @@ bool SyncRes::doCacheCheck(const DNSName &qname, const DNSName& authname, bool w
       LOG(prefix<<qname<<": cache had only stale entries"<<endl);
   }
 
+  /* let's check if we have a NSEC covering that record */
+  if (g_aggressiveNSECCache) {
+    cerr<<"calling get denial"<<endl;
+    if (g_aggressiveNSECCache->getDenial(d_now.tv_sec, qname, qtype, ret, res, d_cacheRemote, d_routingTag, d_doDNSSEC)) {
+      state = vState::Secure;
+      return true;
+    }
+  }
+  else {
+    cerr<<"no cache"<<endl;
+  }
+
   return false;
 }
 
@@ -3015,6 +3028,8 @@ RCode::rcodes_ SyncRes::updateCacheFromRecords(unsigned int depth, LWResult& lwr
   const unsigned int labelCount = qname.countLabels();
   bool isCNAMEAnswer = false;
   bool isDNAMEAnswer = false;
+  DNSName seenAuth;
+
   for (auto& rec : lwr.d_records) {
     if (rec.d_type == QType::OPT || rec.d_class != QClass::IN) {
       continue;
@@ -3022,14 +3037,18 @@ RCode::rcodes_ SyncRes::updateCacheFromRecords(unsigned int depth, LWResult& lwr
 
     rec.d_ttl = min(s_maxcachettl, rec.d_ttl);
 
-    if(!isCNAMEAnswer && rec.d_place == DNSResourceRecord::ANSWER && rec.d_type == QType::CNAME && (!(qtype==QType::CNAME)) && rec.d_name == qname && !isDNAMEAnswer) {
+    if (!isCNAMEAnswer && rec.d_place == DNSResourceRecord::ANSWER && rec.d_type == QType::CNAME && (!(qtype==QType::CNAME)) && rec.d_name == qname && !isDNAMEAnswer) {
       isCNAMEAnswer = true;
     }
-    if(!isDNAMEAnswer && rec.d_place == DNSResourceRecord::ANSWER && rec.d_type == QType::DNAME && qtype != QType::DNAME && qname.isPartOf(rec.d_name)) {
+    if (!isDNAMEAnswer && rec.d_place == DNSResourceRecord::ANSWER && rec.d_type == QType::DNAME && qtype != QType::DNAME && qname.isPartOf(rec.d_name)) {
       isDNAMEAnswer = true;
       isCNAMEAnswer = false;
     }
 
+    if (rec.d_type == QType::SOA && rec.d_place == DNSResourceRecord::AUTHORITY && qname.isPartOf(rec.d_name)) {
+      seenAuth = rec.d_name;
+    }
+
     if (rec.d_type == QType::RRSIG) {
       auto rrsig = getRR<RRSIGRecordContent>(rec);
       if (rrsig) {
@@ -3333,13 +3352,21 @@ RCode::rcodes_ SyncRes::updateCacheFromRecords(unsigned int depth, LWResult& lwr
           }
         }
       }
+
       if (doCache) {
         g_recCache->replace(d_now.tv_sec, i->first.name, i->first.type, i->second.records, i->second.signatures, authorityRecs, i->first.type == QType::DS ? true : isAA, auth, i->first.place == DNSResourceRecord::ANSWER ? ednsmask : boost::none, d_routingTag, recordState, remoteIP);
       }
     }
 
-    if(i->first.place == DNSResourceRecord::ANSWER && ednsmask)
+    if ((i->first.type == QType::NSEC || i->first.type == QType::NSEC3) && recordState == vState::Secure && !seenAuth.empty() && g_aggressiveNSECCache) {
+      cerr<<"Good candidate for aggressive NSEC caching"<<endl;
+
+      g_aggressiveNSECCache->insertNSEC(seenAuth, i->first.name, i->second.records.at(0), i->second.signatures, i->first.type == QType::NSEC3);
+    }
+
+    if (i->first.place == DNSResourceRecord::ANSWER && ednsmask) {
       d_wasVariable=true;
+    }
   }
 
   return RCode::NoError;
diff --git a/pdns/validate.cc b/pdns/validate.cc
index 7f6bac214a..8794798281 100644
--- a/pdns/validate.cc
+++ b/pdns/validate.cc
@@ -63,7 +63,7 @@ static bool isCoveredByNSEC3Hash(const std::string& h, const std::string& beginH
           (beginHash == nextHash && h != beginHash));   // "we have only 1 NSEC3 record, LOL!"
 }
 
-static bool isCoveredByNSEC(const DNSName& name, const DNSName& begin, const DNSName& next)
+bool isCoveredByNSEC(const DNSName& name, const DNSName& begin, const DNSName& next)
 {
   return ((begin.canonCompare(name) && name.canonCompare(next)) ||  // no wrap          BEGINNING --- NAME --- NEXT
           (name.canonCompare(next) && next.canonCompare(begin)) ||  // wrap             NAME --- NEXT --- BEGINNING
@@ -376,6 +376,51 @@ static bool provesNSEC3NoWildCard(DNSName wildcard, uint16_t const qtype, const
   return false;
 }
 
+dState matchesNSEC(const DNSName& name, uint16_t qtype, const DNSName& nsecOwner, const std::shared_ptr<NSECRecordContent>& nsec, const std::vector<std::shared_ptr<RRSIGRecordContent>>& signatures)
+{
+  const DNSName signer = getSigner(signatures);
+  if (!name.isPartOf(signer) || !nsecOwner.isPartOf(signer)) {
+    return dState::NODENIAL;
+  }
+
+  const DNSName owner = getNSECOwnerName(nsecOwner, signatures);
+  /* RFC 6840 section 4.1 "Clarifications on Nonexistence Proofs":
+     Ancestor delegation NSEC or NSEC3 RRs MUST NOT be used to assume
+     nonexistence of any RRs below that zone cut, which include all RRs at
+     that (original) owner name other than DS RRs, and all RRs below that
+     owner name regardless of type.
+  */
+  if (qtype != QType::DS && (name == owner || name.isPartOf(owner)) && isNSECAncestorDelegation(signer, owner, nsec)) {
+    /* this is an "ancestor delegation" NSEC RR */
+    return dState::NODENIAL;
+  }
+
+  /* check if the type is denied */
+  if (name == owner) {
+    if (nsec->isSet(qtype)) {
+      return dState::NODENIAL;
+    }
+
+    /* RFC 6840 section 4.3 */
+    if (nsec->isSet(QType::CNAME)) {
+      return dState::NODENIAL;
+    }
+
+    return dState::NXQTYPE;
+  }
+
+  if (isCoveredByNSEC(name, owner, nsec->d_next)) {
+
+    if (nsecProvesENT(name, owner, nsec->d_next)) {
+      return dState::NXQTYPE;
+    }
+
+    return dState::NXDOMAIN;
+  }
+
+  return dState::NODENIAL;
+}
+
 /*
   This function checks whether the existence of qname|qtype is denied by the NSEC and NSEC3
   in validrrsets.
@@ -402,6 +447,7 @@ dState getDenial(const cspmap_t &validrrsets, const DNSName& qname, const uint16
     if(v.first.second==QType::NSEC) {
       for(const auto& r : v.second.records) {
         LOG("\t"<<r->getZoneRepresentation()<<endl);
+
         auto nsec = std::dynamic_pointer_cast<NSECRecordContent>(r);
         if(!nsec)
           continue;
@@ -425,7 +471,7 @@ dState getDenial(const cspmap_t &validrrsets, const DNSName& qname, const uint16
         }
 
         /* check if the type is denied */
-        if(qname == owner) {
+        if (qname == owner) {
           if (nsec->isSet(qtype)) {
             LOG("Does _not_ deny existence of type "<<QType(qtype).getName()<<endl);
             return dState::NODENIAL;
diff --git a/pdns/validate.hh b/pdns/validate.hh
index 4819eb026e..0493096bc6 100644
--- a/pdns/validate.hh
+++ b/pdns/validate.hh
@@ -72,7 +72,9 @@ struct sharedDNSKeyRecordContentCompare
 
 typedef set<shared_ptr<DNSKEYRecordContent>, sharedDNSKeyRecordContentCompare > skeyset_t;
 
+
 vState validateWithKeySet(time_t now, const DNSName& name, const sortedRecords_t& records, const vector<shared_ptr<RRSIGRecordContent> >& signatures, const skeyset_t& keys, bool validateAllSigs=true);
+bool isCoveredByNSEC(const DNSName& name, const DNSName& begin, const DNSName& next);
 void validateWithKeySet(const cspmap_t& rrsets, cspmap_t& validated, const skeyset_t& keys);
 cspmap_t harvestCSPFromRecs(const vector<DNSRecord>& recs);
 vState getKeysFor(DNSRecordOracle& dro, const DNSName& zone, skeyset_t& keyset);
@@ -88,3 +90,6 @@ bool isRRSIGIncepted(const time_t now, const shared_ptr<RRSIGRecordContent>& sig
 bool isWildcardExpanded(unsigned int labelCount, const std::shared_ptr<RRSIGRecordContent>& sign);
 bool isWildcardExpandedOntoItself(const DNSName& owner, unsigned int labelCount, const std::shared_ptr<RRSIGRecordContent>& sign);
 void updateDNSSECValidationState(vState& state, const vState stateUpdate);
+
+dState matchesNSEC(const DNSName& name, uint16_t qtype, const DNSName& nsecOwner, const std::shared_ptr<NSECRecordContent>& nsecRecord, const std::vector<std::shared_ptr<RRSIGRecordContent>>& signatures);
+