]> git.ipfire.org Git - thirdparty/pdns.git/blobdiff - pdns/syncres.cc
Merge pull request #5570 from rgacogne/rec-neg-validation
[thirdparty/pdns.git] / pdns / syncres.cc
index 3cea08afac1040ccf282e60ee909ef6e81880604..1b1e70208015439414745a20903057cc295c05fd 100644 (file)
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
-#include <boost/algorithm/string.hpp>
 
-#include "lua-recursor4.hh"
-#include "utility.hh"
-#include "syncres.hh"
-#include <iostream>
-#include <map>
-#include "dnsrecords.hh"
-#include <algorithm>
-#include <set>
-#include <cerrno>
-#include <cstdio>
-#include <cstdlib>
-#include <utility>
-#include <deque>
-#include "logger.hh"
-#include "validate.hh"
-#include "misc.hh"
 #include "arguments.hh"
-#include "lwres.hh"
-#include "recursor_cache.hh"
-#include "dnsparser.hh"
+#include "cachecleaner.hh"
 #include "dns_random.hh"
-#include "lock.hh"
+#include "dnsparser.hh"
+#include "dnsrecords.hh"
 #include "ednssubnet.hh"
-#include "cachecleaner.hh"
+#include "logger.hh"
+#include "lua-recursor4.hh"
 #include "rec-lua-conf.hh"
-__thread SyncRes::StaticStorage* t_sstorage;
+#include "syncres.hh"
+#include "validate-recursor.hh"
+
+thread_local SyncRes::ThreadLocalStorage SyncRes::t_sstorage;
+
+std::unordered_set<DNSName> SyncRes::s_delegationOnly;
+std::unique_ptr<NetmaskGroup> SyncRes::s_dontQuery{nullptr};
+NetmaskGroup SyncRes::s_ednssubnets;
+SuffixMatchNode SyncRes::s_ednsdomains;
+string SyncRes::s_serverID;
+SyncRes::LogMode SyncRes::s_lm;
 
 unsigned int SyncRes::s_maxnegttl;
 unsigned int SyncRes::s_maxcachettl;
+unsigned int SyncRes::s_maxqperq;
+unsigned int SyncRes::s_maxtotusec;
+unsigned int SyncRes::s_maxdepth;
+unsigned int SyncRes::s_minimumTTL;
 unsigned int SyncRes::s_packetcachettl;
 unsigned int SyncRes::s_packetcacheservfailttl;
 unsigned int SyncRes::s_serverdownmaxfails;
@@ -67,26 +64,17 @@ std::atomic<uint64_t> SyncRes::s_throttledqueries;
 std::atomic<uint64_t> SyncRes::s_dontqueries;
 std::atomic<uint64_t> SyncRes::s_nodelegated;
 std::atomic<uint64_t> SyncRes::s_unreachables;
+std::atomic<uint64_t> SyncRes::s_ecsqueries;
+std::atomic<uint64_t> SyncRes::s_ecsresponses;
 uint8_t SyncRes::s_ecsipv4limit;
 uint8_t SyncRes::s_ecsipv6limit;
-unsigned int SyncRes::s_minimumTTL;
 bool SyncRes::s_doIPv6;
 bool SyncRes::s_nopacketcache;
 bool SyncRes::s_rootNXTrust;
-unsigned int SyncRes::s_maxqperq;
-unsigned int SyncRes::s_maxtotusec;
-unsigned int SyncRes::s_maxdepth;
-string SyncRes::s_serverID;
-SyncRes::LogMode SyncRes::s_lm;
-std::unordered_set<DNSName> SyncRes::s_delegationOnly;
-std::unique_ptr<NetmaskGroup> SyncRes::s_dontQuery{nullptr};
-NetmaskGroup SyncRes::s_ednssubnets;
-SuffixMatchNode SyncRes::s_ednsdomains;
+bool SyncRes::s_noEDNS;
 
 #define LOG(x) if(d_lm == Log) { L <<Logger::Warning << x; } else if(d_lm == Store) { d_trace << x; }
 
-bool SyncRes::s_noEDNS;
-
 static void accountAuthLatency(int usec, int family)
 {
   if(family == AF_INET) {
@@ -118,23 +106,23 @@ static void accountAuthLatency(int usec, int family)
 
 SyncRes::SyncRes(const struct timeval& now) :  d_outqueries(0), d_tcpoutqueries(0), d_throttledqueries(0), d_timeouts(0), d_unreachables(0),
                                               d_totUsec(0), d_now(now),
-                                              d_cacheonly(false), d_nocache(false), d_doDNSSEC(false), d_doEDNS0(false), d_lm(s_lm)
+                                              d_cacheonly(false), d_doDNSSEC(false), d_doEDNS0(false), d_lm(s_lm)
                                                  
 { 
-  if(!t_sstorage) {
-    t_sstorage = new StaticStorage();
-  }
 }
 
 /** everything begins here - this is the entry point just after receiving a packet */
 int SyncRes::beginResolve(const DNSName &qname, const QType &qtype, uint16_t qclass, vector<DNSRecord>&ret)
 {
+  vState state = Indeterminate;
   s_queries++;
   d_wasVariable=false;
   d_wasOutOfBand=false;
 
-  if (doSpecialNamesResolve(qname, qtype, qclass, ret))
+  if (doSpecialNamesResolve(qname, qtype, qclass, ret)) {
+    d_queryValidationState = Insecure;
     return 0;
+  }
 
   if( (qtype.getCode() == QType::AXFR) || (qtype.getCode() == QType::IXFR))
     return -1;
@@ -145,7 +133,16 @@ int SyncRes::beginResolve(const DNSName &qname, const QType &qtype, uint16_t qcl
     return -1;
 
   set<GetBestNSAnswer> beenthere;
-  int res=doResolve(qname, qtype, ret, 0, beenthere);
+  int res=doResolve(qname, qtype, ret, 0, beenthere, state);
+  d_queryValidationState = state;
+
+  if (d_queryValidationState != Indeterminate) {
+    g_stats.dnssecValidations++;
+  }
+  if (d_DNSSECValidationRequested) {
+    increaseDNSSECStateCounter(d_queryValidationState);
+  }
+
   return res;
 }
 
@@ -162,7 +159,7 @@ int SyncRes::beginResolve(const DNSName &qname, const QType &qtype, uint16_t qcl
  * - version.pdns. CH TXT
  * - id.server. CH TXT
  */
-bool SyncRes::doSpecialNamesResolve(const DNSName &qname, const QType &qtype, const uint16_t &qclass, vector<DNSRecord> &ret)
+bool SyncRes::doSpecialNamesResolve(const DNSName &qname, const QType &qtype, const uint16_t qclass, vector<DNSRecord> &ret)
 {
   static const DNSName arpa("1.0.0.127.in-addr.arpa."), ip6_arpa("1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa."),
     localhost("localhost."), versionbind("version.bind."), idserver("id.server."), versionpdns("version.pdns.");
@@ -208,7 +205,7 @@ bool SyncRes::doSpecialNamesResolve(const DNSName &qname, const QType &qtype, co
     dr.d_ttl = 86400;
     for (const auto& ans : answers) {
       dr.d_type = ans.first;
-      dr.d_content = shared_ptr<DNSRecordContent>(DNSRecordContent::mastermake(ans.first, qclass, ans.second));
+      dr.d_content = DNSRecordContent::mastermake(ans.first, qclass, ans.second);
       ret.push_back(dr);
     }
   }
@@ -218,113 +215,133 @@ bool SyncRes::doSpecialNamesResolve(const DNSName &qname, const QType &qtype, co
 
 
 //! This is the 'out of band resolver', in other words, the authoritative server
-bool SyncRes::doOOBResolve(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, int& res)
+void SyncRes::AuthDomain::addSOA(std::vector<DNSRecord>& records) const
 {
-  string prefix;
-  if(doLog()) {
-    prefix=d_prefix;
-    prefix.append(depth, ' ');
+  SyncRes::AuthDomain::records_t::const_iterator ziter = d_records.find(boost::make_tuple(getName(), QType::SOA));
+  if (ziter != d_records.end()) {
+    DNSRecord dr = *ziter;
+    dr.d_place = DNSResourceRecord::AUTHORITY;
+    records.push_back(dr);
+  }
+  else {
+    // cerr<<qname<<": can't find SOA record '"<<getName()<<"' in our zone!"<<endl;
   }
+}
 
-  LOG(prefix<<qname<<": checking auth storage for '"<<qname<<"|"<<qtype.getName()<<"'"<<endl);
-  DNSName authdomain(qname);
+int SyncRes::AuthDomain::getRecords(const DNSName& qname, uint16_t qtype, std::vector<DNSRecord>& records) const
+{
+  int result = RCode::NoError;
+  records.clear();
 
-  domainmap_t::const_iterator iter=getBestAuthZone(&authdomain);
-  if(iter==t_sstorage->domainmap->end()) {
-    LOG(prefix<<qname<<": auth storage has no zone for this query!"<<endl);
-    return false;
-  }
-  LOG(prefix<<qname<<": auth storage has data, zone='"<<authdomain<<"'"<<endl);
-  pair<AuthDomain::records_t::const_iterator, AuthDomain::records_t::const_iterator> range;
-
-  range=iter->second.d_records.equal_range(tie(qname)); // partial lookup
-
-  ret.clear();
-  AuthDomain::records_t::const_iterator ziter;
-  bool somedata=false;
-  for(ziter=range.first; ziter!=range.second; ++ziter) {
-    somedata=true;
-    if(qtype.getCode()==QType::ANY || ziter->d_type==qtype.getCode() || ziter->d_type==QType::CNAME)  // let rest of nameserver do the legwork on this one
-      ret.push_back(*ziter);
-    else if(ziter->d_type == QType::NS && ziter->d_name.countLabels() > authdomain.countLabels()) { // we hit a delegation point!
-      DNSRecord dr=*ziter;
+  // partial lookup
+  std::pair<records_t::const_iterator,records_t::const_iterator> range = d_records.equal_range(tie(qname));
+
+  SyncRes::AuthDomain::records_t::const_iterator ziter;
+  bool somedata = false;
+
+  for(ziter = range.first; ziter != range.second; ++ziter) {
+    somedata = true;
+
+    if(qtype == QType::ANY || ziter->d_type == qtype || ziter->d_type == QType::CNAME) {
+      // let rest of nameserver do the legwork on this one
+      records.push_back(*ziter);
+    }
+    else if (ziter->d_type == QType::NS && ziter->d_name.countLabels() > getName().countLabels()) {
+      // we hit a delegation point!
+      DNSRecord dr = *ziter;
       dr.d_place=DNSResourceRecord::AUTHORITY;
-      ret.push_back(dr);
+      records.push_back(dr);
     }
   }
-  if(!ret.empty()) {
-    LOG(prefix<<qname<<": exact match in zone '"<<authdomain<<"'"<<endl);
-    res=0;
-    return true;
+
+  if (!records.empty()) {
+    /* We have found an exact match, we're done */
+    // cerr<<qname<<": exact match in zone '"<<getName()<<"'"<<endl;
+    return result;
   }
-  if(somedata) {
-    LOG(prefix<<qname<<": found record in '"<<authdomain<<"', but nothing of the right type, sending SOA"<<endl);
-    ziter=iter->second.d_records.find(boost::make_tuple(authdomain, QType::SOA));
-    if(ziter!=iter->second.d_records.end()) {
-      DNSRecord dr=*ziter;
-      dr.d_place=DNSResourceRecord::AUTHORITY;
-      ret.push_back(dr);
-    }
-    else
-      LOG(prefix<<qname<<": can't find SOA record '"<<authdomain<<"' in our zone!"<<endl);
-    res=RCode::NoError;
-    return true;
+
+  if (somedata) {
+    /* We have records for that name, but not of the wanted qtype */
+    // cerr<<qname<<": found record in '"<<getName()<<"', but nothing of the right type, sending SOA"<<endl;
+    addSOA(records);
+
+    return result;
   }
 
-  LOG(prefix<<qname<<": nothing found so far in '"<<authdomain<<"', trying wildcards"<<endl);
+  // cerr<<qname<<": nothing found so far in '"<<getName()<<"', trying wildcards"<<endl;
   DNSName wcarddomain(qname);
-  while(wcarddomain != iter->first && wcarddomain.chopOff()) {
-    LOG(prefix<<qname<<": trying '*."<<wcarddomain<<"' in "<<authdomain<<endl);
-    range=iter->second.d_records.equal_range(boost::make_tuple(g_wildcarddnsname+wcarddomain));
-    if(range.first==range.second)
+  while(wcarddomain != getName() && wcarddomain.chopOff()) {
+    // cerr<<qname<<": trying '*."<<wcarddomain<<"' in "<<getName()<<endl;
+    range = d_records.equal_range(boost::make_tuple(g_wildcarddnsname + wcarddomain));
+    if (range.first==range.second)
       continue;
 
-    for(ziter=range.first; ziter!=range.second; ++ziter) {
-      DNSRecord dr=*ziter;
+    for(ziter = range.first; ziter != range.second; ++ziter) {
+      DNSRecord dr = *ziter;
       // if we hit a CNAME, just answer that - rest of recursor will do the needful & follow
-      if(dr.d_type == qtype.getCode() || qtype.getCode() == QType::ANY || dr.d_type == QType::CNAME) {
+      if(dr.d_type == qtype || qtype == QType::ANY || dr.d_type == QType::CNAME) {
         dr.d_name = qname;
-        dr.d_place=DNSResourceRecord::ANSWER;
-        ret.push_back(dr);
+        dr.d_place = DNSResourceRecord::ANSWER;
+        records.push_back(dr);
       }
     }
-    LOG(prefix<<qname<<": in '"<<authdomain<<"', had wildcard match on '*."<<wcarddomain<<"'"<<endl);
-    res=RCode::NoError;
-    return true;
+
+    if (records.empty()) {
+      addSOA(records);
+    }
+
+    // cerr<<qname<<": in '"<<getName()<<"', had wildcard match on '*."<<wcarddomain<<"'"<<endl;
+    return result;
   }
 
+  /* Nothing for this name, no wildcard, let's see if there is some NS */
   DNSName nsdomain(qname);
-
-  while(nsdomain.chopOff() && nsdomain != iter->first) {
-    range=iter->second.d_records.equal_range(boost::make_tuple(nsdomain,QType::NS));
-    if(range.first==range.second)
+  while (nsdomain.chopOff() && nsdomain != getName()) {
+    range = d_records.equal_range(boost::make_tuple(nsdomain,QType::NS));
+    if(range.first == range.second)
       continue;
 
-    for(ziter=range.first; ziter!=range.second; ++ziter) {
-      DNSRecord dr=*ziter;
-      dr.d_place=DNSResourceRecord::AUTHORITY;
-      ret.push_back(dr);
+    for(ziter = range.first; ziter != range.second; ++ziter) {
+      DNSRecord dr = *ziter;
+      dr.d_place = DNSResourceRecord::AUTHORITY;
+      records.push_back(dr);
     }
   }
-  if(ret.empty()) {
-    LOG(prefix<<qname<<": no NS match in zone '"<<authdomain<<"' either, handing out SOA"<<endl);
-    ziter=iter->second.d_records.find(boost::make_tuple(authdomain, QType::SOA));
-    if(ziter!=iter->second.d_records.end()) {
-      DNSRecord dr=*ziter;
-      dr.d_place=DNSResourceRecord::AUTHORITY;
-      ret.push_back(dr);
-    }
-    else {
-      LOG(prefix<<qname<<": can't find SOA record '"<<authdomain<<"' in our zone!"<<endl);
-    }
-    res=RCode::NXDomain;
+
+  if(records.empty()) {
+    // cerr<<qname<<": no NS match in zone '"<<getName()<<"' either, handing out SOA"<<endl;
+    addSOA(records);
+    result = RCode::NXDomain;
   }
-  else
-    res=0;
 
+  return result;
+}
+
+bool SyncRes::doOOBResolve(const AuthDomain& domain, const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, int& res) const
+{
+  res = domain.getRecords(qname, qtype.getCode(), ret);
   return true;
 }
 
+bool SyncRes::doOOBResolve(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, int& res)
+{
+  string prefix;
+  if(doLog()) {
+    prefix=d_prefix;
+    prefix.append(depth, ' ');
+  }
+
+  DNSName authdomain(qname);
+  domainmap_t::const_iterator iter=getBestAuthZone(&authdomain);
+  if(iter==t_sstorage.domainmap->end() || !iter->second.isAuth()) {
+    LOG(prefix<<qname<<": auth storage has no zone for this query!"<<endl);
+    return false;
+  }
+
+  LOG(prefix<<qname<<": auth storage has data, zone='"<<authdomain<<"'"<<endl);
+  return doOOBResolve(iter->second, qname, qtype, ret, res);
+}
+
 void SyncRes::doEDNSDumpAndClose(int fd)
 {
   FILE* fp=fdopen(fd, "w");
@@ -332,7 +349,7 @@ void SyncRes::doEDNSDumpAndClose(int fd)
     return;
   }
   fprintf(fp,"IP Address\tMode\tMode last updated at\n");
-  for(const auto& eds : t_sstorage->ednsstatus) {
+  for(const auto& eds : t_sstorage.ednsstatus) {
     fprintf(fp, "%s\t%d\t%s", eds.first.toString().c_str(), (int)eds.second.mode, ctime(&eds.second.modeSetAt));
   }
 
@@ -347,7 +364,7 @@ uint64_t SyncRes::doDumpNSSpeeds(int fd)
   fprintf(fp, "; nsspeed dump from thread follows\n;\n");
   uint64_t count=0;
 
-  for(const auto& i : t_sstorage->nsSpeeds)
+  for(const auto& i : t_sstorage.nsSpeeds)
   {
     count++;
     fprintf(fp, "%s -> ", i.first.toString().c_str());
@@ -406,7 +423,7 @@ int SyncRes::asyncresolveWrapper(const ComboAddress& ip, bool ednsMANDATORY, con
   */
 
   SyncRes::EDNSStatus* ednsstatus;
-  ednsstatus = &t_sstorage->ednsstatus[ip]; // does this include port? YES
+  ednsstatus = &t_sstorage.ednsstatus[ip]; // does this include port? YES
 
   if(ednsstatus->modeSetAt && ednsstatus->modeSetAt + 3600 < d_now.tv_sec) {
     *ednsstatus=SyncRes::EDNSStatus();
@@ -415,7 +432,7 @@ int SyncRes::asyncresolveWrapper(const ComboAddress& ip, bool ednsMANDATORY, con
 
   SyncRes::EDNSStatus::EDNSMode& mode=ednsstatus->mode;
   SyncRes::EDNSStatus::EDNSMode oldmode = mode;
-  int EDNSLevel=0;
+  int EDNSLevel = 0;
   auto luaconfsLocal = g_luaconfs.getLocal();
   ResolveContext ctx;
 #ifdef HAVE_PROTOBUF
@@ -481,7 +498,7 @@ int SyncRes::asyncresolveWrapper(const ComboAddress& ip, bool ednsMANDATORY, con
  * \param beenthere
  * \return DNS RCODE or -1 (Error) or -2 (RPZ hit)
  */
-int SyncRes::doResolve(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, set<GetBestNSAnswer>& beenthere)
+int SyncRes::doResolve(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, set<GetBestNSAnswer>& beenthere, vState& state)
 {
   string prefix;
   if(doLog()) {
@@ -489,7 +506,9 @@ int SyncRes::doResolve(const DNSName &qname, const QType &qtype, vector<DNSRecor
     prefix.append(depth, ' ');
   }
 
-  LOG(prefix<<qname<<": Wants "<< (d_doDNSSEC ? "" : "NO ") << "DNSSEC processing in query for "<<qtype.getName()<<endl);
+  LOG(prefix<<qname<<": Wants "<< (d_doDNSSEC ? "" : "NO ") << "DNSSEC processing, "<<(d_requireAuthData ? "" : "NO ")<<"auth data in query for "<<qtype.getName()<<endl);
+
+  state = Indeterminate;
 
   if(s_maxdepth && depth > s_maxdepth)
     throw ImmediateServFailException("More than "+std::to_string(s_maxdepth)+" (max-recursion-depth) levels of recursion needed while resolving "+qname.toLogString());
@@ -497,20 +516,20 @@ int SyncRes::doResolve(const DNSName &qname, const QType &qtype, vector<DNSRecor
   int res=0;
 
   // This is a difficult way of expressing "this is a normal query", i.e. not getRootNS.
-  if(!(d_nocache && qtype.getCode()==QType::NS && qname.isRoot())) {
+  if(!(d_updatingRootNS && qtype.getCode()==QType::NS && qname.isRoot())) {
     if(d_cacheonly) { // very limited OOB support
       LWResult lwr;
       LOG(prefix<<qname<<": Recursion not requested for '"<<qname<<"|"<<qtype.getName()<<"', peeking at auth/forward zones"<<endl);
       DNSName authname(qname);
       domainmap_t::const_iterator iter=getBestAuthZone(&authname);
-      if(iter != t_sstorage->domainmap->end()) {
-        const vector<ComboAddress>& servers = iter->second.d_servers;
-        if(servers.empty()) {
+      if(iter != t_sstorage.domainmap->end()) {
+        if(iter->second.isAuth()) {
           ret.clear();
           d_wasOutOfBand = doOOBResolve(qname, qtype, ret, depth, res);
           return res;
         }
         else {
+          const vector<ComboAddress>& servers = iter->second.d_servers;
           const ComboAddress remoteIP = servers.front();
           LOG(prefix<<qname<<": forwarding query to hardcoded nameserver '"<< remoteIP.toStringWithPort()<<"' for zone '"<<authname<<"'"<<endl);
 
@@ -531,10 +550,10 @@ int SyncRes::doResolve(const DNSName &qname, const QType &qtype, vector<DNSRecor
       }
     }
 
-    if(!d_skipCNAMECheck && doCNAMECacheCheck(qname,qtype,ret,depth,res)) // will reroute us if needed
+    if(!d_skipCNAMECheck && doCNAMECacheCheck(qname,qtype,ret,depth,res,state)) // will reroute us if needed
       return res;
 
-    if(doCacheCheck(qname,qtype,ret,depth,res)) // we done
+    if(doCacheCheck(qname,qtype,ret,depth,res,state)) // we done
       return res;
   }
 
@@ -549,17 +568,22 @@ int SyncRes::doResolve(const DNSName &qname, const QType &qtype, vector<DNSRecor
   NsSet nsset;
   bool flawedNSSet=false;
 
+  computeZoneCuts(qname, g_rootdnsname, depth);
+
   // the two retries allow getBestNSNamesFromCache&co to reprime the root
   // hints, in case they ever go missing
   for(int tries=0;tries<2 && nsset.empty();++tries) {
     subdomain=getBestNSNamesFromCache(subdomain, qtype, nsset, &flawedNSSet, depth, beenthere); //  pass beenthere to both occasions
   }
 
-  if(!(res=doResolveAt(nsset, subdomain, flawedNSSet, qname, qtype, ret, depth, beenthere)))
+  state = getValidationStatus(subdomain);
+
+  LOG(prefix<<qname<<": initial validation status for "<<qname<<" inherited from "<<subdomain<<" is "<<vStates[state]<<endl);
+
+  if(!(res=doResolveAt(nsset, subdomain, flawedNSSet, qname, qtype, ret, depth, beenthere, state)))
     return 0;
 
   LOG(prefix<<qname<<": failed (res="<<res<<")"<<endl);
-  ;
 
   if (res == -2)
     return res;
@@ -586,6 +610,8 @@ vector<ComboAddress> SyncRes::getAddrs(const DNSName &qname, unsigned int depth,
   ret_t ret;
 
   QType type;
+  bool oldRequireAuthData = d_requireAuthData;
+  d_requireAuthData = false;
 
   for(int j=1; j<2+s_doIPv6; j++)
   {
@@ -602,7 +628,8 @@ vector<ComboAddress> SyncRes::getAddrs(const DNSName &qname, unsigned int depth,
         break;
     }
 
-    if(!doResolve(qname, type, res,depth+1, beenthere) && !res.empty()) {  // this consults cache, OR goes out
+    vState newState = Indeterminate;
+    if(!doResolve(qname, type, res,depth+1, beenthere, newState) && !res.empty()) {  // this consults cache, OR goes out
       for(res_t::const_iterator i=res.begin(); i!= res.end(); ++i) {
         if(i->d_type == QType::A || i->d_type == QType::AAAA) {
          if(auto rec = std::dynamic_pointer_cast<ARecordContent>(i->d_content))
@@ -616,7 +643,7 @@ vector<ComboAddress> SyncRes::getAddrs(const DNSName &qname, unsigned int depth,
     if(done) {
       if(j==1 && s_doIPv6) { // we got an A record, see if we have some AAAA lying around
        vector<DNSRecord> cset;
-       if(t_RC->get(d_now.tv_sec, qname, QType(QType::AAAA), &cset, d_requestor) > 0) {
+       if(t_RC->get(d_now.tv_sec, qname, QType(QType::AAAA), false, &cset, d_incomingECSFound ? d_incomingECSNetwork : d_requestor) > 0) {
          for(auto k=cset.cbegin();k!=cset.cend();++k) {
            if(k->d_ttl > (unsigned int)d_now.tv_sec ) {
              if (auto drc = std::dynamic_pointer_cast<AAAARecordContent>(k->d_content)) {
@@ -631,13 +658,15 @@ vector<ComboAddress> SyncRes::getAddrs(const DNSName &qname, unsigned int depth,
     }
   }
 
+  d_requireAuthData = oldRequireAuthData;
+
   if(ret.size() > 1) {
     random_shuffle(ret.begin(), ret.end(), dns_random);
 
     // move 'best' address for this nameserver name up front
-    nsspeeds_t::iterator best = t_sstorage->nsSpeeds.find(qname);
+    nsspeeds_t::iterator best = t_sstorage.nsSpeeds.find(qname);
 
-    if(best != t_sstorage->nsSpeeds.end())
+    if(best != t_sstorage.nsSpeeds.end())
       for(ret_t::iterator i=ret.begin(); i != ret.end(); ++i) {
         if(*i==best->second.d_best) {  // got the fastest one
           if(i!=ret.begin()) {
@@ -667,7 +696,8 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType& qtype, vecto
     LOG(prefix<<qname<<": Checking if we have NS in cache for '"<<subdomain<<"'"<<endl);
     vector<DNSRecord> ns;
     *flawedNSSet = false;
-    if(t_RC->get(d_now.tv_sec, subdomain, QType(QType::NS), &ns, d_requestor) > 0) {
+
+    if(t_RC->get(d_now.tv_sec, subdomain, QType(QType::NS), false, &ns, d_incomingECSFound ? d_incomingECSNetwork : d_requestor) > 0) {
       for(auto k=ns.cbegin();k!=ns.cend(); ++k) {
         if(k->d_ttl > (unsigned int)d_now.tv_sec ) {
           vector<DNSRecord> aset;
@@ -675,7 +705,7 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType& qtype, vecto
           const DNSRecord& dr=*k;
          auto nrr = getRR<NSRecordContent>(dr);
           if(nrr && (!nrr->getNS().isPartOf(subdomain) || t_RC->get(d_now.tv_sec, nrr->getNS(), s_doIPv6 ? QType(QType::ADDR) : QType(QType::A),
-                                                                    doLog() ? &aset : 0, d_requestor) > 5)) {
+                                                                    false, doLog() ? &aset : 0, d_incomingECSFound ? d_incomingECSNetwork : d_requestor) > 5)) {
             bestns.push_back(dr);
             LOG(prefix<<qname<<": NS (with ip, or non-glue) in cache for '"<<subdomain<<"' -> '"<<nrr->getNS()<<"'"<<endl);
             LOG(prefix<<qname<<": within bailiwick: "<< nrr->getNS().isPartOf(subdomain));
@@ -692,6 +722,7 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType& qtype, vecto
           }
         }
       }
+
       if(!bestns.empty()) {
         GetBestNSAnswer answer;
         answer.qname=qname;
@@ -718,22 +749,25 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType& qtype, vecto
       }
     }
     LOG(prefix<<qname<<": no valid/useful NS in cache for '"<<subdomain<<"'"<<endl);
-    ;
+
     if(subdomain.isRoot() && !brokeloop) {
       // We lost the root NS records
       primeHints();
       LOG(prefix<<qname<<": reprimed the root"<<endl);
-      getRootNS(d_now, d_asyncResolve);
+      /* let's prevent an infinite loop */
+      if (!d_updatingRootNS) {
+        getRootNS(d_now, d_asyncResolve);
+      }
     }
-  }while(subdomain.chopOff());
+  } while(subdomain.chopOff());
 }
 
 SyncRes::domainmap_t::const_iterator SyncRes::getBestAuthZone(DNSName* qname) const
 {
   SyncRes::domainmap_t::const_iterator ret;
   do {
-    ret=t_sstorage->domainmap->find(*qname);
-    if(ret!=t_sstorage->domainmap->end())
+    ret=t_sstorage.domainmap->find(*qname);
+    if(ret!=t_sstorage.domainmap->end())
       break;
   }while(qname->chopOff());
   return ret;
@@ -746,15 +780,16 @@ DNSName SyncRes::getBestNSNamesFromCache(const DNSName &qname, const QType& qtyp
   DNSName authdomain(qname);
 
   domainmap_t::const_iterator iter=getBestAuthZone(&authdomain);
-  if(iter!=t_sstorage->domainmap->end()) {
-    if( iter->second.d_servers.empty() )
+  if(iter!=t_sstorage.domainmap->end()) {
+    if( iter->second.isAuth() )
       // this gets picked up in doResolveAt, the empty DNSName, combined with the
       // empty vector means 'we are auth for this zone'
       nsset.insert({DNSName(), {{}, false}});
     else {
       // Again, picked up in doResolveAt. An empty DNSName, combined with a
       // non-empty vector of ComboAddresses means 'this is a forwarded domain'
-      nsset.insert({DNSName(), {iter->second.d_servers, iter->second.d_rdForward}});
+      // This is actually picked up in retrieveAddressesForNS called from doResolveAt.
+      nsset.insert({DNSName(), {iter->second.d_servers, iter->second.shouldRecurse() }});
     }
     return authdomain;
   }
@@ -771,7 +806,7 @@ DNSName SyncRes::getBestNSNamesFromCache(const DNSName &qname, const QType& qtyp
   return subdomain;
 }
 
-bool SyncRes::doCNAMECacheCheck(const DNSName &qname, const QType &qtype, vector<DNSRecord>& ret, unsigned int depth, int &res)
+bool SyncRes::doCNAMECacheCheck(const DNSName &qname, const QType &qtype, vector<DNSRecord>& ret, unsigned int depth, int &res, vState& state)
 {
   string prefix;
   if(doLog()) {
@@ -788,32 +823,64 @@ bool SyncRes::doCNAMECacheCheck(const DNSName &qname, const QType &qtype, vector
   LOG(prefix<<qname<<": Looking for CNAME cache hit of '"<<qname<<"|CNAME"<<"'"<<endl);
   vector<DNSRecord> cset;
   vector<std::shared_ptr<RRSIGRecordContent>> signatures;
-  if(t_RC->get(d_now.tv_sec, qname,QType(QType::CNAME), &cset, d_requestor, &signatures) > 0) {
+  vector<std::shared_ptr<DNSRecord>> authorityRecs;
+  bool wasAuth;
+  if(t_RC->get(d_now.tv_sec, qname, QType(QType::CNAME), d_requireAuthData, &cset, d_incomingECSFound ? d_incomingECSNetwork : d_requestor, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &state, &wasAuth) > 0) {
 
     for(auto j=cset.cbegin() ; j != cset.cend() ; ++j) {
       if(j->d_ttl>(unsigned int) d_now.tv_sec) {
-        LOG(prefix<<qname<<": Found cache CNAME hit for '"<< qname << "|CNAME" <<"' to '"<<j->d_content->getZoneRepresentation()<<"'"<<endl);
+
+        if (d_DNSSECValidationRequested && wasAuth && state == Indeterminate && d_requireAuthData) {
+          /* This means we couldn't figure out the state when this entry was cached,
+             most likely because we hadn't computed the zone cuts yet. */
+          /* make sure they are computed before validating */
+          computeZoneCuts(qname, g_rootdnsname, depth);
+
+          vState recordState = getValidationStatus(qname);
+          if (recordState == Secure) {
+            LOG(prefix<<qname<<": got Indeterminate state from the CNAME cache, validating.."<<endl);
+            state = SyncRes::validateRecordsWithSigs(depth, qname, QType(QType::CNAME), qname, cset, signatures);
+            if (state != Indeterminate) {
+              LOG(prefix<<qname<<": got Indeterminate state from the CNAME cache, new validation result is "<<vStates[state]<<endl);
+              t_RC->updateValidationStatus(d_now.tv_sec, qname, QType(QType::CNAME), d_incomingECSFound ? d_incomingECSNetwork : d_requestor, d_requireAuthData, state);
+            }
+          }
+        }
+
+        LOG(prefix<<qname<<": Found cache CNAME hit for '"<< qname << "|CNAME" <<"' to '"<<j->d_content->getZoneRepresentation()<<"', validation state is "<<vStates[state]<<endl);
+
         DNSRecord dr=*j;
         dr.d_ttl-=d_now.tv_sec;
         ret.push_back(dr);
 
-       for(const auto& signature : signatures) {
-         DNSRecord sigdr;
-         sigdr.d_type=QType::RRSIG;
-         sigdr.d_name=qname;
-         sigdr.d_ttl=j->d_ttl - d_now.tv_sec;
-         sigdr.d_content=signature;
-         sigdr.d_place=DNSResourceRecord::ANSWER;
-         sigdr.d_class=1;
-         ret.push_back(sigdr);
-       }
+        for(const auto& signature : signatures) {
+          DNSRecord sigdr;
+          sigdr.d_type=QType::RRSIG;
+          sigdr.d_name=qname;
+          sigdr.d_ttl=j->d_ttl - d_now.tv_sec;
+          sigdr.d_content=signature;
+          sigdr.d_place=DNSResourceRecord::ANSWER;
+          sigdr.d_class=QClass::IN;
+          ret.push_back(sigdr);
+        }
+
+        for(const auto& rec : authorityRecs) {
+          DNSRecord dr(*rec);
+          dr.d_ttl=j->d_ttl - d_now.tv_sec;
+          ret.push_back(dr);
+        }
 
         if(qtype != QType::CNAME) { // perhaps they really wanted a CNAME!
           set<GetBestNSAnswer>beenthere;
-          res=doResolve(std::dynamic_pointer_cast<CNAMERecordContent>(j->d_content)->getTarget(), qtype, ret, depth+1, beenthere);
+
+          vState cnameState = Indeterminate;
+          res=doResolve(std::dynamic_pointer_cast<CNAMERecordContent>(j->d_content)->getTarget(), qtype, ret, depth+1, beenthere, cnameState);
+          LOG(prefix<<qname<<": updating validation state for response to "<<qname<<" from "<<vStates[state]<<" with the state from the CNAME quest: "<<vStates[cnameState]<<endl);
+          updateValidationState(state, cnameState);
         }
         else
           res=0;
+
         return true;
       }
     }
@@ -838,7 +905,7 @@ static void addTTLModifiedRecords(const vector<DNSRecord>& records, const uint32
 }
 
 
-bool SyncRes::doCacheCheck(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, int &res)
+bool SyncRes::doCacheCheck(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, int &res, vState& state)
 {
   bool giveNegative=false;
 
@@ -855,10 +922,11 @@ bool SyncRes::doCacheCheck(const DNSName &qname, const QType &qtype, vector<DNSR
   //  cout<<"Lookup for '"<<qname<<"|"<<qtype.getName()<<"' -> "<<getLastLabel(qname)<<endl;
 
   DNSName authname(qname);
+  vState cachedState;
   bool wasForwardedOrAuth = false;
   bool wasAuth = false;
   domainmap_t::const_iterator iter=getBestAuthZone(&authname);
-  if(iter != t_sstorage->domainmap->end()) {
+  if(iter != t_sstorage.domainmap->end()) {
     wasForwardedOrAuth = true;
     const vector<ComboAddress>& servers = iter->second.d_servers;
     if(servers.empty()) {
@@ -868,19 +936,21 @@ bool SyncRes::doCacheCheck(const DNSName &qname, const QType &qtype, vector<DNSR
   NegCache::NegCacheEntry ne;
 
   if(s_rootNXTrust &&
-     t_sstorage->negcache.getRootNXTrust(qname, d_now, ne) &&
+     t_sstorage.negcache.getRootNXTrust(qname, d_now, ne) &&
       ne.d_auth.isRoot() &&
       !(wasForwardedOrAuth && !authname.isRoot())) { // when forwarding, the root may only neg-cache if it was forwarded to.
     sttl = ne.d_ttd - d_now.tv_sec;
     LOG(prefix<<qname<<": Entire name '"<<qname<<"', is negatively cached via '"<<ne.d_auth<<"' & '"<<ne.d_name<<"' for another "<<sttl<<" seconds"<<endl);
     res = RCode::NXDomain;
     giveNegative = true;
+    cachedState = ne.d_validationState;
   }
-  else if (t_sstorage->negcache.get(qname, qtype, d_now, ne) &&
+  else if (t_sstorage.negcache.get(qname, qtype, d_now, ne) &&
            !(wasForwardedOrAuth && ne.d_auth != authname)) { // Only the authname nameserver can neg cache entries
     res = 0;
     sttl = ne.d_ttd - d_now.tv_sec;
     giveNegative = true;
+    cachedState = ne.d_validationState;
     if(ne.d_qtype.getCode()) {
       LOG(prefix<<qname<<": "<<qtype.getName()<<" is negatively cached via '"<<ne.d_auth<<"' for another "<<sttl<<" seconds"<<endl);
       res = RCode::NoError;
@@ -900,15 +970,41 @@ bool SyncRes::doCacheCheck(const DNSName &qname, const QType &qtype, vector<DNSR
     addTTLModifiedRecords(ne.authoritySOA.records, sttl, ret);
     if(d_doDNSSEC)
       addTTLModifiedRecords(ne.authoritySOA.signatures, sttl, ret);
+
+    LOG(prefix<<qname<<": updating validation state with negative cache content for "<<qname<<" to "<<vStates[cachedState]<<endl);
+    state = cachedState;
     return true;
   }
 
   vector<DNSRecord> cset;
   bool found=false, expired=false;
   vector<std::shared_ptr<RRSIGRecordContent>> signatures;
+  vector<std::shared_ptr<DNSRecord>> authorityRecs;
   uint32_t ttl=0;
-  if(t_RC->get(d_now.tv_sec, sqname, sqt, &cset, d_requestor, d_doDNSSEC ? &signatures : 0) > 0) {
+  bool wasCachedAuth;
+  if(t_RC->get(d_now.tv_sec, sqname, sqt, d_requireAuthData, &cset, d_incomingECSFound ? d_incomingECSNetwork : d_requestor, d_doDNSSEC ? &signatures : nullptr, d_doDNSSEC ? &authorityRecs : nullptr, &d_wasVariable, &cachedState, &wasCachedAuth) > 0) {
+
     LOG(prefix<<sqname<<": Found cache hit for "<<sqt.getName()<<": ");
+
+    if (d_DNSSECValidationRequested && sqt != QType::DNSKEY && wasCachedAuth && cachedState == Indeterminate && d_requireAuthData) {
+
+      /* This means we couldn't figure out the state when this entry was cached,
+         most likely because we hadn't computed the zone cuts yet. */
+      /* make sure they are computed before validating */
+      computeZoneCuts(sqname, g_rootdnsname, depth);
+
+      vState recordState = getValidationStatus(qname);
+      if (recordState == Secure) {
+        LOG(prefix<<sqname<<": got Indeterminate state from the cache, validating.."<<endl);
+        cachedState = SyncRes::validateRecordsWithSigs(depth, sqname, sqt, sqname, cset, signatures);
+
+        if (cachedState != Indeterminate) {
+          LOG(prefix<<qname<<": got Indeterminate state from the cache, validation result is "<<vStates[cachedState]<<endl);
+          t_RC->updateValidationStatus(d_now.tv_sec, sqname, sqt, d_incomingECSFound ? d_incomingECSNetwork : d_requestor, d_requireAuthData, cachedState);
+        }
+      }
+    }
+
     for(auto j=cset.cbegin() ; j != cset.cend() ; ++j) {
       LOG(j->d_content->getZoneRepresentation());
       if(j->d_ttl>(unsigned int) d_now.tv_sec) {
@@ -931,15 +1027,23 @@ bool SyncRes::doCacheCheck(const DNSName &qname, const QType &qtype, vector<DNSR
       dr.d_ttl=ttl; 
       dr.d_content=signature;
       dr.d_place = DNSResourceRecord::ANSWER;
-      dr.d_class=1;
+      dr.d_class=QClass::IN;
       ret.push_back(dr);
     }
-  
+
+    for(const auto& rec : authorityRecs) {
+      DNSRecord dr(*rec);
+      dr.d_ttl=ttl;
+      ret.push_back(dr);
+    }
+
     LOG(endl);
     if(found && !expired) {
-      if(!giveNegative)
+      if (!giveNegative)
         res=0;
       d_wasOutOfBand = wasAuth;
+      LOG(prefix<<qname<<": updating validation state with cache content for "<<qname<<" to "<<vStates[cachedState]<<endl);
+      state = cachedState;
       return true;
     }
     else
@@ -975,7 +1079,7 @@ inline vector<DNSName> SyncRes::shuffleInSpeedOrder(NsSet &tnameservers, const s
 
   for(const auto& val: rnameservers) {
     double speed;
-    speed=t_sstorage->nsSpeeds[val].get(&d_now);
+    speed=t_sstorage.nsSpeeds[val].get(&d_now);
     speeds[val]=speed;
   }
   random_shuffle(rnameservers.begin(),rnameservers.end(), dns_random);
@@ -1005,13 +1109,14 @@ static bool magicAddrMatch(const QType& query, const QType& answer)
   return answer.getCode() == QType::A || answer.getCode() == QType::AAAA;
 }
 
+static const set<uint16_t> nsecTypes = {QType::NSEC, QType::NSEC3};
+
 /* Fills the authoritySOA and DNSSECRecords fields from ne with those found in the records
  *
  * \param records The records to parse for the authority SOA and NSEC(3) records
  * \param ne      The NegCacheEntry to be filled out (will not be cleared, only appended to
  */
 static void harvestNXRecords(const vector<DNSRecord>& records, NegCache::NegCacheEntry& ne) {
-  static const set<uint16_t> nsecTypes = {QType::NSEC, QType::NSEC3};
   for(const auto& rec : records) {
     if(rec.d_place != DNSResourceRecord::AUTHORITY)
       // RFC 4035 section 3.1.3. indicates that NSEC records MUST be placed in
@@ -1043,6 +1148,23 @@ static void harvestNXRecords(const vector<DNSRecord>& records, NegCache::NegCach
   }
 }
 
+static cspmap_t harvestCSPFromNE(const NegCache::NegCacheEntry& ne)
+{
+  cspmap_t cspmap;
+  for(const auto& rec : ne.DNSSECRecords.signatures) {
+    if(rec.d_type == QType::RRSIG) {
+      auto rrc = getRR<RRSIGRecordContent>(rec);
+      if (rrc) {
+        cspmap[{rec.d_name,rrc->d_type}].signatures.push_back(rrc);
+      }
+    }
+  }
+  for(const auto& rec : ne.DNSSECRecords.records) {
+    cspmap[{rec.d_name, rec.d_type}].records.push_back(rec.d_content);
+  }
+  return cspmap;
+}
+
 // TODO remove after processRecords is fixed!
 // Adds the RRSIG for the SOA and the NSEC(3) + RRSIGs to ret
 static void addNXNSECS(vector<DNSRecord>&ret, const vector<DNSRecord>& records)
@@ -1115,13 +1237,13 @@ vector<ComboAddress> SyncRes::retrieveAddressesForNS(const std::string& prefix,
 
 bool SyncRes::throttledOrBlocked(const std::string& prefix, const ComboAddress& remoteIP, const DNSName& qname, const QType& qtype, bool pierceDontQuery)
 {
-  if(t_sstorage->throttle.shouldThrottle(d_now.tv_sec, boost::make_tuple(remoteIP, "", 0))) {
+  if(t_sstorage.throttle.shouldThrottle(d_now.tv_sec, boost::make_tuple(remoteIP, "", 0))) {
     LOG(prefix<<qname<<": server throttled "<<endl);
     s_throttledqueries++; d_throttledqueries++;
     return true;
   }
-  else if(t_sstorage->throttle.shouldThrottle(d_now.tv_sec, boost::make_tuple(remoteIP, qname, qtype.getCode()))) {
-    LOG(prefix<<qname<<": query throttled "<<endl);
+  else if(t_sstorage.throttle.shouldThrottle(d_now.tv_sec, boost::make_tuple(remoteIP, qname, qtype.getCode()))) {
+    LOG(prefix<<qname<<": query throttled "<<remoteIP.toString()<<", "<<qname<<"; "<<qtype.getName()<<endl);
     s_throttledqueries++; d_throttledqueries++;
     return true;
   }
@@ -1133,125 +1255,651 @@ bool SyncRes::throttledOrBlocked(const std::string& prefix, const ComboAddress&
   return false;
 }
 
-RCode::rcodes_ SyncRes::updateCacheFromRecords(const std::string& prefix, LWResult& lwr, const DNSName& qname, const DNSName& auth, NsSet& nameservers, const DNSName& tns, const boost::optional<Netmask> ednsmask)
+bool SyncRes::validationEnabled() const
 {
-  struct CachePair
-  {
-    vector<DNSRecord> records;
-    vector<shared_ptr<RRSIGRecordContent>> signatures;
-  };
-  struct CacheKey
-  {
-    DNSName name;
-    uint16_t type;
-    DNSResourceRecord::Place place;
-    bool operator<(const CacheKey& rhs) const {
-      return tie(name, type) < tie(rhs.name, rhs.type);
-    }
-  };
-  typedef map<CacheKey, CachePair> tcache_t;
-  tcache_t tcache;
+  return g_dnssecmode != DNSSECMode::Off && g_dnssecmode != DNSSECMode::ProcessNoValidate;
+}
 
-  for(const auto& rec : lwr.d_records) {
-    if(rec.d_type == QType::RRSIG) {
-      auto rrsig = getRR<RRSIGRecordContent>(rec);
-      if (rrsig) {
-        //         cerr<<"Got an RRSIG for "<<DNSRecordContent::NumberToType(rrsig->d_type)<<" with name '"<<rec.d_name<<"'"<<endl;
-        tcache[{rec.d_name, rrsig->d_type, rec.d_place}].signatures.push_back(rrsig);
+uint32_t SyncRes::computeLowestTTD(const std::vector<DNSRecord>& records, const std::vector<std::shared_ptr<RRSIGRecordContent> >& signatures, uint32_t signaturesTTL) const
+{
+  uint32_t lowestTTD = std::numeric_limits<uint32_t>::max();
+  for(const auto& record : records)
+    lowestTTD = min(lowestTTD, record.d_ttl);
+
+  /* even if it was not requested for that request (Process, and neither AD nor DO set),
+     it might be requested at a later time so we need to be careful with the TTL. */
+  if (validationEnabled() && !signatures.empty()) {
+    /* if we are validating, we don't want to cache records after their signatures
+       expires. */
+    /* records TTL are now TTD, let's add 'now' to the signatures lowest TTL */
+    lowestTTD = min(lowestTTD, static_cast<uint32_t>(signaturesTTL + d_now.tv_sec));
+
+    for(const auto& sig : signatures) {
+      if (sig->d_siginception <= d_now.tv_sec && sig->d_sigexpire > d_now.tv_sec) {
+        // we don't decerement d_sigexpire by 'now' because we actually want a TTD, not a TTL */
+        lowestTTD = min(lowestTTD, static_cast<uint32_t>(sig->d_sigexpire));
       }
     }
   }
 
-  // reap all answers from this packet that are acceptable
-  for(auto& rec : lwr.d_records) {
-    if(rec.d_type == QType::OPT) {
-      LOG(prefix<<qname<<": OPT answer '"<<rec.d_name<<"' from '"<<auth<<"' nameservers" <<endl);
-      continue;
-    }
-    LOG(prefix<<qname<<": accept answer '"<<rec.d_name<<"|"<<DNSRecordContent::NumberToType(rec.d_type)<<"|"<<rec.d_content->getZoneRepresentation()<<"' from '"<<auth<<"' nameservers? "<<(int)rec.d_place<<" ");
-    if(rec.d_type == QType::ANY) {
-      LOG("NO! - we don't accept 'ANY' data"<<endl);
-      continue;
-    }
+  return lowestTTD;
+}
 
-    if(rec.d_name.isPartOf(auth)) {
-      if(rec.d_type == QType::RRSIG) {
-        LOG("RRSIG - separate"<<endl);
-      }
-      else if(lwr.d_aabit && lwr.d_rcode==RCode::NoError && rec.d_place==DNSResourceRecord::ANSWER && (rec.d_type != QType::DNSKEY || rec.d_name != auth) && s_delegationOnly.count(auth)) {
-        LOG("NO! Is from delegation-only zone"<<endl);
-        s_nodelegated++;
-        return RCode::NXDomain;
-      }
-      else {
-        bool haveLogged = false;
-        if (!t_sstorage->domainmap->empty()) {
-          // Check if we are authoritative for a zone in this answer
-          DNSName tmp_qname(rec.d_name);
-          auto auth_domain_iter=getBestAuthZone(&tmp_qname);
-          if(auth_domain_iter!=t_sstorage->domainmap->end() &&
-             auth.countLabels() <= auth_domain_iter->first.countLabels()) {
-            if (auth_domain_iter->first != auth) {
-              LOG("NO! - we are authoritative for the zone "<<auth_domain_iter->first<<endl);
-              continue;
-            } else {
-              LOG("YES! - This answer was ");
-              if (nameservers[tns].first.empty()) {
-                LOG("retrieved from the local auth store.");
-              } else {
-                LOG("received from a server we forward to.");
-              }
-              haveLogged = true;
-              LOG(endl);
-            }
-          }
-        }
-        if (!haveLogged) {
-          LOG("YES!"<<endl);
-        }
+void SyncRes::updateValidationState(vState& state, const vState stateUpdate)
+{
+  LOG(d_prefix<<"validation state was "<<std::string(vStates[state])<<", state update is "<<std::string(vStates[stateUpdate])<<endl);
 
-        rec.d_ttl=min(s_maxcachettl, rec.d_ttl);
+  if (stateUpdate == TA) {
+    state = Secure;
+  }
+  else if (stateUpdate == NTA) {
+    state = Insecure;
+  }
+  else if (stateUpdate == Bogus) {
+    state = Bogus;
+  }
+  else if (state == Indeterminate) {
+    state = stateUpdate;
+  }
+  else if (stateUpdate == Insecure) {
+    if (state != Bogus) {
+      state = Insecure;
+    }
+  }
+  LOG(d_prefix<<" validation state is now "<<std::string(vStates[state])<<endl);
+}
 
-        DNSRecord dr(rec);
-        dr.d_place=DNSResourceRecord::ANSWER;
+vState SyncRes::getTA(const DNSName& zone, dsmap_t& ds)
+{
+  auto luaLocal = g_luaconfs.getLocal();
 
-        dr.d_ttl += d_now.tv_sec;
-        tcache[{rec.d_name,rec.d_type,rec.d_place}].records.push_back(dr);
-      }
-    }
-    else
-      LOG("NO!"<<endl);
+  if (luaLocal->dsAnchors.empty()) {
+    /* We have no TA, everything is insecure */
+    return Insecure;
   }
 
-  // supplant
-  for(tcache_t::iterator i=tcache.begin();i!=tcache.end();++i) {
-    if(i->second.records.size() > 1) {  // need to group the ttl to be the minimum of the RRSET (RFC 2181, 5.2)
-      uint32_t lowestTTL=std::numeric_limits<uint32_t>::max();
-      for(const auto& record : i->second.records)
-        lowestTTL=min(lowestTTL, record.d_ttl);
+  std::string reason;
+  if (haveNegativeTrustAnchor(luaLocal->negAnchors, zone, reason)) {
+    LOG(d_prefix<<": got NTA for "<<zone<<endl);
+    return NTA;
+  }
 
-      for(auto& record : i->second.records)
-        *const_cast<uint32_t*>(&record.d_ttl)=lowestTTL; // boom
-    }
+  if (getTrustAnchor(luaLocal->dsAnchors, zone, ds)) {
+    LOG(d_prefix<<": got TA for "<<zone<<endl);
+    return TA;
+  }
 
-//             cout<<"Have "<<i->second.records.size()<<" records and "<<i->second.signatures.size()<<" signatures for "<<i->first.name;
-//             cout<<'|'<<DNSRecordContent::NumberToType(i->first.type)<<endl;
-    if(i->second.records.empty()) // this happens when we did store signatures, but passed on the records themselves
-      continue;
-    t_RC->replace(d_now.tv_sec, i->first.name, QType(i->first.type), i->second.records, i->second.signatures, lwr.d_aabit, i->first.place == DNSResourceRecord::ANSWER ? ednsmask : boost::optional<Netmask>());
-    if(i->first.place == DNSResourceRecord::ANSWER && ednsmask)
-      d_wasVariable=true;
+  if (zone.isRoot()) {
+    /* No TA for the root */
+    return Insecure;
   }
 
-  return RCode::NoError;
+  return Indeterminate;
 }
 
-bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, const QType& qtype, const DNSName& auth, LWResult& lwr, const bool sendRDQuery, vector<DNSRecord>& ret, set<DNSName>& nsset, DNSName& newtarget, DNSName& newauth, bool& realreferral, bool& negindic, bool& sawDS)
+static size_t countSupportedDS(const dsmap_t& dsmap)
 {
-  bool done = false;
+  size_t count = 0;
 
-  for(auto& rec : lwr.d_records) {
-    if (rec.d_type!=QType::OPT && rec.d_class!=QClass::IN)
+  for (const auto& ds : dsmap) {
+    if (isSupportedDS(ds)) {
+      count++;
+    }
+  }
+
+  return count;
+}
+
+vState SyncRes::getDSRecords(const DNSName& zone, dsmap_t& ds, bool taOnly, unsigned int depth)
+{
+  vState result = getTA(zone, ds);
+
+  if (result != Indeterminate || taOnly) {
+    if ((result == Secure || result == TA) && countSupportedDS(ds) == 0) {
+      ds.clear();
+      result = Insecure;
+    }
+
+    return result;
+  }
+
+  bool oldSkipCNAME = d_skipCNAMECheck;
+  bool oldRequireAuthData = d_requireAuthData;
+  d_skipCNAMECheck = true;
+  d_requireAuthData = false;
+
+  std::set<GetBestNSAnswer> beenthere;
+  std::vector<DNSRecord> dsrecords;
+
+  vState state = Indeterminate;
+  int rcode = doResolve(zone, QType(QType::DS), dsrecords, depth + 1, beenthere, state);
+  d_skipCNAMECheck = oldSkipCNAME;
+  d_requireAuthData = oldRequireAuthData;
+
+  if (rcode == RCode::NoError || rcode == RCode::NXDomain) {
+    if (state == Secure) {
+      for (const auto& record : dsrecords) {
+        if (record.d_type == QType::DS) {
+          const auto dscontent = getRR<DSRecordContent>(record);
+          if (dscontent && isSupportedDS(*dscontent)) {
+            ds.insert(*dscontent);
+          }
+        }
+      }
+
+      if (ds.empty()) {
+        return Insecure;
+      }
+    }
+
+    return state;
+  }
+
+  LOG(d_prefix<<": returning Bogus state from "<<__func__<<"("<<zone<<")"<<endl);
+  return Bogus;
+}
+
+bool SyncRes::haveExactValidationStatus(const DNSName& domain)
+{
+  if (!d_DNSSECValidationRequested) {
+    return false;
+  }
+  const auto& it = d_cutStates.find(domain);
+  if (it != d_cutStates.cend()) {
+    return true;
+  }
+  return false;
+}
+
+vState SyncRes::getValidationStatus(const DNSName& subdomain)
+{
+  vState result = Indeterminate;
+
+  if (!d_DNSSECValidationRequested) {
+    return result;
+  }
+  DNSName name(subdomain);
+  do {
+    const auto& it = d_cutStates.find(name);
+    if (it != d_cutStates.cend()) {
+      LOG(d_prefix<<": got status "<<vStates[it->second]<<" for name "<<subdomain<<" (from "<<name<<")"<<endl);
+      return it->second;
+    }
+  }
+  while (name.chopOff());
+
+  return result;
+}
+
+void SyncRes::computeZoneCuts(const DNSName& begin, const DNSName& end, unsigned int depth)
+{
+  if(!begin.isPartOf(end)) {
+    LOG(d_prefix<<" "<<end.toLogString()<<" is not part of "<<begin.toString()<<endl);
+    throw PDNSException(end.toLogString() + " is not part of " + begin.toString());
+  }
+
+  if (d_cutStates.count(begin) != 0) {
+    return;
+  }
+
+  dsmap_t ds;
+  vState cutState = getDSRecords(end, ds, false, depth);
+  if (cutState == TA) {
+    cutState = Secure;
+  }
+  else if (cutState == NTA) {
+    cutState = Insecure;
+  }
+  LOG(d_prefix<<": setting cut state for "<<end<<" to "<<vStates[cutState]<<endl);
+  d_cutStates[end] = cutState;
+
+  if (!d_DNSSECValidationRequested) {
+    return;
+  }
+
+  DNSName qname(end);
+  std::vector<string> labelsToAdd = begin.makeRelative(end).getRawLabels();
+
+  bool oldSkipCNAME = d_skipCNAMECheck;
+  bool oldRequireAuthData = d_requireAuthData;
+  d_skipCNAMECheck = true;
+  d_requireAuthData = false;
+
+  while(qname != begin) {
+    bool foundCut = false;
+    if (labelsToAdd.empty())
+      break;
+
+    qname.prependRawLabel(labelsToAdd.back());
+    labelsToAdd.pop_back();
+    LOG(d_prefix<<": - Looking for a cut at "<<qname<<endl);
+
+    const auto cutIt = d_cutStates.find(qname);
+    if (cutIt != d_cutStates.cend()) {
+      if (cutIt->second != Indeterminate) {
+        LOG(d_prefix<<": - Cut already known at "<<qname<<endl);
+        continue;
+      }
+    }
+
+    std::set<GetBestNSAnswer> beenthere;
+    std::vector<DNSRecord> nsrecords;
+
+    vState state = Indeterminate;
+    /* temporarily mark as Indeterminate, so that we won't enter an endless loop
+       trying to determine that zone cut again. */
+    d_cutStates[qname] = state;
+    int rcode = doResolve(qname, QType(QType::NS), nsrecords, depth + 1, beenthere, state);
+
+    if (rcode == RCode::NoError && !nsrecords.empty()) {
+      for (const auto& record : nsrecords) {
+        if(record.d_type != QType::NS || record.d_name != qname)
+          continue;
+        foundCut = true;
+        break;
+      }
+      if (foundCut) {
+        LOG(d_prefix<<": - Found cut at "<<qname<<endl);
+        /* if we get a Bogus state while retrieving the NS,
+           the cut state is Bogus (we'll look for a (N)TA below though). */
+        if (state == Bogus) {
+          cutState = Bogus;
+        }
+        dsmap_t ds;
+        vState newState = getDSRecords(qname, ds, cutState == Insecure || cutState == Bogus, depth);
+        if (newState != Indeterminate) {
+          cutState = newState;
+        }
+        LOG(d_prefix<<": New state for "<<qname<<" is "<<vStates[cutState]<<endl);
+        if (cutState == TA) {
+          cutState = Secure;
+        }
+        else if (cutState == NTA) {
+          cutState = Insecure;
+        }
+        d_cutStates[qname] = cutState;
+      }
+    }
+    if (!foundCut) {
+      /* remove the temporary cut */
+      LOG(d_prefix<<qname<<": removing cut state for "<<qname<<", was "<<vStates[d_cutStates[qname]]<<endl);
+      d_cutStates.erase(qname);
+    }
+  }
+
+  d_skipCNAMECheck = oldSkipCNAME;
+  d_requireAuthData = oldRequireAuthData;
+
+  LOG(d_prefix<<": list of cuts from "<<begin<<" to "<<end<<endl);
+  for (const auto& cut : d_cutStates) {
+    if (cut.first.isRoot() || (begin.isPartOf(cut.first) && cut.first.isPartOf(end))) {
+      LOG(" - "<<cut.first<<": "<<vStates[cut.second]<<endl);
+    }
+  }
+}
+
+vState SyncRes::validateDNSKeys(const DNSName& zone, const std::vector<DNSRecord>& dnskeys, const std::vector<std::shared_ptr<RRSIGRecordContent> >& signatures, unsigned int depth)
+{
+  dsmap_t ds;
+  if (!signatures.empty()) {
+    DNSName signer = getSigner(signatures);
+
+    if (!signer.empty() && signer.isPartOf(zone)) {
+      vState state = getDSRecords(signer, ds, false, depth);
+
+      if (state == TA) {
+        state = Secure;
+      }
+      if (state != Secure) {
+        if (state == NTA) {
+          state = Insecure;
+        }
+        return state;
+      }
+    }
+  }
+
+  skeyset_t tentativeKeys;
+  std::vector<shared_ptr<DNSRecordContent> > toSign;
+
+  for (const auto& dnskey : dnskeys) {
+    if (dnskey.d_type == QType::DNSKEY) {
+      auto content = getRR<DNSKEYRecordContent>(dnskey);
+      if (content) {
+        tentativeKeys.insert(content);
+        toSign.push_back(content);
+      }
+    }
+  }
+
+  LOG(d_prefix<<": trying to validate "<<std::to_string(tentativeKeys.size())<<" DNSKEYs with "<<std::to_string(ds.size())<<" DS"<<endl);
+  skeyset_t validatedKeys;
+  validateDNSKeysAgainstDS(d_now.tv_sec, zone, ds, tentativeKeys, toSign, signatures, validatedKeys);
+
+  LOG(d_prefix<<": we now have "<<std::to_string(validatedKeys.size())<<" DNSKEYs"<<endl);
+
+  /* if we found at least one valid RRSIG covering the set,
+     all tentative keys are validated keys. Otherwise it means
+     we haven't found at least one DNSKEY and a matching RRSIG
+     covering this set, this looks Bogus. */
+  if (validatedKeys.size() != tentativeKeys.size()) {
+    LOG(d_prefix<<": returning Bogus state from "<<__func__<<"("<<zone<<")"<<endl);
+    return Bogus;
+  }
+
+  return Secure;
+}
+
+vState SyncRes::getDNSKeys(const DNSName& signer, skeyset_t& keys, unsigned int depth)
+{
+  std::vector<DNSRecord> records;
+  std::set<GetBestNSAnswer> beenthere;
+  LOG(d_prefix<<"Retrieving DNSKeys for "<<signer<<endl);
+
+  vState state = Indeterminate;
+  /* following CNAME might lead to us to the wrong DNSKEY */
+  bool oldSkipCNAME = d_skipCNAMECheck;
+  d_skipCNAMECheck = true;
+  int rcode = doResolve(signer, QType(QType::DNSKEY), records, depth + 1, beenthere, state);
+  d_skipCNAMECheck = oldSkipCNAME;
+
+  if (rcode == RCode::NoError) {
+    if (state == Secure) {
+      for (const auto& key : records) {
+        if (key.d_type == QType::DNSKEY) {
+          auto content = getRR<DNSKEYRecordContent>(key);
+          if (content) {
+            keys.insert(content);
+          }
+        }
+      }
+    }
+    LOG(d_prefix<<"Retrieved "<<keys.size()<<" DNSKeys for "<<signer<<", state is "<<vStates[state]<<endl);
+    return state;
+  }
+
+  LOG(d_prefix<<"Returning Bogus state from "<<__func__<<"("<<signer<<")"<<endl);
+  return Bogus;
+}
+
+vState SyncRes::validateRecordsWithSigs(unsigned int depth, const DNSName& qname, const QType& qtype, const DNSName& name, const std::vector<DNSRecord>& records, const std::vector<std::shared_ptr<RRSIGRecordContent> >& signatures)
+{
+  skeyset_t keys;
+  if (!signatures.empty()) {
+    const DNSName signer = getSigner(signatures);
+    if (!signer.empty() && name.isPartOf(signer)) {
+      if (qtype == QType::DNSKEY && signer == qname) {
+        /* we are already retrieving those keys, sorry */
+        return Indeterminate;
+      }
+      vState state = getDNSKeys(signer, keys, depth);
+      if (state != Secure) {
+        return state;
+      }
+    }
+  } else {
+    LOG(d_prefix<<"Bogus!"<<endl);
+    return Bogus;
+  }
+
+  std::vector<std::shared_ptr<DNSRecordContent> > recordcontents;
+  for (const auto& record : records) {
+    recordcontents.push_back(record.d_content);
+  }
+
+  LOG(d_prefix<<"Going to validate "<<recordcontents.size()<< " record contents with "<<signatures.size()<<" sigs and "<<keys.size()<<" keys for "<<name<<endl);
+  if (validateWithKeySet(d_now.tv_sec, name, recordcontents, signatures, keys, false)) {
+    LOG(d_prefix<<"Secure!"<<endl);
+    return Secure;
+  }
+
+  LOG(d_prefix<<"Bogus!"<<endl);
+  return Bogus;
+}
+
+RCode::rcodes_ SyncRes::updateCacheFromRecords(unsigned int depth, LWResult& lwr, const DNSName& qname, const QType& qtype, const DNSName& auth, bool wasForwarded, const boost::optional<Netmask> ednsmask, vState& state, bool& needWildcardProof)
+{
+  struct CacheEntry
+  {
+    vector<DNSRecord> records;
+    vector<shared_ptr<RRSIGRecordContent>> signatures;
+    uint32_t signaturesTTL{std::numeric_limits<uint32_t>::max()};
+  };
+  struct CacheKey
+  {
+    DNSName name;
+    uint16_t type;
+    DNSResourceRecord::Place place;
+    bool operator<(const CacheKey& rhs) const {
+      return tie(name, type) < tie(rhs.name, rhs.type);
+    }
+  };
+  typedef map<CacheKey, CacheEntry> tcache_t;
+  tcache_t tcache;
+
+  string prefix;
+  if(doLog()) {
+    prefix=d_prefix;
+    prefix.append(depth, ' ');
+  }
+
+  std::vector<std::shared_ptr<DNSRecord>> authorityRecs;
+  bool isCNAMEAnswer = false;
+  for(const auto& rec : lwr.d_records) {
+    if(!isCNAMEAnswer && rec.d_place == DNSResourceRecord::ANSWER && rec.d_type == QType::CNAME && (!(qtype==QType(QType::CNAME))) && rec.d_name == qname) {
+      isCNAMEAnswer = true;
+    }
+
+    if(needWildcardProof) {
+      if (nsecTypes.count(rec.d_type)) {
+        authorityRecs.push_back(std::make_shared<DNSRecord>(rec));
+      }
+      else if (rec.d_type == QType::RRSIG) {
+        auto rrsig = getRR<RRSIGRecordContent>(rec);
+        if (rrsig && nsecTypes.count(rrsig->d_type)) {
+          authorityRecs.push_back(std::make_shared<DNSRecord>(rec));
+        }
+      }
+    }
+    if(rec.d_type == QType::RRSIG) {
+      auto rrsig = getRR<RRSIGRecordContent>(rec);
+      if (rrsig) {
+        unsigned int labelCount = rec.d_name.countLabels();
+        /* As illustrated in rfc4035's Appendix B.6, the RRSIG label
+           count can be lower than the name's label count if it was
+           synthesized from the wildcard. Note that the difference might
+           be > 1. */
+        if (rec.d_name == qname && rrsig->d_labels < labelCount) {
+          LOG(prefix<<qname<<": RRSIG indicates the name was expanded from a wildcard, we need a wildcard proof"<<endl);
+          needWildcardProof = true;
+        }
+
+        //         cerr<<"Got an RRSIG for "<<DNSRecordContent::NumberToType(rrsig->d_type)<<" with name '"<<rec.d_name<<"'"<<endl;
+        tcache[{rec.d_name, rrsig->d_type, rec.d_place}].signatures.push_back(rrsig);
+        tcache[{rec.d_name, rrsig->d_type, rec.d_place}].signaturesTTL = std::min(tcache[{rec.d_name, rrsig->d_type, rec.d_place}].signaturesTTL, rec.d_ttl);
+      }
+    }
+  }
+
+  // reap all answers from this packet that are acceptable
+  for(auto& rec : lwr.d_records) {
+    if(rec.d_type == QType::OPT) {
+      LOG(prefix<<qname<<": OPT answer '"<<rec.d_name<<"' from '"<<auth<<"' nameservers" <<endl);
+      continue;
+    }
+    LOG(prefix<<qname<<": accept answer '"<<rec.d_name<<"|"<<DNSRecordContent::NumberToType(rec.d_type)<<"|"<<rec.d_content->getZoneRepresentation()<<"' from '"<<auth<<"' nameservers? "<<(int)rec.d_place<<" ");
+    if(rec.d_type == QType::ANY) {
+      LOG("NO! - we don't accept 'ANY' data"<<endl);
+      continue;
+    }
+
+    if(rec.d_name.isPartOf(auth)) {
+      if(rec.d_type == QType::RRSIG) {
+        LOG("RRSIG - separate"<<endl);
+      }
+      else if(lwr.d_aabit && lwr.d_rcode==RCode::NoError && rec.d_place==DNSResourceRecord::ANSWER && ((rec.d_type != QType::DNSKEY && rec.d_type != QType::DS) || rec.d_name != auth) && s_delegationOnly.count(auth)) {
+        LOG("NO! Is from delegation-only zone"<<endl);
+        s_nodelegated++;
+        return RCode::NXDomain;
+      }
+      else {
+        bool haveLogged = false;
+        if (!t_sstorage.domainmap->empty()) {
+          // Check if we are authoritative for a zone in this answer
+          DNSName tmp_qname(rec.d_name);
+          auto auth_domain_iter=getBestAuthZone(&tmp_qname);
+          if(auth_domain_iter!=t_sstorage.domainmap->end() &&
+             auth.countLabels() <= auth_domain_iter->first.countLabels()) {
+            if (auth_domain_iter->first != auth) {
+              LOG("NO! - we are authoritative for the zone "<<auth_domain_iter->first<<endl);
+              continue;
+            } else {
+              LOG("YES! - This answer was ");
+              if (!wasForwarded) {
+                LOG("retrieved from the local auth store.");
+              } else {
+                LOG("received from a server we forward to.");
+              }
+              haveLogged = true;
+              LOG(endl);
+            }
+          }
+        }
+        if (!haveLogged) {
+          LOG("YES!"<<endl);
+        }
+
+        rec.d_ttl=min(s_maxcachettl, rec.d_ttl);
+
+        DNSRecord dr(rec);
+        dr.d_ttl += d_now.tv_sec;
+        dr.d_place=DNSResourceRecord::ANSWER;
+        tcache[{rec.d_name,rec.d_type,rec.d_place}].records.push_back(dr);
+      }
+    }
+    else
+      LOG("NO!"<<endl);
+  }
+
+  // supplant
+  for(tcache_t::iterator i = tcache.begin(); i != tcache.end(); ++i) {
+    if((i->second.records.size() + i->second.signatures.size()) > 1) {  // need to group the ttl to be the minimum of the RRSET (RFC 2181, 5.2)
+      uint32_t lowestTTD=computeLowestTTD(i->second.records, i->second.signatures, i->second.signaturesTTL);
+
+      for(auto& record : i->second.records)
+        record.d_ttl = lowestTTD; // boom
+    }
+
+//             cout<<"Have "<<i->second.records.size()<<" records and "<<i->second.signatures.size()<<" signatures for "<<i->first.name;
+//             cout<<'|'<<DNSRecordContent::NumberToType(i->first.type)<<endl;
+  }
+
+  for(tcache_t::iterator i = tcache.begin(); i != tcache.end(); ++i) {
+
+    if(i->second.records.empty()) // this happens when we did store signatures, but passed on the records themselves
+      continue;
+
+    bool isAA = lwr.d_aabit;
+    if (isAA && isCNAMEAnswer && (i->first.place != DNSResourceRecord::ANSWER || i->first.type != QType::CNAME)) {
+      /*
+        rfc2181 states:
+        Note that the answer section of an authoritative answer normally
+        contains only authoritative data.  However when the name sought is an
+        alias (see section 10.1.1) only the record describing that alias is
+        necessarily authoritative.  Clients should assume that other records
+        may have come from the server's cache.  Where authoritative answers
+        are required, the client should query again, using the canonical name
+        associated with the alias.
+      */
+      isAA = false;
+    }
+
+    vState recordState = getValidationStatus(auth);
+    LOG(d_prefix<<": got initial zone status "<<vStates[recordState]<<" for record "<<i->first.name<<endl);
+
+    if (d_DNSSECValidationRequested && recordState == Secure) {
+      if (isAA) {
+        if (i->first.place != DNSResourceRecord::ADDITIONAL) {
+          /* the additional entries can be insecure,
+             like glue:
+             "Glue address RRsets associated with delegations MUST NOT be signed"
+          */
+          if (i->first.type == QType::DNSKEY && i->first.place == DNSResourceRecord::ANSWER) {
+            LOG(d_prefix<<"Validating DNSKEY for "<<i->first.name<<endl);
+            recordState = validateDNSKeys(i->first.name, i->second.records, i->second.signatures, depth);
+          }
+          else {
+            LOG(d_prefix<<"Validating non-additional record for "<<i->first.name<<endl);
+            recordState = validateRecordsWithSigs(depth, qname, qtype, i->first.name, i->second.records, i->second.signatures);
+            /* we might have missed a cut (zone cut within the same auth servers), causing the NS query for an Insecure zone to seem Bogus during zone cut determination */
+            if (qtype == QType::NS && i->second.signatures.empty() && recordState == Bogus && haveExactValidationStatus(i->first.name) && getValidationStatus(i->first.name) == Indeterminate) {
+              recordState = Indeterminate;
+            }
+          }
+        }
+      }
+      else {
+        /* in a non authoritative answer, we only care about the DS record (or lack of)  */
+        if ((i->first.type == QType::DS || i->first.type == QType::NSEC || i->first.type == QType::NSEC3) && i->first.place == DNSResourceRecord::AUTHORITY) {
+          LOG(d_prefix<<"Validating DS record for "<<i->first.name<<endl);
+          recordState = validateRecordsWithSigs(depth, qname, qtype, i->first.name, i->second.records, i->second.signatures);
+        }
+      }
+
+      updateValidationState(state, recordState);
+    }
+    else {
+      if (d_DNSSECValidationRequested) {
+        LOG(d_prefix<<"Skipping validation because the current state is "<<vStates[recordState]<<endl);
+      }
+    }
+
+    t_RC->replace(d_now.tv_sec, i->first.name, QType(i->first.type), i->second.records, i->second.signatures, authorityRecs, isAA, i->first.place == DNSResourceRecord::ANSWER ? ednsmask : boost::none, recordState);
+
+    if(i->first.place == DNSResourceRecord::ANSWER && ednsmask)
+      d_wasVariable=true;
+  }
+
+  return RCode::NoError;
+}
+
+void SyncRes::getDenialValidationState(NegCache::NegCacheEntry& ne, vState& state, const dState expectedState, bool allowOptOut)
+{
+  ne.d_validationState = state;
+
+  if (state == Secure) {
+    cspmap_t csp = harvestCSPFromNE(ne);
+    dState res = getDenial(csp, ne.d_name, ne.d_qtype.getCode());
+    if (res != expectedState) {
+      if (res == OPTOUT && allowOptOut) {
+        LOG(d_prefix<<"OPT-out denial found for "<<ne.d_name<<", retuning Insecure"<<endl);
+        ne.d_validationState = Secure;
+        updateValidationState(state, Insecure);
+        return;
+      }
+      else if (res == INSECURE) {
+        LOG(d_prefix<<"Insecure denial found for "<<ne.d_name<<", retuning Insecure"<<endl);
+        ne.d_validationState = Insecure;
+      }
+      if (res == NXDOMAIN && expectedState == NXQTYPE) {
+        /* might happen for empty non-terminal, have fun */
+        return;
+      }
+      else {
+        LOG(d_prefix<<"Invalid denial found for "<<ne.d_name<<", retuning Bogus"<<endl);
+        ne.d_validationState = Bogus;
+      }
+      updateValidationState(state, ne.d_validationState);
+    }
+  }
+}
+
+bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, const QType& qtype, const DNSName& auth, LWResult& lwr, const bool sendRDQuery, vector<DNSRecord>& ret, set<DNSName>& nsset, DNSName& newtarget, DNSName& newauth, bool& realreferral, bool& negindic, vState& state, bool needWildcardProof)
+{
+  bool done = false;
+
+  for(auto& rec : lwr.d_records) {
+    if (rec.d_type!=QType::OPT && rec.d_class!=QClass::IN)
       continue;
 
     if(rec.d_place==DNSResourceRecord::AUTHORITY && rec.d_type==QType::SOA &&
@@ -1261,33 +1909,39 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co
       rec.d_ttl = min(rec.d_ttl, s_maxnegttl);
       if(newtarget.empty()) // only add a SOA if we're not going anywhere after this
         ret.push_back(rec);
-      if(!wasVariable()) {
-        NegCache::NegCacheEntry ne;
 
-        ne.d_ttd = d_now.tv_sec + rec.d_ttl;
-        ne.d_name = qname;
-        ne.d_qtype = QType(0); // this encodes 'whole record'
-        ne.d_auth = rec.d_name;
-        harvestNXRecords(lwr.d_records, ne);
-        t_sstorage->negcache.add(ne);
+      NegCache::NegCacheEntry ne;
+
+      ne.d_ttd = d_now.tv_sec + rec.d_ttl;
+      ne.d_name = qname;
+      ne.d_qtype = QType(0); // this encodes 'whole record'
+      ne.d_auth = rec.d_name;
+      harvestNXRecords(lwr.d_records, ne);
+      getDenialValidationState(ne, state, NXDOMAIN, false);
+
+      if(!wasVariable()) {
+        t_sstorage.negcache.add(ne);
         if(s_rootNXTrust && ne.d_auth.isRoot() && auth.isRoot()) {
           ne.d_name = ne.d_name.getLastLabel();
-          t_sstorage->negcache.add(ne);
+          t_sstorage.negcache.add(ne);
         }
       }
 
       negindic=true;
     }
-    else if(rec.d_place==DNSResourceRecord::ANSWER && rec.d_name == qname && rec.d_type==QType::CNAME && (!(qtype==QType(QType::CNAME)))) {
+    else if(rec.d_place==DNSResourceRecord::ANSWER && rec.d_type==QType::CNAME && (!(qtype==QType(QType::CNAME))) && rec.d_name == qname) {
       ret.push_back(rec);
       if (auto content = getRR<CNAMERecordContent>(rec)) {
         newtarget=content->getTarget();
       }
     }
-    else if((rec.d_type==QType::RRSIG || rec.d_type==QType::NSEC || rec.d_type==QType::NSEC3) && rec.d_place==DNSResourceRecord::ANSWER){
+    else if((rec.d_type==QType::RRSIG || rec.d_type==QType::NSEC || rec.d_type==QType::NSEC3) && rec.d_place==DNSResourceRecord::ANSWER) {
       if(rec.d_type != QType::RRSIG || rec.d_name == qname)
         ret.push_back(rec); // enjoy your DNSSEC
     }
+    else if(needWildcardProof && (rec.d_type==QType::RRSIG || rec.d_type==QType::NSEC || rec.d_type==QType::NSEC3) && rec.d_place==DNSResourceRecord::AUTHORITY) {
+      ret.push_back(rec); // enjoy your DNSSEC
+    }
     // for ANY answers we *must* have an authoritative answer, unless we are forwarding recursively
     else if(rec.d_place==DNSResourceRecord::ANSWER && rec.d_name == qname &&
             (
@@ -1300,7 +1954,7 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co
       done=true;
       ret.push_back(rec);
     }
-    else if(rec.d_place==DNSResourceRecord::AUTHORITY && qname.isPartOf(rec.d_name) && rec.d_type==QType::NS) {
+    else if(rec.d_place==DNSResourceRecord::AUTHORITY && rec.d_type==QType::NS && qname.isPartOf(rec.d_name)) {
       if(moreSpecificThan(rec.d_name,auth)) {
         newauth=rec.d_name;
         LOG(prefix<<qname<<": got NS record '"<<rec.d_name<<"' -> '"<<rec.d_content->getZoneRepresentation()<<"'"<<endl);
@@ -1313,12 +1967,43 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co
         nsset.insert(content->getNS());
       }
     }
-    else if(rec.d_place==DNSResourceRecord::AUTHORITY && qname.isPartOf(rec.d_name) && rec.d_type==QType::DS) {
+    else if(rec.d_place==DNSResourceRecord::AUTHORITY && rec.d_type==QType::DS && qname.isPartOf(rec.d_name)) {
       LOG(prefix<<qname<<": got DS record '"<<rec.d_name<<"' -> '"<<rec.d_content->getZoneRepresentation()<<"'"<<endl);
-      sawDS=true;
     }
-    else if(!done && rec.d_place==DNSResourceRecord::AUTHORITY && qname.isPartOf(rec.d_name) && rec.d_type==QType::SOA &&
-            lwr.d_rcode==RCode::NoError) {
+    else if(realreferral && rec.d_place==DNSResourceRecord::AUTHORITY && (rec.d_type==QType::NSEC || rec.d_type==QType::NSEC3) && newauth.isPartOf(auth)) {
+      /* we might have received a denial of the DS, let's check */
+      if (state == Secure) {
+        NegCache::NegCacheEntry ne;
+        ne.d_auth = auth;
+        ne.d_ttd = d_now.tv_sec + rec.d_ttl;
+        ne.d_name = newauth;
+        ne.d_qtype = QType::DS;
+        harvestNXRecords(lwr.d_records, ne);
+        cspmap_t csp = harvestCSPFromNE(ne);
+        dState denialState = getDenial(csp, newauth, QType::DS);
+        if (denialState == NXQTYPE || denialState == OPTOUT || denialState == INSECURE) {
+          ne.d_validationState = Secure;
+          rec.d_ttl = min(s_maxnegttl, rec.d_ttl);
+          LOG(prefix<<qname<<": got negative indication of DS record for '"<<newauth<<"'"<<endl);
+          updateValidationState(state, Insecure);
+          auto cut = d_cutStates.find(newauth);
+          if (cut != d_cutStates.end()) {
+            if (cut->second == Indeterminate) {
+              cut->second = state;
+            }
+          }
+          else {
+            LOG(prefix<<qname<<": setting cut state for "<<newauth<<" to "<<vStates[state]<<endl);
+            d_cutStates[newauth] = state;
+          }
+          if(!wasVariable()) {
+            t_sstorage.negcache.add(ne);
+          }
+        }
+      }
+    }
+    else if(!done && rec.d_place==DNSResourceRecord::AUTHORITY && rec.d_type==QType::SOA &&
+            lwr.d_rcode==RCode::NoError && qname.isPartOf(rec.d_name)) {
       LOG(prefix<<qname<<": got negative caching indication for '"<< qname<<"|"<<qtype.getName()<<"'"<<endl);
 
       if(!newtarget.empty()) {
@@ -1327,15 +2012,18 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co
       else {
         rec.d_ttl = min(s_maxnegttl, rec.d_ttl);
         ret.push_back(rec);
+
+        NegCache::NegCacheEntry ne;
+        ne.d_auth = rec.d_name;
+        ne.d_ttd = d_now.tv_sec + rec.d_ttl;
+        ne.d_name = qname;
+        ne.d_qtype = qtype;
+        harvestNXRecords(lwr.d_records, ne);
+        getDenialValidationState(ne, state, NXQTYPE, qtype == QType::DS);
+
         if(!wasVariable()) {
-          NegCache::NegCacheEntry ne;
-          ne.d_auth = rec.d_name;
-          ne.d_ttd = d_now.tv_sec + rec.d_ttl;
-          ne.d_name = qname;
-          ne.d_qtype = qtype;
-          harvestNXRecords(lwr.d_records, ne);
           if(qtype.getCode()) {  // prevents us from blacking out a whole domain
-            t_sstorage->negcache.add(ne);
+            t_sstorage.negcache.add(ne);
           }
         }
         negindic=true;
@@ -1346,6 +2034,234 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co
   return done;
 }
 
+bool SyncRes::doResolveAtThisIP(const std::string& prefix, const DNSName& qname, const QType& qtype, LWResult& lwr, boost::optional<Netmask>& ednsmask, const DNSName& auth, bool const sendRDQuery, const DNSName& nsName, const ComboAddress& remoteIP, bool doTCP, bool* truncated)
+{
+  int resolveret;
+  s_outqueries++;
+  d_outqueries++;
+
+  if(d_outqueries + d_throttledqueries > s_maxqperq) {
+    throw ImmediateServFailException("more than "+std::to_string(s_maxqperq)+" (max-qperq) queries sent while resolving "+qname.toLogString());
+  }
+
+  if(s_maxtotusec && d_totUsec > s_maxtotusec) {
+    throw ImmediateServFailException("Too much time waiting for "+qname.toLogString()+"|"+qtype.getName()+", timeouts: "+std::to_string(d_timeouts) +", throttles: "+std::to_string(d_throttledqueries) + ", queries: "+std::to_string(d_outqueries)+", "+std::to_string(d_totUsec/1000)+"msec");
+  }
+
+  if(doTCP) {
+    LOG(prefix<<qname<<": using TCP with "<< remoteIP.toStringWithPort() <<endl);
+    s_tcpoutqueries++;
+    d_tcpoutqueries++;
+  }
+
+  if(d_pdl && d_pdl->preoutquery(remoteIP, d_requestor, qname, qtype, doTCP, lwr.d_records, resolveret)) {
+    LOG(prefix<<qname<<": query handled by Lua"<<endl);
+  }
+  else {
+    ednsmask=getEDNSSubnetMask(d_requestor, qname, remoteIP);
+    if(ednsmask) {
+      LOG(prefix<<qname<<": Adding EDNS Client Subnet Mask "<<ednsmask->toString()<<" to query"<<endl);
+      s_ecsqueries++;
+    }
+    resolveret = asyncresolveWrapper(remoteIP, d_doDNSSEC, qname,  qtype.getCode(),
+                                     doTCP, sendRDQuery, &d_now, ednsmask, &lwr);    // <- we go out on the wire!
+    if(ednsmask) {
+      s_ecsresponses++;
+      LOG(prefix<<qname<<": Received EDNS Client Subnet Mask "<<ednsmask->toString()<<" on response"<<endl);
+    }
+  }
+
+  /* preoutquery killed the query by setting dq.rcode to -3 */
+  if(resolveret==-3) {
+    throw ImmediateServFailException("Query killed by policy");
+  }
+
+  d_totUsec += lwr.d_usec;
+  accountAuthLatency(lwr.d_usec, remoteIP.sin4.sin_family);
+
+  if(resolveret != 1) {
+    /* Error while resolving */
+    if(resolveret == 0) {
+      /* Time out */
+
+      LOG(prefix<<qname<<": timeout resolving after "<<lwr.d_usec/1000.0<<"msec "<< (doTCP ? "over TCP" : "")<<endl);
+      d_timeouts++;
+      s_outgoingtimeouts++;
+
+      if(remoteIP.sin4.sin_family == AF_INET)
+        s_outgoing4timeouts++;
+      else
+        s_outgoing6timeouts++;
+    }
+    else if(resolveret == -2) {
+      /* OS resource limit reached */
+      LOG(prefix<<qname<<": hit a local resource limit resolving"<< (doTCP ? " over TCP" : "")<<", probable error: "<<stringerror()<<endl);
+      g_stats.resourceLimits++;
+    }
+    else {
+      /* -1 means server unreachable */
+      s_unreachables++;
+      d_unreachables++;
+      LOG(prefix<<qname<<": error resolving from "<<remoteIP.toString()<< (doTCP ? " over TCP" : "") <<", possible error: "<<strerror(errno)<< endl);
+    }
+
+    if(resolveret != -2) { // don't account for resource limits, they are our own fault
+      t_sstorage.nsSpeeds[nsName].submit(remoteIP, 1000000, &d_now); // 1 sec
+
+      // code below makes sure we don't filter COM or the root
+      if (s_serverdownmaxfails > 0 && (auth != g_rootdnsname) && t_sstorage.fails.incr(remoteIP) >= s_serverdownmaxfails) {
+        LOG(prefix<<qname<<": Max fails reached resolving on "<< remoteIP.toString() <<". Going full throttle for "<< s_serverdownthrottletime <<" seconds" <<endl);
+        // mark server as down
+        t_sstorage.throttle.throttle(d_now.tv_sec, boost::make_tuple(remoteIP, "", 0), s_serverdownthrottletime, 10000);
+      }
+      else if (resolveret == -1) {
+        // unreachable, 1 minute or 100 queries
+        t_sstorage.throttle.throttle(d_now.tv_sec, boost::make_tuple(remoteIP, qname, qtype.getCode()), 60, 100);
+      }
+      else {
+        // timeout
+        t_sstorage.throttle.throttle(d_now.tv_sec, boost::make_tuple(remoteIP, qname, qtype.getCode()), 10, 5);
+      }
+    }
+
+    return false;
+  }
+
+  /* we got an answer */
+  if(lwr.d_rcode==RCode::ServFail || lwr.d_rcode==RCode::Refused) {
+    LOG(prefix<<qname<<": "<<nsName<<" ("<<remoteIP.toString()<<") returned a "<< (lwr.d_rcode==RCode::ServFail ? "ServFail" : "Refused") << ", trying sibling IP or NS"<<endl);
+    t_sstorage.throttle.throttle(d_now.tv_sec, boost::make_tuple(remoteIP, qname, qtype.getCode()), 60, 3);
+    return false;
+  }
+
+  /* this server sent a valid answer, mark it backup up if it was down */
+  if(s_serverdownmaxfails > 0) {
+    t_sstorage.fails.clear(remoteIP);
+  }
+
+  if(lwr.d_tcbit) {
+    *truncated = true;
+
+    if (doTCP) {
+      LOG(prefix<<qname<<": truncated bit set, over TCP?"<<endl);
+      /* let's treat that as a ServFail answer from this server */
+      t_sstorage.throttle.throttle(d_now.tv_sec, boost::make_tuple(remoteIP, qname, qtype.getCode()), 60, 3);
+      return false;
+    }
+
+    return true;
+  }
+
+  return true;
+}
+
+bool SyncRes::processAnswer(unsigned int depth, LWResult& lwr, const DNSName& qname, const QType& qtype, DNSName& auth, bool wasForwarded, const boost::optional<Netmask> ednsmask, bool sendRDQuery, NsSet &nameservers, std::vector<DNSRecord>& ret, const DNSFilterEngine& dfe, bool* gotNewServers, int* rcode, vState& state)
+{
+  string prefix;
+  if(doLog()) {
+    prefix=d_prefix;
+    prefix.append(depth, ' ');
+  }
+
+  if(s_minimumTTL) {
+    for(auto& rec : lwr.d_records) {
+      rec.d_ttl = max(rec.d_ttl, s_minimumTTL);
+    }
+  }
+
+  bool needWildcardProof = false;
+  *rcode = updateCacheFromRecords(depth, lwr, qname, qtype, auth, wasForwarded, ednsmask, state, needWildcardProof);
+  if (*rcode != RCode::NoError) {
+    return true;
+  }
+
+  LOG(prefix<<qname<<": determining status after receiving this packet"<<endl);
+
+  set<DNSName> nsset;
+  bool realreferral=false, negindic=false;
+  DNSName newauth;
+  DNSName newtarget;
+
+  bool done = processRecords(prefix, qname, qtype, auth, lwr, sendRDQuery, ret, nsset, newtarget, newauth, realreferral, negindic, state, needWildcardProof);
+
+  if(done){
+    LOG(prefix<<qname<<": status=got results, this level of recursion done"<<endl);
+    *rcode = RCode::NoError;
+    return true;
+  }
+
+  if(!newtarget.empty()) {
+    if(newtarget == qname) {
+      LOG(prefix<<qname<<": status=got a CNAME referral to self, returning SERVFAIL"<<endl);
+      *rcode = RCode::ServFail;
+      return true;
+    }
+
+    if(depth > 10) {
+      LOG(prefix<<qname<<": status=got a CNAME referral, but recursing too deep, returning SERVFAIL"<<endl);
+      *rcode = RCode::ServFail;
+      return true;
+    }
+
+    LOG(prefix<<qname<<": status=got a CNAME referral, starting over with "<<newtarget<<endl);
+
+    set<GetBestNSAnswer> beenthere2;
+    vState cnameState = Indeterminate;
+    *rcode = doResolve(newtarget, qtype, ret, depth + 1, beenthere2, cnameState);
+    LOG(prefix<<qname<<": updating validation state for response to "<<qname<<" from "<<vStates[state]<<" with the state from the CNAME quest: "<<vStates[cnameState]<<endl);
+    updateValidationState(state, cnameState);
+    return true;
+  }
+
+  if(lwr.d_rcode == RCode::NXDomain) {
+    LOG(prefix<<qname<<": status=NXDOMAIN, we are done "<<(negindic ? "(have negative SOA)" : "")<<endl);
+
+    if(d_doDNSSEC)
+      addNXNSECS(ret, lwr.d_records);
+
+    *rcode = RCode::NXDomain;
+    return true;
+  }
+
+  if(nsset.empty() && !lwr.d_rcode && (negindic || lwr.d_aabit || sendRDQuery)) {
+    LOG(prefix<<qname<<": status=noerror, other types may exist, but we are done "<<(negindic ? "(have negative SOA) " : "")<<(lwr.d_aabit ? "(have aa bit) " : "")<<endl);
+
+    if(state == Secure && lwr.d_aabit && !negindic) {
+      updateValidationState(state, Bogus);
+    }
+
+    if(d_doDNSSEC)
+      addNXNSECS(ret, lwr.d_records);
+
+    *rcode = RCode::NoError;
+    return true;
+  }
+
+  if(realreferral) {
+    LOG(prefix<<qname<<": status=did not resolve, got "<<(unsigned int)nsset.size()<<" NS, ");
+
+    nameservers.clear();
+    for (auto const &nameserver : nsset) {
+      if (d_wantsRPZ) {
+        d_appliedPolicy = dfe.getProcessingPolicy(nameserver, d_discardedPolicies);
+        if (d_appliedPolicy.d_kind != DNSFilterEngine::PolicyKind::NoAction) { // client query needs an RPZ response
+          LOG("however "<<nameserver<<" was blocked by RPZ policy '"<<(d_appliedPolicy.d_name ? *d_appliedPolicy.d_name : "")<<"'"<<endl);
+          *rcode = -2;
+          return true;
+        }
+      }
+      nameservers.insert({nameserver, {{}, false}});
+    }
+    LOG("looping to them"<<endl);
+    *gotNewServers = true;
+    auth=newauth;
+
+    return false;
+  }
+
+  return false;
+}
+
 /** returns:
  *  -1 in case of no results
  *  -2 when a FilterEngine Policy was hit
@@ -1353,7 +2269,7 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co
  */
 int SyncRes::doResolveAt(NsSet &nameservers, DNSName auth, bool flawedNSSet, const DNSName &qname, const QType &qtype,
                          vector<DNSRecord>&ret,
-                         unsigned int depth, set<GetBestNSAnswer>&beenthere)
+                         unsigned int depth, set<GetBestNSAnswer>&beenthere, vState& state)
 {
   auto luaconfsLocal = g_luaconfs.getLocal();
   string prefix;
@@ -1384,6 +2300,7 @@ int SyncRes::doResolveAt(NsSet &nameservers, DNSName auth, bool flawedNSSet, con
         }
         return -1;
       }
+
       // this line needs to identify the 'self-resolving' behaviour, but we get it wrong now
       if(qname == *tns && qtype.getCode()==QType::A && rnameservers.size() > (size_t)(1+1*s_doIPv6)) {
         LOG(prefix<<qname<<": Not using NS to resolve itself! ("<<(1+tns-rnameservers.cbegin())<<"/"<<rnameservers.size()<<")"<<endl);
@@ -1393,18 +2310,32 @@ int SyncRes::doResolveAt(NsSet &nameservers, DNSName auth, bool flawedNSSet, con
       typedef vector<ComboAddress> remoteIPs_t;
       remoteIPs_t remoteIPs;
       remoteIPs_t::const_iterator remoteIP;
-      bool doTCP=false;
       bool pierceDontQuery=false;
       bool sendRDQuery=false;
       boost::optional<Netmask> ednsmask;
       LWResult lwr;
-      if(tns->empty() && nameservers[*tns].first.empty() ) {
+      const bool wasForwarded = tns->empty() && (!nameservers[*tns].first.empty());
+      int rcode = RCode::NoError;
+      bool gotNewServers = false;
+
+      if(tns->empty() && !wasForwarded) {
         LOG(prefix<<qname<<": Domain is out-of-band"<<endl);
+        state = Insecure;
         d_wasOutOfBand = doOOBResolve(qname, qtype, lwr.d_records, depth, lwr.d_rcode);
         lwr.d_tcbit=false;
         lwr.d_aabit=true;
+
+        /* we have received an answer, are we done ? */
+        bool done = processAnswer(depth, lwr, qname, qtype, auth, false, ednsmask, sendRDQuery, nameservers, ret, luaconfsLocal->dfe, &gotNewServers, &rcode, state);
+        if (done) {
+          return rcode;
+        }
+        if (gotNewServers) {
+          break;
+        }
       }
       else {
+        /* if tns is empty, retrieveAddressesForNS() knows we have hardcoded servers (i.e. "forwards") */
         remoteIPs = retrieveAddressesForNS(prefix, qname, tns, depth, beenthere, rnameservers, nameservers, sendRDQuery, pierceDontQuery, flawedNSSet);
 
         if(remoteIPs.empty()) {
@@ -1431,197 +2362,53 @@ int SyncRes::doResolveAt(NsSet &nameservers, DNSName auth, bool flawedNSSet, con
 
         for(remoteIP = remoteIPs.cbegin(); remoteIP != remoteIPs.cend(); ++remoteIP) {
           LOG(prefix<<qname<<": Trying IP "<< remoteIP->toStringWithPort() <<", asking '"<<qname<<"|"<<qtype.getName()<<"'"<<endl);
+
           if (throttledOrBlocked(prefix, *remoteIP, qname, qtype, pierceDontQuery)) {
             continue;
           }
-          else {
-            int resolveret;
-            s_outqueries++; d_outqueries++;
-            if(d_outqueries + d_throttledqueries > s_maxqperq) throw ImmediateServFailException("more than "+std::to_string(s_maxqperq)+" (max-qperq) queries sent while resolving "+qname.toLogString());
-          TryTCP:
-            if(doTCP) {
-              LOG(prefix<<qname<<": using TCP with "<< remoteIP->toStringWithPort() <<endl);
-              s_tcpoutqueries++; d_tcpoutqueries++;
-            }
-
-           if(s_maxtotusec && d_totUsec > s_maxtotusec)
-             throw ImmediateServFailException("Too much time waiting for "+qname.toLogString()+"|"+qtype.getName()+", timeouts: "+std::to_string(d_timeouts) +", throttles: "+std::to_string(d_throttledqueries) + ", queries: "+std::to_string(d_outqueries)+", "+std::to_string(d_totUsec/1000)+"msec");
-
-           if(d_pdl && d_pdl->preoutquery(*remoteIP, d_requestor, qname, qtype, doTCP, lwr.d_records, resolveret)) {
-             LOG(prefix<<qname<<": query handled by Lua"<<endl);
-           }
-           else {
-             ednsmask=getEDNSSubnetMask(d_requestor, qname, *remoteIP);
-              if(ednsmask) {
-                LOG(prefix<<qname<<": Adding EDNS Client Subnet Mask "<<ednsmask->toString()<<" to query"<<endl);
-              }
-             resolveret=asyncresolveWrapper(*remoteIP, d_doDNSSEC, qname,  qtype.getCode(),
-                                            doTCP, sendRDQuery, &d_now, ednsmask, &lwr);    // <- we go out on the wire!
-              if(ednsmask) {
-                LOG(prefix<<qname<<": Received EDNS Client Subnet Mask "<<ednsmask->toString()<<" on response"<<endl);
-              }
-
 
-           }
-            if(resolveret==-3)
-             throw ImmediateServFailException("Query killed by policy");
-
-           d_totUsec += lwr.d_usec;
-           accountAuthLatency(lwr.d_usec, remoteIP->sin4.sin_family);
-           if(resolveret != 1) {
-              if(resolveret==0) {
-                LOG(prefix<<qname<<": timeout resolving after "<<lwr.d_usec/1000.0<<"msec "<< (doTCP ? "over TCP" : "")<<endl);
-                d_timeouts++;
-                s_outgoingtimeouts++;
-               if(remoteIP->sin4.sin_family == AF_INET)
-                 s_outgoing4timeouts++;
-               else
-                 s_outgoing6timeouts++;
-              }
-              else if(resolveret==-2) {
-                LOG(prefix<<qname<<": hit a local resource limit resolving"<< (doTCP ? " over TCP" : "")<<", probable error: "<<stringerror()<<endl);
-                g_stats.resourceLimits++;
-              }
-              else {
-                s_unreachables++; d_unreachables++;
-                LOG(prefix<<qname<<": error resolving from "<<remoteIP->toString()<< (doTCP ? " over TCP" : "") <<", possible error: "<<strerror(errno)<< endl);
-              }
+          bool truncated = false;
+          bool gotAnswer = doResolveAtThisIP(prefix, qname, qtype, lwr, ednsmask, auth, sendRDQuery,
+                                             *tns, *remoteIP, false, &truncated);
+          if (gotAnswer && truncated ) {
+            /* retry, over TCP this time */
+            gotAnswer = doResolveAtThisIP(prefix, qname, qtype, lwr, ednsmask, auth, sendRDQuery,
+                                          *tns, *remoteIP, true, &truncated);
+          }
 
-              if(resolveret!=-2) { // don't account for resource limits, they are our own fault
-               t_sstorage->nsSpeeds[*tns].submit(*remoteIP, 1000000, &d_now); // 1 sec
-
-               // code below makes sure we don't filter COM or the root
-                if (s_serverdownmaxfails > 0 && (auth != g_rootdnsname) && t_sstorage->fails.incr(*remoteIP) >= s_serverdownmaxfails) {
-                  LOG(prefix<<qname<<": Max fails reached resolving on "<< remoteIP->toString() <<". Going full throttle for "<< s_serverdownthrottletime <<" seconds" <<endl);
-                  t_sstorage->throttle.throttle(d_now.tv_sec, boost::make_tuple(*remoteIP, "", 0), s_serverdownthrottletime, 10000); // mark server as down
-                } else if(resolveret==-1)
-                  t_sstorage->throttle.throttle(d_now.tv_sec, boost::make_tuple(*remoteIP, qname, qtype.getCode()), 60, 100); // unreachable, 1 minute or 100 queries
-                else
-                  t_sstorage->throttle.throttle(d_now.tv_sec, boost::make_tuple(*remoteIP, qname, qtype.getCode()), 10, 5);  // timeout
-              }
-              continue;
-            }
+          if (!gotAnswer) {
+            continue;
+          }
 
-//         if(d_timeouts + 0.5*d_throttledqueries > 6.0 && d_timeouts > 2) throw ImmediateServFailException("Too much work resolving "+qname+"|"+qtype.getName()+", timeouts: "+std::to_string(d_timeouts) +", throttles: "+std::to_string(d_throttledqueries));
+          LOG(prefix<<qname<<": Got "<<(unsigned int)lwr.d_records.size()<<" answers from "<<*tns<<" ("<< remoteIP->toString() <<"), rcode="<<lwr.d_rcode<<" ("<<RCode::to_s(lwr.d_rcode)<<"), aa="<<lwr.d_aabit<<", in "<<lwr.d_usec/1000<<"ms"<<endl);
 
-            if(lwr.d_rcode==RCode::ServFail || lwr.d_rcode==RCode::Refused) {
-              LOG(prefix<<qname<<": "<<*tns<<" ("<<remoteIP->toString()<<") returned a "<< (lwr.d_rcode==RCode::ServFail ? "ServFail" : "Refused") << ", trying sibling IP or NS"<<endl);
-              t_sstorage->throttle.throttle(d_now.tv_sec,boost::make_tuple(*remoteIP, qname, qtype.getCode()),60,3); // servfail or refused
-              continue;
-            }
+          /*  // for you IPv6 fanatics :-)
+              if(remoteIP->sin4.sin_family==AF_INET6)
+              lwr.d_usec/=3;
+          */
+          //        cout<<"msec: "<<lwr.d_usec/1000.0<<", "<<g_avgLatency/1000.0<<'\n';
 
-            if(s_serverdownmaxfails > 0)
-              t_sstorage->fails.clear(*remoteIP);
+          t_sstorage.nsSpeeds[*tns].submit(*remoteIP, lwr.d_usec, &d_now);
 
-            break;  // this IP address worked!
-          wasLame:; // well, it didn't
-            LOG(prefix<<qname<<": status=NS "<<*tns<<" ("<< remoteIP->toString() <<") is lame for '"<<auth<<"', trying sibling IP or NS"<<endl);
-            t_sstorage->throttle.throttle(d_now.tv_sec, boost::make_tuple(*remoteIP, qname, qtype.getCode()), 60, 100); // lame
+          /* we have received an answer, are we done ? */
+          bool done = processAnswer(depth, lwr, qname, qtype, auth, wasForwarded, ednsmask, sendRDQuery, nameservers, ret, luaconfsLocal->dfe, &gotNewServers, &rcode, state);
+          if (done) {
+            return rcode;
           }
-        }
-
-        if(remoteIP == remoteIPs.cend())  // we tried all IP addresses, none worked
-          continue;
-
-        if(lwr.d_tcbit) {
-          if(!doTCP) {
-            doTCP=true;
-            LOG(prefix<<qname<<": truncated bit set, retrying via TCP"<<endl);
-            goto TryTCP;
+          if (gotNewServers) {
+            break;
           }
-          LOG(prefix<<qname<<": truncated bit set, over TCP?"<<endl);
-          return RCode::ServFail;
+          /* was lame */
+          t_sstorage.throttle.throttle(d_now.tv_sec, boost::make_tuple(*remoteIP, qname, qtype.getCode()), 60, 100);
         }
-        LOG(prefix<<qname<<": Got "<<(unsigned int)lwr.d_records.size()<<" answers from "<<*tns<<" ("<< remoteIP->toString() <<"), rcode="<<lwr.d_rcode<<" ("<<RCode::to_s(lwr.d_rcode)<<"), aa="<<lwr.d_aabit<<", in "<<lwr.d_usec/1000<<"ms"<<endl);
-
-        /*  // for you IPv6 fanatics :-)
-        if(remoteIP->sin4.sin_family==AF_INET6)
-          lwr.d_usec/=3;
-        */
-        //        cout<<"msec: "<<lwr.d_usec/1000.0<<", "<<g_avgLatency/1000.0<<'\n';
-
-        t_sstorage->nsSpeeds[*tns].submit(*remoteIP, lwr.d_usec, &d_now);
-      }
-
-      if(s_minimumTTL) {
-       for(auto& rec : lwr.d_records) {
-         rec.d_ttl = max(rec.d_ttl, s_minimumTTL);
-       }
-      }
-
-      RCode::rcodes_ rcode = updateCacheFromRecords(prefix, lwr, qname, auth, nameservers, *tns, ednsmask);
-      if (rcode != RCode::NoError) {
-        return rcode;
-      }
-
-      LOG(prefix<<qname<<": determining status after receiving this packet"<<endl);
 
-      set<DNSName> nsset;
-      bool realreferral=false, negindic=false, sawDS=false;
-      DNSName newauth;
-      DNSName newtarget;
-
-      bool done = processRecords(prefix, qname, qtype, auth, lwr, sendRDQuery, ret, nsset, newtarget, newauth, realreferral, negindic, sawDS);
-
-      if(done){
-        LOG(prefix<<qname<<": status=got results, this level of recursion done"<<endl);
-        return 0;
-      }
-      if(!newtarget.empty()) {
-        if(newtarget == qname) {
-          LOG(prefix<<qname<<": status=got a CNAME referral to self, returning SERVFAIL"<<endl);
-          return RCode::ServFail;
-        }
-        if(depth > 10) {
-          LOG(prefix<<qname<<": status=got a CNAME referral, but recursing too deep, returning SERVFAIL"<<endl);
-          return RCode::ServFail;
+        if (gotNewServers) {
+          break;
         }
-        LOG(prefix<<qname<<": status=got a CNAME referral, starting over with "<<newtarget<<endl);
-
-        set<GetBestNSAnswer> beenthere2;
-        return doResolve(newtarget, qtype, ret, depth + 1, beenthere2);
-      }
-      if(lwr.d_rcode==RCode::NXDomain) {
-        LOG(prefix<<qname<<": status=NXDOMAIN, we are done "<<(negindic ? "(have negative SOA)" : "")<<endl);
 
-        if(d_doDNSSEC)
-          addNXNSECS(ret, lwr.d_records);
+        if(remoteIP == remoteIPs.cend())  // we tried all IP addresses, none worked
+          continue;
 
-        return RCode::NXDomain;
-      }
-      if(nsset.empty() && !lwr.d_rcode && (negindic || lwr.d_aabit || sendRDQuery)) {
-        LOG(prefix<<qname<<": status=noerror, other types may exist, but we are done "<<(negindic ? "(have negative SOA) " : "")<<(lwr.d_aabit ? "(have aa bit) " : "")<<endl);
-        
-        if(d_doDNSSEC)
-          addNXNSECS(ret, lwr.d_records);
-        return 0;
-      }
-      else if(realreferral) {
-        LOG(prefix<<qname<<": status=did not resolve, got "<<(unsigned int)nsset.size()<<" NS, ");
-       if(sawDS) {
-         t_sstorage->dnssecmap[newauth]=true;
-         /*      for(const auto& e : t_sstorage->dnssecmap)
-           cout<<e.first<<' ';
-           cout<<endl;*/
-       }
-        auth=newauth;
-
-        nameservers.clear();
-        for (auto const &nameserver : nsset) {
-          if (d_wantsRPZ) {
-            d_appliedPolicy = luaconfsLocal->dfe.getProcessingPolicy(nameserver, d_discardedPolicies);
-            if (d_appliedPolicy.d_kind != DNSFilterEngine::PolicyKind::NoAction) { // client query needs an RPZ response
-              LOG("however "<<nameserver<<" was blocked by RPZ policy '"<<(d_appliedPolicy.d_name ? *d_appliedPolicy.d_name : "")<<"'"<<endl);
-              return -2;
-            }
-          }
-          nameservers.insert({nameserver, {{}, false}});
-        }
-        LOG("looping to them"<<endl);
-        break;
-      }
-      else if(!tns->empty()) { // means: not OOB, OOB == empty
-        goto wasLame;
       }
     }
   }
@@ -1638,7 +2425,7 @@ boost::optional<Netmask> SyncRes::getEDNSSubnetMask(const ComboAddress& local, c
       /* RFC7871 says we MUST NOT send any ECS if the source scope is 0 */
       return result;
     }
-    trunc = d_incomingECS->source.getMaskedNetwork();
+    trunc = d_incomingECSNetwork;
     bits = d_incomingECS->source.getBits();
   }
   else if(!local.isIPv4() || local.sin4.sin_addr.s_addr) { // detect unset 'requestor'
@@ -1685,13 +2472,12 @@ int directResolve(const DNSName& qname, const QType& qtype, int qclass, vector<D
   return res;
 }
 
-#include "validate-recursor.hh"
-
 int SyncRes::getRootNS(struct timeval now, asyncresolve_t asyncCallback) {
   SyncRes sr(now);
   sr.setDoEDNS0(true);
-  sr.setNoCache();
+  sr.setUpdatingRootNS();
   sr.setDoDNSSEC(g_dnssecmode != DNSSECMode::Off);
+  sr.setDNSSECValidationRequested(g_dnssecmode != DNSSECMode::Off && g_dnssecmode != DNSSECMode::ProcessNoValidate);
   sr.setAsyncCallback(asyncCallback);
 
   vector<DNSRecord> ret;
@@ -1699,31 +2485,30 @@ int SyncRes::getRootNS(struct timeval now, asyncresolve_t asyncCallback) {
   try {
     res=sr.beginResolve(g_rootdnsname, QType(QType::NS), 1, ret);
     if (g_dnssecmode != DNSSECMode::Off && g_dnssecmode != DNSSECMode::ProcessNoValidate) {
-      ResolveContext ctx;
-      auto state = validateRecords(ctx, ret);
+      auto state = sr.getValidationState();
       if (state == Bogus)
         throw PDNSException("Got Bogus validation result for .|NS");
     }
     return res;
   }
-  catch(PDNSException& e)
-  {
+  catch(const PDNSException& e) {
     L<<Logger::Error<<"Failed to update . records, got an exception: "<<e.reason<<endl;
   }
-
-  catch(std::exception& e)
-  {
+  catch(const ImmediateServFailException& e) {
+    L<<Logger::Error<<"Failed to update . records, got an exception: "<<e.reason<<endl;
+  }
+  catch(const std::exception& e) {
     L<<Logger::Error<<"Failed to update . records, got an exception: "<<e.what()<<endl;
   }
-
-  catch(...)
-  {
+  catch(...) {
     L<<Logger::Error<<"Failed to update . records, got an exception"<<endl;
   }
+
   if(!res) {
     L<<Logger::Notice<<"Refreshed . records"<<endl;
   }
   else
     L<<Logger::Error<<"Failed to update . records, RCODE="<<res<<endl;
+
   return res;
 }