pdns/slavecommunicator.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #ifdef HAVE_CONFIG_H
  23 #include "config.h"
  24 #endif
  25
  26 #include "utility.hh"
  27 #include "dnssecinfra.hh"
  28 #include "dnsseckeeper.hh"
  29 #include "base32.hh"
  30 #include <errno.h>
  31 #include "communicator.hh"
  32 #include <set>
  33 #include <boost/utility.hpp>
  34 #include "dnsbackend.hh"
  35 #include "ueberbackend.hh"
  36 #include "packethandler.hh"
  37 #include "resolver.hh"
  38 #include "logger.hh"
  39 #include "dns.hh"
  40 #include "arguments.hh"
  41 #include "auth-caches.hh"
  42
  43 #include "base64.hh"
  44 #include "inflighter.cc"
  45 #include "namespaces.hh"
  46 #include "common_startup.hh"
  47
  48 #include "ixfr.hh"
  49
  50 void CommunicatorClass::addSuckRequest(const DNSName &domain, const ComboAddress& master)
  51 {
  52   Lock l(&d_lock);
  53   SuckRequest sr;
  54   sr.domain = domain;
  55   sr.master = master;
  56   pair<UniQueue::iterator, bool>  res;
  57
  58   res=d_suckdomains.push_back(sr);
  59   if(res.second) {
  60     d_suck_sem.post();
  61   }
  62
  63 }
  64
  65 struct ZoneStatus
  66 {
  67   bool isDnssecZone{false};
  68   bool isPresigned{false};
  69   bool isNSEC3 {false};
  70   bool optOutFlag {false};
  71   NSEC3PARAMRecordContent ns3pr;
  72
  73   bool isNarrow{false};
  74   unsigned int soa_serial{0};
  75   set<DNSName> nsset, qnames, secured;
  76   uint32_t domain_id;
  77   int numDeltas{0};
  78 };
  79
  80
  81 void CommunicatorClass::ixfrSuck(const DNSName &domain, const TSIGTriplet& tt, const ComboAddress& laddr, const ComboAddress& remote, unique_ptr<AuthLua4>& pdl,
  82                                  ZoneStatus& zs, vector<DNSRecord>* axfr)
  83 {
  84   UeberBackend B; // fresh UeberBackend
  85
  86   DomainInfo di;
  87   di.backend=0;
  88   //  bool transaction=false;
  89   try {
  90     DNSSECKeeper dk (&B); // reuse our UeberBackend copy for DNSSECKeeper
  91
  92     bool wrongDomainKind = false;
  93     // this checks three error conditions, and sets wrongDomainKind if we hit the third & had an error
  94     if(!B.getDomainInfo(domain, di) || !di.backend || (wrongDomainKind = true, di.kind != DomainInfo::Slave)) { // di.backend and B are mostly identical
  95       if(wrongDomainKind)
  96         g_log<<Logger::Error<<"Can't determine backend for domain '"<<domain<<"', not configured as slave"<<endl;
  97       else
  98         g_log<<Logger::Error<<"Can't determine backend for domain '"<<domain<<"'"<<endl;
  99       return;
 100     }
 101
 102     soatimes st;
 103     memset(&st, 0, sizeof(st));
 104     st.serial=di.serial;
 105
 106     DNSRecord drsoa;
 107     drsoa.d_content = std::make_shared<SOARecordContent>(g_rootdnsname, g_rootdnsname, st);
 108     auto deltas = getIXFRDeltas(remote, domain, drsoa, tt, laddr.sin4.sin_family ? &laddr : 0, ((size_t) ::arg().asNum("xfr-max-received-mbytes")) * 1024 * 1024);
 109     zs.numDeltas=deltas.size();
 110     //    cout<<"Got "<<deltas.size()<<" deltas from serial "<<di.serial<<", applying.."<<endl;
 111
 112     for(const auto& d : deltas) {
 113       const auto& remove = d.first;
 114       const auto& add = d.second;
 115       //      cout<<"Delta sizes: "<<remove.size()<<", "<<add.size()<<endl;
 116
 117       if(remove.empty()) { // we got passed an AXFR!
 118         *axfr = add;
 119         return;
 120       }
 121
 122
 123       // our hammer is 'replaceRRSet(domain_id, qname, qt, vector<DNSResourceRecord>& rrset)
 124       // which thinks in terms of RRSETs
 125       // however, IXFR does not, and removes and adds *records* (bummer)
 126       // this means that we must group updates by {qname,qtype}, retrieve the RRSET, apply
 127       // the add/remove updates, and replaceRRSet the whole thing.
 128
 129
 130       map<pair<DNSName,uint16_t>, pair<vector<DNSRecord>, vector<DNSRecord> > > grouped;
 131
 132       for(const auto& x: remove)
 133         grouped[{x.d_name, x.d_type}].first.push_back(x);
 134       for(const auto& x: add)
 135         grouped[{x.d_name, x.d_type}].second.push_back(x);
 136
 137       di.backend->startTransaction(domain, -1);
 138       for(const auto g : grouped) {
 139         vector<DNSRecord> rrset;
 140         {
 141           DNSZoneRecord zrr;
 142           B.lookup(QType(g.first.second), g.first.first+domain, di.id);
 143           while(B.get(zrr)) {
 144             zrr.dr.d_name.makeUsRelative(domain);
 145             rrset.push_back(zrr.dr);
 146           }
 147         }
 148         // O(N^2)!
 149         rrset.erase(remove_if(rrset.begin(), rrset.end(),
 150                               [&g](const DNSRecord& dr) {
 151                                 return count(g.second.first.cbegin(),
 152                                              g.second.first.cend(), dr);
 153                               }), rrset.end());
 154         // the DNSRecord== operator compares on name, type, class and lowercase content representation
 155
 156         for(const auto& x : g.second.second) {
 157           rrset.push_back(x);
 158         }
 159
 160         vector<DNSResourceRecord> replacement;
 161         for(const auto& dr : rrset) {
 162           auto rr = DNSResourceRecord::fromWire(dr);
 163           rr.qname += domain;
 164           rr.domain_id = di.id;
 165           if(dr.d_type == QType::SOA) {
 166             //            cout<<"New SOA: "<<x.d_content->getZoneRepresentation()<<endl;
 167             auto sr = getRR<SOARecordContent>(dr);
 168             zs.soa_serial=sr->d_st.serial;
 169           }
 170
 171           replacement.push_back(rr);
 172         }
 173
 174         di.backend->replaceRRSet(di.id, g.first.first+domain, QType(g.first.second), replacement);
 175       }
 176       di.backend->commitTransaction();
 177     }
 178   }
 179   catch(std::exception& p) {
 180     g_log<<Logger::Error<<"Got exception during IXFR: "<<p.what()<<endl;
 181     throw;
 182   }
 183   catch(PDNSException& p) {
 184     g_log<<Logger::Error<<"Got exception during IXFR: "<<p.reason<<endl;
 185     throw;
 186   }
 187 }
 188
 189
 190 static bool processRecordForZS(const DNSName& domain, bool& firstNSEC3, DNSResourceRecord& rr, ZoneStatus& zs)
 191 {
 192   switch(rr.qtype.getCode()) {
 193   case QType::NSEC3PARAM:
 194     zs.ns3pr = NSEC3PARAMRecordContent(rr.content);
 195     zs.isDnssecZone = zs.isNSEC3 = true;
 196     zs.isNarrow = false;
 197     return false;
 198   case QType::NSEC3: {
 199     NSEC3RecordContent ns3rc(rr.content);
 200     if (firstNSEC3) {
 201       zs.isDnssecZone = zs.isPresigned = true;
 202       firstNSEC3 = false;
 203     } else if (zs.optOutFlag != (ns3rc.d_flags & 1))
 204       throw PDNSException("Zones with a mixture of Opt-Out NSEC3 RRs and non-Opt-Out NSEC3 RRs are not supported.");
 205     zs.optOutFlag = ns3rc.d_flags & 1;
 206     if (ns3rc.isSet(QType::NS) && !(rr.qname==domain)) {
 207       DNSName hashPart = rr.qname.makeRelative(domain);
 208       zs.secured.insert(hashPart);
 209     }
 210     return false;
 211   }
 212
 213   case QType::NSEC:
 214     zs.isDnssecZone = zs.isPresigned = true;
 215     return false;
 216
 217   case QType::NS:
 218     if(rr.qname!=domain)
 219       zs.nsset.insert(rr.qname);
 220     break;
 221   }
 222
 223   zs.qnames.insert(rr.qname);
 224
 225   rr.domain_id=zs.domain_id;
 226   return true;
 227 }
 228
 229 /* So this code does a number of things.
 230    1) It will AXFR a domain from a master
 231       The code can retrieve the current serial number in the database itself.
 232       It may attempt an IXFR
 233    2) It will filter the zone through a lua *filter* script
 234    3) The code walks through the zone records do determine DNSSEC status (secured, nsec/nsec3, optout)
 235    4) It inserts the zone into the database
 236       With the right 'ordername' fields
 237    5) It updates the Empty Non Terminals
 238 */
 239
 240 static vector<DNSResourceRecord> doAxfr(const ComboAddress& raddr, const DNSName& domain, const TSIGTriplet& tt, const ComboAddress& laddr,  unique_ptr<AuthLua4>& pdl, ZoneStatus& zs)
 241 {
 242   uint16_t axfr_timeout=::arg().asNum("axfr-fetch-timeout");
 243   vector<DNSResourceRecord> rrs;
 244   AXFRRetriever retriever(raddr, domain, tt, (laddr.sin4.sin_family == 0) ? NULL : &laddr, ((size_t) ::arg().asNum("xfr-max-received-mbytes")) * 1024 * 1024, axfr_timeout);
 245   Resolver::res_t recs;
 246   bool first=true;
 247   bool firstNSEC3{true};
 248   bool soa_received {false};
 249   while(retriever.getChunk(recs, nullptr, axfr_timeout)) {
 250     if(first) {
 251       g_log<<Logger::Error<<"AXFR started for '"<<domain<<"'"<<endl;
 252       first=false;
 253     }
 254
 255     for(Resolver::res_t::iterator i=recs.begin();i!=recs.end();++i) {
 256       i->qname.makeUsLowerCase();
 257       if(i->qtype.getCode() == QType::OPT || i->qtype.getCode() == QType::TSIG) // ignore EDNS0 & TSIG
 258         continue;
 259
 260       if(!i->qname.isPartOf(domain)) {
 261         g_log<<Logger::Error<<"Remote "<<raddr.toStringWithPort()<<" tried to sneak in out-of-zone data '"<<i->qname<<"'|"<<i->qtype.getName()<<" during AXFR of zone '"<<domain<<"', ignoring"<<endl;
 262         continue;
 263       }
 264
 265       vector<DNSResourceRecord> out;
 266       if(!pdl || !pdl->axfrfilter(raddr, domain, *i, out)) {
 267         out.push_back(*i); // if axfrfilter didn't do anything, we put our record in 'out' ourselves
 268       }
 269
 270       for(DNSResourceRecord& rr :  out) {
 271         if(!rr.qname.isPartOf(domain)) {
 272           g_log<<Logger::Error<<"Lua axfrfilter() filter tried to sneak in out-of-zone data '"<<i->qname<<"'|"<<i->qtype.getName()<<" during AXFR of zone '"<<domain<<"', ignoring"<<endl;
 273           continue;
 274         }
 275         if(!processRecordForZS(domain, firstNSEC3, rr, zs))
 276           continue;
 277         if(rr.qtype.getCode() == QType::SOA) {
 278           if(soa_received)
 279             continue; //skip the last SOA
 280           SOAData sd;
 281           fillSOAData(rr.content,sd);
 282           zs.soa_serial = sd.serial;
 283           soa_received = true;
 284         }
 285
 286         rrs.push_back(rr);
 287
 288       }
 289     }
 290   }
 291   return rrs;
 292 }
 293
 294
 295 void CommunicatorClass::suck(const DNSName &domain, const ComboAddress& remote)
 296 {
 297   {
 298     Lock l(&d_lock);
 299     if(d_inprogress.count(domain)) {
 300       return;
 301     }
 302     d_inprogress.insert(domain);
 303   }
 304   RemoveSentinel rs(domain, this); // this removes us from d_inprogress when we go out of scope
 305
 306   g_log<<Logger::Error<<"Initiating transfer of '"<<domain<<"' from remote '"<<remote<<"'"<<endl;
 307   UeberBackend B; // fresh UeberBackend
 308
 309   DomainInfo di;
 310   di.backend=0;
 311   bool transaction=false;
 312   try {
 313     DNSSECKeeper dk (&B); // reuse our UeberBackend copy for DNSSECKeeper
 314     bool wrongDomainKind = false;
 315     // this checks three error conditions & sets wrongDomainKind if we hit the third
 316     if(!B.getDomainInfo(domain, di) || !di.backend || (wrongDomainKind = true, di.kind != DomainInfo::Slave)) { // di.backend and B are mostly identical
 317       if(wrongDomainKind)
 318         g_log<<Logger::Error<<"Can't determine backend for domain '"<<domain<<"', not configured as slave"<<endl;
 319       else
 320         g_log<<Logger::Error<<"Can't determine backend for domain '"<<domain<<"'"<<endl;
 321       return;
 322     }
 323     ZoneStatus zs;
 324     zs.domain_id=di.id;
 325
 326     TSIGTriplet tt;
 327     if(dk.getTSIGForAccess(domain, remote, &tt.name)) {
 328       string tsigsecret64;
 329       if(B.getTSIGKey(tt.name, &tt.algo, &tsigsecret64)) {
 330         if(B64Decode(tsigsecret64, tt.secret)) {
 331           g_log<<Logger::Error<<"Unable to Base-64 decode TSIG key '"<<tt.name<<"' for domain '"<<domain<<"' not found"<<endl;
 332           return;
 333         }
 334       } else {
 335         g_log<<Logger::Error<<"TSIG key '"<<tt.name<<"' for domain '"<<domain<<"' not found"<<endl;
 336         return;
 337       }
 338     }
 339
 340
 341     unique_ptr<AuthLua4> pdl{nullptr};
 342     vector<string> scripts;
 343     string script=::arg()["lua-axfr-script"];
 344     if(B.getDomainMetadata(domain, "LUA-AXFR-SCRIPT", scripts) && !scripts.empty()) {
 345       if (pdns_iequals(scripts[0], "NONE")) {
 346         script.clear();
 347       } else {
 348         script=scripts[0];
 349       }
 350     }
 351     if(!script.empty()){
 352       try {
 353         pdl = make_unique<AuthLua4>();
 354         pdl->loadFile(script);
 355         g_log<<Logger::Info<<"Loaded Lua script '"<<script<<"' to edit the incoming AXFR of '"<<domain<<"'"<<endl;
 356       }
 357       catch(std::exception& e) {
 358         g_log<<Logger::Error<<"Failed to load Lua editing script '"<<script<<"' for incoming AXFR of '"<<domain<<"': "<<e.what()<<endl;
 359         return;
 360       }
 361     }
 362
 363     vector<string> localaddr;
 364     ComboAddress laddr;
 365
 366     if(B.getDomainMetadata(domain, "AXFR-SOURCE", localaddr) && !localaddr.empty()) {
 367       try {
 368         laddr = ComboAddress(localaddr[0]);
 369         g_log<<Logger::Info<<"AXFR source for domain '"<<domain<<"' set to "<<localaddr[0]<<endl;
 370       }
 371       catch(std::exception& e) {
 372         g_log<<Logger::Error<<"Failed to load AXFR source '"<<localaddr[0]<<"' for incoming AXFR of '"<<domain<<"': "<<e.what()<<endl;
 373         return;
 374       }
 375     } else {
 376       if(remote.sin4.sin_family == AF_INET && !::arg()["query-local-address"].empty()) {
 377         laddr = ComboAddress(::arg()["query-local-address"]);
 378       } else if(remote.sin4.sin_family == AF_INET6 && !::arg()["query-local-address6"].empty()) {
 379         laddr = ComboAddress(::arg()["query-local-address6"]);
 380       } else {
 381         bool isv6 = remote.sin4.sin_family == AF_INET6;
 382         g_log<<Logger::Error<<"Unable to AXFR, destination address is IPv" << (isv6 ? "6" : "4") << ", but query-local-address"<< (isv6 ? "6" : "") << " is unset!"<<endl;
 383         return;
 384       }
 385     }
 386
 387     bool hadDnssecZone = false;
 388     bool hadPresigned = false;
 389     bool hadNSEC3 = false;
 390     NSEC3PARAMRecordContent hadNs3pr;
 391     bool hadNarrow=false;
 392
 393
 394     vector<DNSResourceRecord> rrs;
 395     if(dk.isSecuredZone(domain)) {
 396       hadDnssecZone=true;
 397       hadPresigned=dk.isPresigned(domain);
 398       if (dk.getNSEC3PARAM(domain, &zs.ns3pr, &zs.isNarrow)) {
 399         hadNSEC3 = true;
 400         hadNs3pr = zs.ns3pr;
 401         hadNarrow = zs.isNarrow;
 402       }
 403     }
 404     else if(di.serial) {
 405       vector<string> meta;
 406       B.getDomainMetadata(domain, "IXFR", meta);
 407       if(!meta.empty() && meta[0]=="1") {
 408         vector<DNSRecord> axfr;
 409         g_log<<Logger::Warning<<"Starting IXFR of '"<<domain<<"' from remote "<<remote<<endl;
 410         ixfrSuck(domain, tt, laddr, remote, pdl, zs, &axfr);
 411         if(!axfr.empty()) {
 412           g_log<<Logger::Warning<<"IXFR of '"<<domain<<"' from remote '"<<remote<<"' turned into an AXFR"<<endl;
 413           bool firstNSEC3=true;
 414           rrs.reserve(axfr.size());
 415           for(const auto& dr : axfr) {
 416             auto rr = DNSResourceRecord::fromWire(dr);
 417             (rr.qname += domain).makeUsLowerCase();
 418             rr.domain_id = zs.domain_id;
 419             if(!processRecordForZS(domain, firstNSEC3, rr, zs))
 420               continue;
 421             if(dr.d_type == QType::SOA) {
 422               auto sd = getRR<SOARecordContent>(dr);
 423               zs.soa_serial = sd->d_st.serial;
 424             }
 425             rrs.push_back(rr);
 426           }
 427         }
 428         else {
 429           g_log<<Logger::Warning<<"Done with IXFR of '"<<domain<<"' from remote '"<<remote<<"', got "<<zs.numDeltas<<" delta"<<addS(zs.numDeltas)<<", serial now "<<zs.soa_serial<<endl;
 430           purgeAuthCaches(domain.toString()+"$");
 431           return;
 432         }
 433       }
 434     }
 435
 436     if(rrs.empty()) {
 437       g_log<<Logger::Warning<<"Starting AXFR of '"<<domain<<"' from remote "<<remote<<endl;
 438       rrs = doAxfr(remote, domain, tt, laddr, pdl, zs);
 439       g_log<<Logger::Warning<<"AXFR of '"<<domain<<"' from remote "<<remote<<" done"<<endl;
 440     }
 441
 442     if(zs.isNSEC3) {
 443       zs.ns3pr.d_flags = zs.optOutFlag ? 1 : 0;
 444     }
 445
 446     if(!zs.isPresigned) {
 447       DNSSECKeeper::keyset_t keys = dk.getKeys(domain);
 448       if(!keys.empty()) {
 449         zs.isDnssecZone = true;
 450         zs.isNSEC3 = hadNSEC3;
 451         zs.ns3pr = hadNs3pr;
 452         zs.optOutFlag = (hadNs3pr.d_flags & 1);
 453         zs.isNarrow = hadNarrow;
 454       }
 455     }
 456
 457     if(zs.isDnssecZone) {
 458       if(!zs.isNSEC3)
 459         g_log<<Logger::Info<<"Adding NSEC ordering information"<<endl;
 460       else if(!zs.isNarrow)
 461         g_log<<Logger::Info<<"Adding NSEC3 hashed ordering information for '"<<domain<<"'"<<endl;
 462       else
 463         g_log<<Logger::Info<<"Erasing NSEC3 ordering since we are narrow, only setting 'auth' fields"<<endl;
 464     }
 465
 466
 467     transaction=di.backend->startTransaction(domain, zs.domain_id);
 468     g_log<<Logger::Error<<"Backend transaction started for '"<<domain<<"' storage"<<endl;
 469
 470     // update the presigned flag and NSEC3PARAM
 471     if (zs.isDnssecZone) {
 472       // update presigned if there was a change
 473       if (zs.isPresigned && !hadPresigned) {
 474         // zone is now presigned
 475         dk.setPresigned(domain);
 476       } else if (hadPresigned && !zs.isPresigned) {
 477         // zone is no longer presigned
 478         dk.unsetPresigned(domain);
 479       }
 480       // update NSEC3PARAM
 481       if (zs.isNSEC3) {
 482         // zone is NSEC3, only update if there was a change
 483         if (!hadNSEC3 || (hadNarrow  != zs.isNarrow) ||
 484             (zs.ns3pr.d_algorithm != hadNs3pr.d_algorithm) ||
 485             (zs.ns3pr.d_flags != hadNs3pr.d_flags) ||
 486             (zs.ns3pr.d_iterations != hadNs3pr.d_iterations) ||
 487             (zs.ns3pr.d_salt != hadNs3pr.d_salt)) {
 488           dk.setNSEC3PARAM(domain, zs.ns3pr, zs.isNarrow);
 489         }
 490       } else if (hadNSEC3 ) {
 491          // zone is no longer NSEC3
 492          dk.unsetNSEC3PARAM(domain);
 493       }
 494     } else if (hadDnssecZone) {
 495       // zone is no longer signed
 496       if (hadPresigned) {
 497         // remove presigned
 498         dk.unsetPresigned(domain);
 499       }
 500       if (hadNSEC3) {
 501         // unset NSEC3PARAM
 502         dk.unsetNSEC3PARAM(domain);
 503       }
 504     }
 505
 506     bool doent=true;
 507     uint32_t maxent = ::arg().asNum("max-ent-entries");
 508     DNSName shorter, ordername;
 509     set<DNSName> rrterm;
 510     map<DNSName,bool> nonterm;
 511
 512
 513     for(DNSResourceRecord& rr :  rrs) {
 514       if(!zs.isPresigned) {
 515         if (rr.qtype.getCode() == QType::RRSIG)
 516           continue;
 517         if(zs.isDnssecZone && rr.qtype.getCode() == QType::DNSKEY && !::arg().mustDo("direct-dnskey"))
 518           continue;
 519       }
 520
 521       // Figure out auth and ents
 522       rr.auth=true;
 523       shorter=rr.qname;
 524       rrterm.clear();
 525       do {
 526         if(doent) {
 527           if (!zs.qnames.count(shorter))
 528             rrterm.insert(shorter);
 529         }
 530         if(zs.nsset.count(shorter) && rr.qtype.getCode() != QType::DS)
 531           rr.auth=false;
 532
 533         if (shorter==domain) // stop at apex
 534           break;
 535       }while(shorter.chopOff());
 536
 537       // Insert ents
 538       if(doent && !rrterm.empty()) {
 539         bool auth;
 540         if (!rr.auth && rr.qtype.getCode() == QType::NS) {
 541           if (zs.isNSEC3)
 542             ordername=DNSName(toBase32Hex(hashQNameWithSalt(zs.ns3pr, rr.qname)));
 543           auth=(!zs.isNSEC3 || !zs.optOutFlag || zs.secured.count(ordername));
 544         } else
 545           auth=rr.auth;
 546
 547         for(const auto &nt: rrterm){
 548           if (!nonterm.count(nt))
 549               nonterm.insert(pair<DNSName, bool>(nt, auth));
 550             else if (auth)
 551               nonterm[nt]=true;
 552         }
 553
 554         if(nonterm.size() > maxent) {
 555           g_log<<Logger::Error<<"AXFR zone "<<domain<<" has too many empty non terminals."<<endl;
 556           nonterm.clear();
 557           doent=false;
 558         }
 559       }
 560
 561       // RRSIG is always auth, even inside a delegation
 562       if (rr.qtype.getCode() == QType::RRSIG)
 563         rr.auth=true;
 564
 565       // Add ordername and insert record
 566       if (zs.isDnssecZone && rr.qtype.getCode() != QType::RRSIG) {
 567         if (zs.isNSEC3) {
 568           // NSEC3
 569           ordername=DNSName(toBase32Hex(hashQNameWithSalt(zs.ns3pr, rr.qname)));
 570           if(!zs.isNarrow && (rr.auth || (rr.qtype.getCode() == QType::NS && (!zs.optOutFlag || zs.secured.count(ordername))))) {
 571             di.backend->feedRecord(rr, ordername, true);
 572           } else
 573             di.backend->feedRecord(rr, DNSName());
 574         } else {
 575           // NSEC
 576           if (rr.auth || rr.qtype.getCode() == QType::NS) {
 577             ordername=rr.qname.makeRelative(domain);
 578             di.backend->feedRecord(rr, ordername);
 579           } else
 580             di.backend->feedRecord(rr, DNSName());
 581         }
 582       } else
 583         di.backend->feedRecord(rr, DNSName());
 584     }
 585
 586     // Insert empty non-terminals
 587     if(doent && !nonterm.empty()) {
 588       if (zs.isNSEC3) {
 589         di.backend->feedEnts3(zs.domain_id, domain, nonterm, zs.ns3pr, zs.isNarrow);
 590       } else
 591         di.backend->feedEnts(zs.domain_id, nonterm);
 592     }
 593
 594     di.backend->commitTransaction();
 595     transaction = false;
 596     di.backend->setFresh(zs.domain_id);
 597     purgeAuthCaches(domain.toString()+"$");
 598
 599     g_log<<Logger::Error<<"AXFR done for '"<<domain<<"', zone committed with serial number "<<zs.soa_serial<<endl;
 600
 601     // Send slave re-notifications
 602     bool doNotify;
 603     vector<string> meta;
 604     if(B.getDomainMetadata(domain, "SLAVE-RENOTIFY", meta ) && !meta.empty()) {
 605       doNotify=(meta.front() == "1");
 606     } else {
 607       doNotify=(::arg().mustDo("slave-renotify"));
 608     }
 609     if(doNotify) {
 610       notifyDomain(domain, &B);
 611     }
 612
 613   }
 614   catch(DBException &re) {
 615     g_log<<Logger::Error<<"Unable to feed record during incoming AXFR of '" << domain<<"': "<<re.reason<<endl;
 616     if(di.backend && transaction) {
 617       g_log<<Logger::Error<<"Aborting possible open transaction for domain '"<<domain<<"' AXFR"<<endl;
 618       di.backend->abortTransaction();
 619     }
 620   }
 621   catch(const MOADNSException &mde) {
 622     g_log<<Logger::Error<<"Unable to parse record during incoming AXFR of '"<<domain<<"' (MOADNSException): "<<mde.what()<<endl;
 623     if(di.backend && transaction) {
 624       g_log<<Logger::Error<<"Aborting possible open transaction for domain '"<<domain<<"' AXFR"<<endl;
 625       di.backend->abortTransaction();
 626     }
 627   }
 628   catch(std::exception &re) {
 629     g_log<<Logger::Error<<"Unable to parse record during incoming AXFR of '"<<domain<<"' (std::exception): "<<re.what()<<endl;
 630     if(di.backend && transaction) {
 631       g_log<<Logger::Error<<"Aborting possible open transaction for domain '"<<domain<<"' AXFR"<<endl;
 632       di.backend->abortTransaction();
 633     }
 634   }
 635   catch(ResolverException &re) {
 636     {
 637       Lock l(&d_lock);
 638       // The AXFR probably failed due to a problem on the master server. If SOA-checks against this master
 639       // still succeed, we would constantly try to AXFR the zone. To avoid this, we add the zone to the list of
 640       // failed slave-checks. This will suspend slave-checks (and subsequent AXFR) for this zone for some time.
 641       uint64_t newCount = 1;
 642       time_t now = time(0);
 643       const auto failedEntry = d_failedSlaveRefresh.find(domain);
 644       if (failedEntry != d_failedSlaveRefresh.end())
 645         newCount = d_failedSlaveRefresh[domain].first + 1;
 646       time_t nextCheck = now + std::min(newCount * d_tickinterval, (uint64_t)::arg().asNum("soa-retry-default"));
 647       d_failedSlaveRefresh[domain] = {newCount, nextCheck};
 648       g_log<<Logger::Error<<"Unable to AXFR zone '"<<domain<<"' from remote '"<<remote<<"' (resolver): "<<re.reason<<" (This was the "<<(newCount == 1 ? "first" : std::to_string(newCount) + "th")<<" time. Excluding zone from slave-checks until "<<nextCheck<<")"<<endl;
 649     }
 650     if(di.backend && transaction) {
 651       g_log<<Logger::Error<<"Aborting possible open transaction for domain '"<<domain<<"' AXFR"<<endl;
 652       di.backend->abortTransaction();
 653     }
 654   }
 655   catch(PDNSException &ae) {
 656     g_log<<Logger::Error<<"Unable to AXFR zone '"<<domain<<"' from remote '"<<remote<<"' (PDNSException): "<<ae.reason<<endl;
 657     if(di.backend && transaction) {
 658       g_log<<Logger::Error<<"Aborting possible open transaction for domain '"<<domain<<"' AXFR"<<endl;
 659       di.backend->abortTransaction();
 660     }
 661   }
 662 }
 663 namespace {
 664 struct DomainNotificationInfo
 665 {
 666   DomainInfo di;
 667   bool dnssecOk;
 668   ComboAddress localaddr;
 669   DNSName tsigkeyname, tsigalgname;
 670   string tsigsecret;
 671 };
 672 }
 673
 674
 675 struct SlaveSenderReceiver
 676 {
 677   typedef std::tuple<DNSName, ComboAddress, uint16_t> Identifier;
 678
 679   struct Answer {
 680     uint32_t theirSerial;
 681     uint32_t theirInception;
 682     uint32_t theirExpire;
 683   };
 684
 685   map<uint32_t, Answer> d_freshness;
 686
 687   SlaveSenderReceiver()
 688   {
 689   }
 690
 691   void deliverTimeout(const Identifier& i)
 692   {
 693   }
 694
 695   Identifier send(DomainNotificationInfo& dni)
 696   {
 697     random_shuffle(dni.di.masters.begin(), dni.di.masters.end());
 698     try {
 699       return std::make_tuple(dni.di.zone,
 700                              *dni.di.masters.begin(),
 701                              d_resolver.sendResolve(*dni.di.masters.begin(),
 702                                                     dni.localaddr,
 703                                                     dni.di.zone,
 704                                                     QType::SOA,
 705                                                     nullptr,
 706                                                     dni.dnssecOk, dni.tsigkeyname, dni.tsigalgname, dni.tsigsecret)
 707         );
 708     }
 709     catch(PDNSException& e) {
 710       throw runtime_error("While attempting to query freshness of '"+dni.di.zone.toLogString()+"': "+e.reason);
 711     }
 712   }
 713
 714   bool receive(Identifier& id, Answer& a)
 715   {
 716     if(d_resolver.tryGetSOASerial(&(std::get<0>(id)), &(std::get<1>(id)), &a.theirSerial, &a.theirInception, &a.theirExpire, &(std::get<2>(id)))) {
 717       return 1;
 718     }
 719     return 0;
 720   }
 721
 722   void deliverAnswer(DomainNotificationInfo& dni, const Answer& a, unsigned int usec)
 723   {
 724     d_freshness[dni.di.id]=a;
 725   }
 726
 727   Resolver d_resolver;
 728 };
 729
 730 void CommunicatorClass::addSlaveCheckRequest(const DomainInfo& di, const ComboAddress& remote)
 731 {
 732   Lock l(&d_lock);
 733   DomainInfo ours = di;
 734   ours.backend = 0;
 735
 736   // When adding a check, if the remote addr from which notification was
 737   // received is a master, clear all other masters so we can be sure the
 738   // query goes to that one.
 739   for (const auto& master : di.masters) {
 740     if (ComboAddress::addressOnlyEqual()(remote, master)) {
 741       ours.masters.clear();
 742       ours.masters.push_back(master);
 743       break;
 744     }
 745   }
 746   d_tocheck.erase(di);
 747   d_tocheck.insert(ours);
 748   d_any_sem.post(); // kick the loop!
 749 }
 750
 751 void CommunicatorClass::addTrySuperMasterRequest(const DNSPacket& p)
 752 {
 753   Lock l(&d_lock);
 754   DNSPacket ours = p;
 755   if(d_potentialsupermasters.insert(ours).second)
 756     d_any_sem.post(); // kick the loop!
 757 }
 758
 759 void CommunicatorClass::slaveRefresh(PacketHandler *P)
 760 {
 761   // not unless we are slave
 762   if (!::arg().mustDo("slave")) return;
 763
 764   UeberBackend *B=P->getBackend();
 765   vector<DomainInfo> rdomains;
 766   vector<DomainNotificationInfo> sdomains;
 767   set<DNSPacket, cmp> trysuperdomains;
 768   {
 769     Lock l(&d_lock);
 770     set<DomainInfo> requeue;
 771     for(const auto& di: d_tocheck) {
 772       if(d_inprogress.count(di.zone)) {
 773         g_log<<Logger::Debug<<"Got NOTIFY for "<<di.zone<<" while AXFR in progress, requeueing SOA check"<<endl;
 774         requeue.insert(di);
 775       }
 776       else {
 777         // We received a NOTIFY for a zone. This means at least one of the zone's master server is working.
 778         // Therefore we delete the zone from the list of failed slave-checks to allow immediate checking.
 779         const auto wasFailedDomain = d_failedSlaveRefresh.find(di.zone);
 780         if (wasFailedDomain != d_failedSlaveRefresh.end()) {
 781           g_log<<Logger::Debug<<"Got NOTIFY for "<<di.zone<<", removing zone from list of failed slave-checks and going to check SOA serial"<<endl;
 782           d_failedSlaveRefresh.erase(di.zone);
 783         } else {
 784           g_log<<Logger::Debug<<"Got NOTIFY for "<<di.zone<<", going to check SOA serial"<<endl;
 785         }
 786         rdomains.push_back(di);
 787       }
 788     }
 789     d_tocheck.swap(requeue);
 790
 791     trysuperdomains = d_potentialsupermasters;
 792     d_potentialsupermasters.clear();
 793   }
 794
 795   for(const DNSPacket& dp :  trysuperdomains) {
 796     // get the TSIG key name
 797     TSIGRecordContent trc;
 798     DNSName tsigkeyname;
 799     dp.getTSIGDetails(&trc, &tsigkeyname);
 800     P->trySuperMasterSynchronous(dp, tsigkeyname); // FIXME could use some error loging
 801   }
 802   if(rdomains.empty()) { // if we have priority domains, check them first
 803     B->getUnfreshSlaveInfos(&rdomains);
 804   }
 805   DNSSECKeeper dk(B); // NOW HEAR THIS! This DK uses our B backend, so no interleaved access!
 806   {
 807     Lock l(&d_lock);
 808     domains_by_name_t& nameindex=boost::multi_index::get<IDTag>(d_suckdomains);
 809     time_t now = time(0);
 810
 811     for(DomainInfo& di :  rdomains) {
 812       const auto failed = d_failedSlaveRefresh.find(di.zone);
 813       if (failed != d_failedSlaveRefresh.end() && now < failed->second.second ) {
 814         // If the domain has failed before and the time before the next check has not expired, skip this domain
 815         g_log<<Logger::Debug<<"Zone '"<<di.zone<<"' is on the list of failed SOA checks. Skipping SOA checks until "<< failed->second.second<<endl;
 816         continue;
 817       }
 818       std::vector<std::string> localaddr;
 819       SuckRequest sr;
 820       sr.domain=di.zone;
 821       if(di.masters.empty()) // slave domains w/o masters are ignored
 822         continue;
 823       // remove unfresh domains already queued for AXFR, no sense polling them again
 824       sr.master=*di.masters.begin();
 825       if(nameindex.count(sr)) {  // this does NOT however protect us against AXFRs already in progress!
 826         continue;
 827       }
 828       if(d_inprogress.count(sr.domain)) // this does
 829         continue;
 830
 831       DomainNotificationInfo dni;
 832       dni.di=di;
 833       dni.dnssecOk = dk.doesDNSSEC();
 834
 835       if(dk.getTSIGForAccess(di.zone, sr.master, &dni.tsigkeyname)) {
 836         string secret64;
 837         if(!B->getTSIGKey(dni.tsigkeyname, &dni.tsigalgname, &secret64)) {
 838           g_log<<Logger::Error<<"TSIG key '"<<dni.tsigkeyname<<"' for domain '"<<di.zone<<"' not found, can not AXFR."<<endl;
 839           continue;
 840         }
 841         if (B64Decode(secret64, dni.tsigsecret) == -1) {
 842           g_log<<Logger::Error<<"Unable to Base-64 decode TSIG key '"<<dni.tsigkeyname<<"' for domain '"<<di.zone<<"', can not AXFR."<<endl;
 843           continue;
 844         }
 845       }
 846
 847       localaddr.clear();
 848       // check for AXFR-SOURCE
 849       if(B->getDomainMetadata(di.zone, "AXFR-SOURCE", localaddr) && !localaddr.empty()) {
 850         try {
 851           dni.localaddr = ComboAddress(localaddr[0]);
 852           g_log<<Logger::Info<<"Freshness check source (AXFR-SOURCE) for domain '"<<di.zone<<"' set to "<<localaddr[0]<<endl;
 853         }
 854         catch(std::exception& e) {
 855           g_log<<Logger::Error<<"Failed to load freshness check source '"<<localaddr[0]<<"' for '"<<di.zone<<"': "<<e.what()<<endl;
 856           return;
 857         }
 858       } else {
 859         dni.localaddr.sin4.sin_family = 0;
 860       }
 861
 862       sdomains.push_back(dni);
 863     }
 864   }
 865   if(sdomains.empty())
 866   {
 867     if(d_slaveschanged) {
 868       Lock l(&d_lock);
 869       g_log<<Logger::Warning<<"No new unfresh slave domains, "<<d_suckdomains.size()<<" queued for AXFR already, "<<d_inprogress.size()<<" in progress"<<endl;
 870     }
 871     d_slaveschanged = !rdomains.empty();
 872     return;
 873   }
 874   else {
 875     Lock l(&d_lock);
 876     g_log<<Logger::Warning<<sdomains.size()<<" slave domain"<<(sdomains.size()>1 ? "s" : "")<<" need"<<
 877       (sdomains.size()>1 ? "" : "s")<<
 878       " checking, "<<d_suckdomains.size()<<" queued for AXFR"<<endl;
 879   }
 880
 881   SlaveSenderReceiver ssr;
 882
 883   Inflighter<vector<DomainNotificationInfo>, SlaveSenderReceiver> ifl(sdomains, ssr);
 884
 885   ifl.d_maxInFlight = 200;
 886
 887   for(;;) {
 888     try {
 889       ifl.run();
 890       break;
 891     }
 892     catch(std::exception& e) {
 893       g_log<<Logger::Error<<"While checking domain freshness: " << e.what()<<endl;
 894     }
 895     catch(PDNSException &re) {
 896       g_log<<Logger::Error<<"While checking domain freshness: " << re.reason<<endl;
 897     }
 898   }
 899   g_log<<Logger::Warning<<"Received serial number updates for "<<ssr.d_freshness.size()<<" zone"<<addS(ssr.d_freshness.size())<<", had "<<ifl.getTimeouts()<<" timeout"<<addS(ifl.getTimeouts())<<endl;
 900
 901   typedef DomainNotificationInfo val_t;
 902   time_t now = time(0);
 903   for(val_t& val :  sdomains) {
 904     DomainInfo& di(val.di);
 905     DomainInfo tempdi;
 906     // might've come from the packethandler
 907     // Please do not overwrite received DI just to make sure it exists in backend.
 908     if(!di.backend) {
 909       if (!B->getDomainInfo(di.zone, tempdi)) {
 910         g_log<<Logger::Warning<<"Ignore domain "<< di.zone<<" since it has been removed from our backend"<<endl;
 911         continue;
 912       }
 913       // Backend for di still doesn't exist and this might cause us to
 914       // SEGFAULT on the setFresh command later on
 915       di.backend = tempdi.backend;
 916     }
 917
 918     if(!ssr.d_freshness.count(di.id)) { // If we don't have an answer for the domain
 919       uint64_t newCount = 1;
 920       Lock l(&d_lock);
 921       const auto failedEntry = d_failedSlaveRefresh.find(di.zone);
 922       if (failedEntry != d_failedSlaveRefresh.end())
 923         newCount = d_failedSlaveRefresh[di.zone].first + 1;
 924       time_t nextCheck = now + std::min(newCount * d_tickinterval, (uint64_t)::arg().asNum("soa-retry-default"));
 925       d_failedSlaveRefresh[di.zone] = {newCount, nextCheck};
 926       if (newCount == 1) {
 927         g_log<<Logger::Warning<<"Unable to retrieve SOA for "<<di.zone<<
 928           ", this was the first time. NOTE: For every subsequent failed SOA check the domain will be suspended from freshness checks for 'num-errors x "<<
 929           d_tickinterval<<" seconds', with a maximum of "<<(uint64_t)::arg().asNum("soa-retry-default")<<" seconds. Skipping SOA checks until "<<nextCheck<<endl;
 930       } else if (newCount % 10 == 0) {
 931         g_log<<Logger::Warning<<"Unable to retrieve SOA for "<<di.zone<<", this was the "<<std::to_string(newCount)<<"th time. Skipping SOA checks until "<<nextCheck<<endl;
 932       }
 933       continue;
 934     }
 935
 936     {
 937       Lock l(&d_lock);
 938       const auto wasFailedDomain = d_failedSlaveRefresh.find(di.zone);
 939       if (wasFailedDomain != d_failedSlaveRefresh.end())
 940         d_failedSlaveRefresh.erase(di.zone);
 941     }
 942
 943     bool hasSOA = false;
 944     SOAData sd;
 945     try{
 946       hasSOA = B->getSOA(di.zone, sd);
 947     }
 948     catch(...) {}
 949
 950     uint32_t theirserial = ssr.d_freshness[di.id].theirSerial, ourserial = sd.serial;
 951
 952     if(rfc1982LessThan(theirserial, ourserial) && ourserial != 0 && !::arg().mustDo("axfr-lower-serial"))  {
 953       g_log<<Logger::Error<<"Domain '"<<di.zone<<"' more recent than master, our serial " << ourserial << " > their serial "<< theirserial << endl;
 954       di.backend->setFresh(di.id);
 955     }
 956     else if(hasSOA && theirserial == ourserial) {
 957       uint32_t maxExpire=0, maxInception=0;
 958       if(dk.isPresigned(di.zone)) {
 959         B->lookup(QType(QType::RRSIG), di.zone, di.id); // can't use DK before we are done with this lookup!
 960         DNSZoneRecord zr;
 961         while(B->get(zr)) {
 962           auto rrsig = getRR<RRSIGRecordContent>(zr.dr);
 963           if(rrsig->d_type == QType::SOA) {
 964             maxInception = std::max(maxInception, rrsig->d_siginception);
 965             maxExpire = std::max(maxExpire, rrsig->d_sigexpire);
 966           }
 967         }
 968       }
 969       if(! maxInception && ! ssr.d_freshness[di.id].theirInception) {
 970         g_log<<Logger::Info<<"Domain '"<< di.zone<<"' is fresh (no DNSSEC), serial is "<<ourserial<<endl;
 971         di.backend->setFresh(di.id);
 972       }
 973       else if(maxInception == ssr.d_freshness[di.id].theirInception && maxExpire == ssr.d_freshness[di.id].theirExpire) {
 974         g_log<<Logger::Info<<"Domain '"<< di.zone<<"' is fresh and SOA RRSIGs match, serial is "<<ourserial<<endl;
 975         di.backend->setFresh(di.id);
 976       }
 977       else if(maxExpire >= now && ! ssr.d_freshness[di.id].theirInception ) {
 978         g_log<<Logger::Info<<"Domain '"<< di.zone<<"' is fresh, master is no longer signed but (some) signatures are still vallid, serial is "<<ourserial<<endl;
 979         di.backend->setFresh(di.id);
 980       }
 981       else if(maxInception && ! ssr.d_freshness[di.id].theirInception ) {
 982         g_log<<Logger::Warning<<"Domain '"<< di.zone<<"' is stale, master is no longer signed and all signatures have expired, serial is "<<ourserial<<endl;
 983         addSuckRequest(di.zone, *di.masters.begin());
 984       }
 985       else if(dk.doesDNSSEC() && ! maxInception && ssr.d_freshness[di.id].theirInception) {
 986         g_log<<Logger::Warning<<"Domain '"<< di.zone<<"' is stale, master has signed, serial is "<<ourserial<<endl;
 987         addSuckRequest(di.zone, *di.masters.begin());
 988       }
 989       else {
 990         g_log<<Logger::Warning<<"Domain '"<< di.zone<<"' is fresh, but RRSIGs differ, so DNSSEC is stale, serial is "<<ourserial<<endl;
 991         addSuckRequest(di.zone, *di.masters.begin());
 992       }
 993     }
 994     else {
 995       if(hasSOA) {
 996         g_log<<Logger::Warning<<"Domain '"<< di.zone<<"' is stale, master serial "<<theirserial<<", our serial "<< ourserial <<endl;
 997       }
 998       else {
 999         g_log<<Logger::Warning<<"Domain '"<< di.zone<<"' is empty, master serial "<<theirserial<<endl;
1000       }
1001       addSuckRequest(di.zone, *di.masters.begin());
1002     }
1003   }
1004 }