pdns/slavecommunicator.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #ifdef HAVE_CONFIG_H
  23 #include "config.h"
  24 #endif
  25
  26 #include "utility.hh"
  27 #include "dnssecinfra.hh"
  28 #include "dnsseckeeper.hh"
  29 #include "base32.hh"
  30 #include <errno.h>
  31 #include "communicator.hh"
  32 #include <set>
  33 #include <boost/utility.hpp>
  34 #include "dnsbackend.hh"
  35 #include "ueberbackend.hh"
  36 #include "packethandler.hh"
  37 #include "resolver.hh"
  38 #include "logger.hh"
  39 #include "dns.hh"
  40 #include "arguments.hh"
  41 #include "auth-caches.hh"
  42
  43 #include "base64.hh"
  44 #include "inflighter.cc"
  45 #include "namespaces.hh"
  46 #include "common_startup.hh"
  47
  48 #include "ixfr.hh"
  49 using boost::scoped_ptr;
  50
  51
  52 void CommunicatorClass::addSuckRequest(const DNSName &domain, const ComboAddress& master)
  53 {
  54   Lock l(&d_lock);
  55   SuckRequest sr;
  56   sr.domain = domain;
  57   sr.master = master;
  58   pair<UniQueue::iterator, bool>  res;
  59
  60   res=d_suckdomains.push_back(sr);
  61   if(res.second) {
  62     d_suck_sem.post();
  63   }
  64
  65 }
  66
  67 struct ZoneStatus
  68 {
  69   bool isDnssecZone{false};
  70   bool isPresigned{false};
  71   bool isNSEC3 {false};
  72   bool optOutFlag {false};
  73   NSEC3PARAMRecordContent ns3pr;
  74
  75   bool isNarrow{false};
  76   unsigned int soa_serial{0};
  77   set<DNSName> nsset, qnames, secured;
  78   uint32_t domain_id;
  79   int numDeltas{0};
  80 };
  81
  82
  83 void CommunicatorClass::ixfrSuck(const DNSName &domain, const TSIGTriplet& tt, const ComboAddress& laddr, const ComboAddress& remote, scoped_ptr<AuthLua4>& pdl,
  84                                  ZoneStatus& zs, vector<DNSRecord>* axfr)
  85 {
  86   UeberBackend B; // fresh UeberBackend
  87
  88   DomainInfo di;
  89   di.backend=0;
  90   //  bool transaction=false;
  91   try {
  92     DNSSECKeeper dk (&B); // reuse our UeberBackend copy for DNSSECKeeper
  93
  94     bool wrongDomainKind = false;
  95     // this checks three error conditions, and sets wrongDomainKind if we hit the third & had an error
  96     if(!B.getDomainInfo(domain, di) || !di.backend || (wrongDomainKind = true, di.kind != DomainInfo::Slave)) { // di.backend and B are mostly identical
  97       if(wrongDomainKind)
  98         g_log<<Logger::Error<<"Can't determine backend for domain '"<<domain<<"', not configured as slave"<<endl;
  99       else
 100         g_log<<Logger::Error<<"Can't determine backend for domain '"<<domain<<"'"<<endl;
 101       return;
 102     }
 103
 104     soatimes st;
 105     memset(&st, 0, sizeof(st));
 106     st.serial=di.serial;
 107
 108     DNSRecord drsoa;
 109     drsoa.d_content = std::make_shared<SOARecordContent>(g_rootdnsname, g_rootdnsname, st);
 110     auto deltas = getIXFRDeltas(remote, domain, drsoa, tt, laddr.sin4.sin_family ? &laddr : 0, ((size_t) ::arg().asNum("xfr-max-received-mbytes")) * 1024 * 1024);
 111     zs.numDeltas=deltas.size();
 112     //    cout<<"Got "<<deltas.size()<<" deltas from serial "<<di.serial<<", applying.."<<endl;
 113
 114     for(const auto& d : deltas) {
 115       const auto& remove = d.first;
 116       const auto& add = d.second;
 117       //      cout<<"Delta sizes: "<<remove.size()<<", "<<add.size()<<endl;
 118
 119       if(remove.empty()) { // we got passed an AXFR!
 120         *axfr = add;
 121         return;
 122       }
 123
 124
 125       // our hammer is 'replaceRRSet(domain_id, qname, qt, vector<DNSResourceRecord>& rrset)
 126       // which thinks in terms of RRSETs
 127       // however, IXFR does not, and removes and adds *records* (bummer)
 128       // this means that we must group updates by {qname,qtype}, retrieve the RRSET, apply
 129       // the add/remove updates, and replaceRRSet the whole thing.
 130
 131
 132       map<pair<DNSName,uint16_t>, pair<vector<DNSRecord>, vector<DNSRecord> > > grouped;
 133
 134       for(const auto& x: remove)
 135         grouped[{x.d_name, x.d_type}].first.push_back(x);
 136       for(const auto& x: add)
 137         grouped[{x.d_name, x.d_type}].second.push_back(x);
 138
 139       di.backend->startTransaction(domain, -1);
 140       for(const auto g : grouped) {
 141         vector<DNSRecord> rrset;
 142         {
 143           DNSZoneRecord zrr;
 144           B.lookup(QType(g.first.second), g.first.first+domain, 0, di.id);
 145           while(B.get(zrr)) {
 146             zrr.dr.d_name.makeUsRelative(domain);
 147             rrset.push_back(zrr.dr);
 148           }
 149         }
 150         // O(N^2)!
 151         rrset.erase(remove_if(rrset.begin(), rrset.end(),
 152                               [&g](const DNSRecord& dr) {
 153                                 return count(g.second.first.cbegin(),
 154                                              g.second.first.cend(), dr);
 155                               }), rrset.end());
 156         // the DNSRecord== operator compares on name, type, class and lowercase content representation
 157
 158         for(const auto& x : g.second.second) {
 159           rrset.push_back(x);
 160         }
 161
 162         vector<DNSResourceRecord> replacement;
 163         for(const auto& dr : rrset) {
 164           auto rr = DNSResourceRecord::fromWire(dr);
 165           rr.qname += domain;
 166           rr.domain_id = di.id;
 167           if(dr.d_type == QType::SOA) {
 168             //            cout<<"New SOA: "<<x.d_content->getZoneRepresentation()<<endl;
 169             auto sr = getRR<SOARecordContent>(dr);
 170             zs.soa_serial=sr->d_st.serial;
 171           }
 172
 173           replacement.push_back(rr);
 174         }
 175
 176         di.backend->replaceRRSet(di.id, g.first.first+domain, QType(g.first.second), replacement);
 177       }
 178       di.backend->commitTransaction();
 179     }
 180   }
 181   catch(std::exception& p) {
 182     g_log<<Logger::Error<<"Got exception during IXFR: "<<p.what()<<endl;
 183     throw;
 184   }
 185   catch(PDNSException& p) {
 186     g_log<<Logger::Error<<"Got exception during IXFR: "<<p.reason<<endl;
 187     throw;
 188   }
 189 }
 190
 191
 192 static bool processRecordForZS(const DNSName& domain, bool& firstNSEC3, DNSResourceRecord& rr, ZoneStatus& zs)
 193 {
 194   switch(rr.qtype.getCode()) {
 195   case QType::NSEC3PARAM:
 196     zs.ns3pr = NSEC3PARAMRecordContent(rr.content);
 197     zs.isDnssecZone = zs.isNSEC3 = true;
 198     zs.isNarrow = false;
 199     return false;
 200   case QType::NSEC3: {
 201     NSEC3RecordContent ns3rc(rr.content);
 202     if (firstNSEC3) {
 203       zs.isDnssecZone = zs.isPresigned = true;
 204       firstNSEC3 = false;
 205     } else if (zs.optOutFlag != (ns3rc.d_flags & 1))
 206       throw PDNSException("Zones with a mixture of Opt-Out NSEC3 RRs and non-Opt-Out NSEC3 RRs are not supported.");
 207     zs.optOutFlag = ns3rc.d_flags & 1;
 208     if (ns3rc.isSet(QType::NS) && !(rr.qname==domain)) {
 209       DNSName hashPart = rr.qname.makeRelative(domain);
 210       zs.secured.insert(hashPart);
 211     }
 212     return false;
 213   }
 214
 215   case QType::NSEC:
 216     zs.isDnssecZone = zs.isPresigned = true;
 217     return false;
 218
 219   case QType::NS:
 220     if(rr.qname!=domain)
 221       zs.nsset.insert(rr.qname);
 222     break;
 223   }
 224
 225   zs.qnames.insert(rr.qname);
 226
 227   rr.domain_id=zs.domain_id;
 228   return true;
 229 }
 230
 231 /* So this code does a number of things.
 232    1) It will AXFR a domain from a master
 233       The code can retrieve the current serial number in the database itself.
 234       It may attempt an IXFR
 235    2) It will filter the zone through a lua *filter* script
 236    3) The code walks through the zone records do determine DNSSEC status (secured, nsec/nsec3, optout)
 237    4) It inserts the zone into the database
 238       With the right 'ordername' fields
 239    5) It updates the Empty Non Terminals
 240 */
 241
 242 static vector<DNSResourceRecord> doAxfr(const ComboAddress& raddr, const DNSName& domain, const TSIGTriplet& tt, const ComboAddress& laddr,  scoped_ptr<AuthLua4>& pdl, ZoneStatus& zs)
 243 {
 244   vector<DNSResourceRecord> rrs;
 245   AXFRRetriever retriever(raddr, domain, tt, (laddr.sin4.sin_family == 0) ? NULL : &laddr, ((size_t) ::arg().asNum("xfr-max-received-mbytes")) * 1024 * 1024);
 246   Resolver::res_t recs;
 247   bool first=true;
 248   bool firstNSEC3{true};
 249   bool soa_received {false};
 250   while(retriever.getChunk(recs)) {
 251     if(first) {
 252       g_log<<Logger::Error<<"AXFR started for '"<<domain<<"'"<<endl;
 253       first=false;
 254     }
 255
 256     for(Resolver::res_t::iterator i=recs.begin();i!=recs.end();++i) {
 257       i->qname.makeUsLowerCase();
 258       if(i->qtype.getCode() == QType::OPT || i->qtype.getCode() == QType::TSIG) // ignore EDNS0 & TSIG
 259         continue;
 260
 261       if(!i->qname.isPartOf(domain)) {
 262         g_log<<Logger::Error<<"Remote "<<raddr.toStringWithPort()<<" tried to sneak in out-of-zone data '"<<i->qname<<"'|"<<i->qtype.getName()<<" during AXFR of zone '"<<domain<<"', ignoring"<<endl;
 263         continue;
 264       }
 265
 266       vector<DNSResourceRecord> out;
 267       if(!pdl || !pdl->axfrfilter(raddr, domain, *i, out)) {
 268         out.push_back(*i); // if axfrfilter didn't do anything, we put our record in 'out' ourselves
 269       }
 270
 271       for(DNSResourceRecord& rr :  out) {
 272         if(!rr.qname.isPartOf(domain)) {
 273           g_log<<Logger::Error<<"Lua axfrfilter() filter tried to sneak in out-of-zone data '"<<i->qname<<"'|"<<i->qtype.getName()<<" during AXFR of zone '"<<domain<<"', ignoring"<<endl;
 274           continue;
 275         }
 276         if(!processRecordForZS(domain, firstNSEC3, rr, zs))
 277           continue;
 278         if(rr.qtype.getCode() == QType::SOA) {
 279           if(soa_received)
 280             continue; //skip the last SOA
 281           SOAData sd;
 282           fillSOAData(rr.content,sd);
 283           zs.soa_serial = sd.serial;
 284           soa_received = true;
 285         }
 286
 287         rrs.push_back(rr);
 288
 289       }
 290     }
 291   }
 292   return rrs;
 293 }
 294
 295
 296 void CommunicatorClass::suck(const DNSName &domain, const ComboAddress& remote)
 297 {
 298   {
 299     Lock l(&d_lock);
 300     if(d_inprogress.count(domain)) {
 301       return;
 302     }
 303     d_inprogress.insert(domain);
 304   }
 305   RemoveSentinel rs(domain, this); // this removes us from d_inprogress when we go out of scope
 306
 307   g_log<<Logger::Error<<"Initiating transfer of '"<<domain<<"' from remote '"<<remote<<"'"<<endl;
 308   UeberBackend B; // fresh UeberBackend
 309
 310   DomainInfo di;
 311   di.backend=0;
 312   bool transaction=false;
 313   try {
 314     DNSSECKeeper dk (&B); // reuse our UeberBackend copy for DNSSECKeeper
 315     bool wrongDomainKind = false;
 316     // this checks three error conditions & sets wrongDomainKind if we hit the third
 317     if(!B.getDomainInfo(domain, di) || !di.backend || (wrongDomainKind = true, di.kind != DomainInfo::Slave)) { // di.backend and B are mostly identical
 318       if(wrongDomainKind)
 319         g_log<<Logger::Error<<"Can't determine backend for domain '"<<domain<<"', not configured as slave"<<endl;
 320       else
 321         g_log<<Logger::Error<<"Can't determine backend for domain '"<<domain<<"'"<<endl;
 322       return;
 323     }
 324     ZoneStatus zs;
 325     zs.domain_id=di.id;
 326
 327     TSIGTriplet tt;
 328     if(dk.getTSIGForAccess(domain, remote, &tt.name)) {
 329       string tsigsecret64;
 330       if(B.getTSIGKey(tt.name, &tt.algo, &tsigsecret64)) {
 331         if(B64Decode(tsigsecret64, tt.secret)) {
 332           g_log<<Logger::Error<<"Unable to Base-64 decode TSIG key '"<<tt.name<<"' for domain '"<<domain<<"' not found"<<endl;
 333           return;
 334         }
 335       } else {
 336         g_log<<Logger::Error<<"TSIG key '"<<tt.name<<"' for domain '"<<domain<<"' not found"<<endl;
 337         return;
 338       }
 339     }
 340
 341
 342     scoped_ptr<AuthLua4> pdl;
 343     vector<string> scripts;
 344     string script=::arg()["lua-axfr-script"];
 345     if(B.getDomainMetadata(domain, "LUA-AXFR-SCRIPT", scripts) && !scripts.empty()) {
 346       if (pdns_iequals(scripts[0], "NONE")) {
 347         script.clear();
 348       } else {
 349         script=scripts[0];
 350       }
 351     }
 352     if(!script.empty()){
 353       try {
 354         pdl.reset(new AuthLua4());
 355         pdl->loadFile(script);
 356         g_log<<Logger::Info<<"Loaded Lua script '"<<script<<"' to edit the incoming AXFR of '"<<domain<<"'"<<endl;
 357       }
 358       catch(std::exception& e) {
 359         g_log<<Logger::Error<<"Failed to load Lua editing script '"<<script<<"' for incoming AXFR of '"<<domain<<"': "<<e.what()<<endl;
 360         return;
 361       }
 362     }
 363
 364     vector<string> localaddr;
 365     ComboAddress laddr;
 366
 367     if(B.getDomainMetadata(domain, "AXFR-SOURCE", localaddr) && !localaddr.empty()) {
 368       try {
 369         laddr = ComboAddress(localaddr[0]);
 370         g_log<<Logger::Info<<"AXFR source for domain '"<<domain<<"' set to "<<localaddr[0]<<endl;
 371       }
 372       catch(std::exception& e) {
 373         g_log<<Logger::Error<<"Failed to load AXFR source '"<<localaddr[0]<<"' for incoming AXFR of '"<<domain<<"': "<<e.what()<<endl;
 374         return;
 375       }
 376     } else {
 377       if(remote.sin4.sin_family == AF_INET && !::arg()["query-local-address"].empty()) {
 378         laddr = ComboAddress(::arg()["query-local-address"]);
 379       } else if(remote.sin4.sin_family == AF_INET6 && !::arg()["query-local-address6"].empty()) {
 380         laddr = ComboAddress(::arg()["query-local-address6"]);
 381       } else {
 382         bool isv6 = remote.sin4.sin_family == AF_INET6;
 383         g_log<<Logger::Error<<"Unable to AXFR, destination address is IPv" << (isv6 ? "6" : "4") << ", but query-local-address"<< (isv6 ? "6" : "") << " is unset!"<<endl;
 384         return;
 385       }
 386     }
 387
 388     bool hadDnssecZone = false;
 389     bool hadPresigned = false;
 390     bool hadNSEC3 = false;
 391     NSEC3PARAMRecordContent hadNs3pr;
 392     bool hadNarrow=false;
 393
 394
 395     vector<DNSResourceRecord> rrs;
 396     if(dk.isSecuredZone(domain)) {
 397       hadDnssecZone=true;
 398       hadPresigned=dk.isPresigned(domain);
 399       if (dk.getNSEC3PARAM(domain, &zs.ns3pr, &zs.isNarrow)) {
 400         hadNSEC3 = true;
 401         hadNs3pr = zs.ns3pr;
 402         hadNarrow = zs.isNarrow;
 403       }
 404     }
 405     else if(di.serial) {
 406       vector<string> meta;
 407       B.getDomainMetadata(domain, "IXFR", meta);
 408       if(!meta.empty() && meta[0]=="1") {
 409         vector<DNSRecord> axfr;
 410         g_log<<Logger::Warning<<"Starting IXFR of '"<<domain<<"' from remote "<<remote<<endl;
 411         ixfrSuck(domain, tt, laddr, remote, pdl, zs, &axfr);
 412         if(!axfr.empty()) {
 413           g_log<<Logger::Warning<<"IXFR of '"<<domain<<"' from remote '"<<remote<<"' turned into an AXFR"<<endl;
 414           bool firstNSEC3=true;
 415           rrs.reserve(axfr.size());
 416           for(const auto& dr : axfr) {
 417             auto rr = DNSResourceRecord::fromWire(dr);
 418             (rr.qname += domain).makeUsLowerCase();
 419             rr.domain_id = zs.domain_id;
 420             if(!processRecordForZS(domain, firstNSEC3, rr, zs))
 421               continue;
 422             if(dr.d_type == QType::SOA) {
 423               auto sd = getRR<SOARecordContent>(dr);
 424               zs.soa_serial = sd->d_st.serial;
 425             }
 426             rrs.push_back(rr);
 427           }
 428         }
 429         else {
 430           g_log<<Logger::Warning<<"Done with IXFR of '"<<domain<<"' from remote '"<<remote<<"', got "<<zs.numDeltas<<" delta"<<addS(zs.numDeltas)<<", serial now "<<zs.soa_serial<<endl;
 431           purgeAuthCaches(domain.toString()+"$");
 432           return;
 433         }
 434       }
 435     }
 436
 437     if(rrs.empty()) {
 438       g_log<<Logger::Warning<<"Starting AXFR of '"<<domain<<"' from remote "<<remote<<endl;
 439       rrs = doAxfr(remote, domain, tt, laddr, pdl, zs);
 440       g_log<<Logger::Warning<<"AXFR of '"<<domain<<"' from remote "<<remote<<" done"<<endl;
 441     }
 442
 443     if(zs.isNSEC3) {
 444       zs.ns3pr.d_flags = zs.optOutFlag ? 1 : 0;
 445     }
 446
 447     if(!zs.isPresigned) {
 448       DNSSECKeeper::keyset_t keys = dk.getKeys(domain);
 449       if(!keys.empty()) {
 450         zs.isDnssecZone = true;
 451         zs.isNSEC3 = hadNSEC3;
 452         zs.ns3pr = hadNs3pr;
 453         zs.optOutFlag = (hadNs3pr.d_flags & 1);
 454         zs.isNarrow = hadNarrow;
 455       }
 456     }
 457
 458     if(zs.isDnssecZone) {
 459       if(!zs.isNSEC3)
 460         g_log<<Logger::Info<<"Adding NSEC ordering information"<<endl;
 461       else if(!zs.isNarrow)
 462         g_log<<Logger::Info<<"Adding NSEC3 hashed ordering information for '"<<domain<<"'"<<endl;
 463       else
 464         g_log<<Logger::Info<<"Erasing NSEC3 ordering since we are narrow, only setting 'auth' fields"<<endl;
 465     }
 466
 467
 468     transaction=di.backend->startTransaction(domain, zs.domain_id);
 469     g_log<<Logger::Error<<"Backend transaction started for '"<<domain<<"' storage"<<endl;
 470
 471     // update the presigned flag and NSEC3PARAM
 472     if (zs.isDnssecZone) {
 473       // update presigned if there was a change
 474       if (zs.isPresigned && !hadPresigned) {
 475         // zone is now presigned
 476         dk.setPresigned(domain);
 477       } else if (hadPresigned && !zs.isPresigned) {
 478         // zone is no longer presigned
 479         dk.unsetPresigned(domain);
 480       }
 481       // update NSEC3PARAM
 482       if (zs.isNSEC3) {
 483         // zone is NSEC3, only update if there was a change
 484         if (!hadNSEC3 || (hadNarrow  != zs.isNarrow) ||
 485             (zs.ns3pr.d_algorithm != hadNs3pr.d_algorithm) ||
 486             (zs.ns3pr.d_flags != hadNs3pr.d_flags) ||
 487             (zs.ns3pr.d_iterations != hadNs3pr.d_iterations) ||
 488             (zs.ns3pr.d_salt != hadNs3pr.d_salt)) {
 489           dk.setNSEC3PARAM(domain, zs.ns3pr, zs.isNarrow);
 490         }
 491       } else if (hadNSEC3 ) {
 492          // zone is no longer NSEC3
 493          dk.unsetNSEC3PARAM(domain);
 494       }
 495     } else if (hadDnssecZone) {
 496       // zone is no longer signed
 497       if (hadPresigned) {
 498         // remove presigned
 499         dk.unsetPresigned(domain);
 500       }
 501       if (hadNSEC3) {
 502         // unset NSEC3PARAM
 503         dk.unsetNSEC3PARAM(domain);
 504       }
 505     }
 506
 507     bool doent=true;
 508     uint32_t maxent = ::arg().asNum("max-ent-entries");
 509     DNSName shorter, ordername;
 510     set<DNSName> rrterm;
 511     map<DNSName,bool> nonterm;
 512
 513
 514     for(DNSResourceRecord& rr :  rrs) {
 515       if(!zs.isPresigned) {
 516         if (rr.qtype.getCode() == QType::RRSIG)
 517           continue;
 518         if(zs.isDnssecZone && rr.qtype.getCode() == QType::DNSKEY && !::arg().mustDo("direct-dnskey"))
 519           continue;
 520       }
 521
 522       // Figure out auth and ents
 523       rr.auth=true;
 524       shorter=rr.qname;
 525       rrterm.clear();
 526       do {
 527         if(doent) {
 528           if (!zs.qnames.count(shorter))
 529             rrterm.insert(shorter);
 530         }
 531         if(zs.nsset.count(shorter) && rr.qtype.getCode() != QType::DS)
 532           rr.auth=false;
 533
 534         if (shorter==domain) // stop at apex
 535           break;
 536       }while(shorter.chopOff());
 537
 538       // Insert ents
 539       if(doent && !rrterm.empty()) {
 540         bool auth;
 541         if (!rr.auth && rr.qtype.getCode() == QType::NS) {
 542           if (zs.isNSEC3)
 543             ordername=DNSName(toBase32Hex(hashQNameWithSalt(zs.ns3pr, rr.qname)));
 544           auth=(!zs.isNSEC3 || !zs.optOutFlag || zs.secured.count(ordername));
 545         } else
 546           auth=rr.auth;
 547
 548         for(const auto &nt: rrterm){
 549           if (!nonterm.count(nt))
 550               nonterm.insert(pair<DNSName, bool>(nt, auth));
 551             else if (auth)
 552               nonterm[nt]=true;
 553         }
 554
 555         if(nonterm.size() > maxent) {
 556           g_log<<Logger::Error<<"AXFR zone "<<domain<<" has too many empty non terminals."<<endl;
 557           nonterm.clear();
 558           doent=false;
 559         }
 560       }
 561
 562       // RRSIG is always auth, even inside a delegation
 563       if (rr.qtype.getCode() == QType::RRSIG)
 564         rr.auth=true;
 565
 566       // Add ordername and insert record
 567       if (zs.isDnssecZone && rr.qtype.getCode() != QType::RRSIG) {
 568         if (zs.isNSEC3) {
 569           // NSEC3
 570           ordername=DNSName(toBase32Hex(hashQNameWithSalt(zs.ns3pr, rr.qname)));
 571           if(!zs.isNarrow && (rr.auth || (rr.qtype.getCode() == QType::NS && (!zs.optOutFlag || zs.secured.count(ordername))))) {
 572             di.backend->feedRecord(rr, ordername, true);
 573           } else
 574             di.backend->feedRecord(rr, DNSName());
 575         } else {
 576           // NSEC
 577           if (rr.auth || rr.qtype.getCode() == QType::NS) {
 578             ordername=rr.qname.makeRelative(domain);
 579             di.backend->feedRecord(rr, ordername);
 580           } else
 581             di.backend->feedRecord(rr, DNSName());
 582         }
 583       } else
 584         di.backend->feedRecord(rr, DNSName());
 585     }
 586
 587     // Insert empty non-terminals
 588     if(doent && !nonterm.empty()) {
 589       if (zs.isNSEC3) {
 590         di.backend->feedEnts3(zs.domain_id, domain, nonterm, zs.ns3pr, zs.isNarrow);
 591       } else
 592         di.backend->feedEnts(zs.domain_id, nonterm);
 593     }
 594
 595     di.backend->commitTransaction();
 596     transaction = false;
 597     di.backend->setFresh(zs.domain_id);
 598     purgeAuthCaches(domain.toString()+"$");
 599
 600
 601     g_log<<Logger::Error<<"AXFR done for '"<<domain<<"', zone committed with serial number "<<zs.soa_serial<<endl;
 602     if(::arg().mustDo("slave-renotify"))
 603       notifyDomain(domain);
 604   }
 605   catch(DBException &re) {
 606     g_log<<Logger::Error<<"Unable to feed record during incoming AXFR of '" << domain<<"': "<<re.reason<<endl;
 607     if(di.backend && transaction) {
 608       g_log<<Logger::Error<<"Aborting possible open transaction for domain '"<<domain<<"' AXFR"<<endl;
 609       di.backend->abortTransaction();
 610     }
 611   }
 612   catch(const MOADNSException &mde) {
 613     g_log<<Logger::Error<<"Unable to parse record during incoming AXFR of '"<<domain<<"' (MOADNSException): "<<mde.what()<<endl;
 614     if(di.backend && transaction) {
 615       g_log<<Logger::Error<<"Aborting possible open transaction for domain '"<<domain<<"' AXFR"<<endl;
 616       di.backend->abortTransaction();
 617     }
 618   }
 619   catch(std::exception &re) {
 620     g_log<<Logger::Error<<"Unable to parse record during incoming AXFR of '"<<domain<<"' (std::exception): "<<re.what()<<endl;
 621     if(di.backend && transaction) {
 622       g_log<<Logger::Error<<"Aborting possible open transaction for domain '"<<domain<<"' AXFR"<<endl;
 623       di.backend->abortTransaction();
 624     }
 625   }
 626   catch(ResolverException &re) {
 627     {
 628       Lock l(&d_lock);
 629       // The AXFR probably failed due to a problem on the master server. If SOA-checks against this master
 630       // still succeed, we would constantly try to AXFR the zone. To avoid this, we add the zone to the list of
 631       // failed slave-checks. This will suspend slave-checks (and subsequent AXFR) for this zone for some time.
 632       uint64_t newCount = 1;
 633       time_t now = time(0);
 634       const auto failedEntry = d_failedSlaveRefresh.find(domain);
 635       if (failedEntry != d_failedSlaveRefresh.end())
 636         newCount = d_failedSlaveRefresh[domain].first + 1;
 637       time_t nextCheck = now + std::min(newCount * d_tickinterval, (uint64_t)::arg().asNum("soa-retry-default"));
 638       d_failedSlaveRefresh[domain] = {newCount, nextCheck};
 639       g_log<<Logger::Error<<"Unable to AXFR zone '"<<domain<<"' from remote '"<<remote<<"' (resolver): "<<re.reason<<" (This was the "<<(newCount == 1 ? "first" : std::to_string(newCount) + "th")<<" time. Excluding zone from slave-checks until "<<nextCheck<<")"<<endl;
 640     }
 641     if(di.backend && transaction) {
 642       g_log<<Logger::Error<<"Aborting possible open transaction for domain '"<<domain<<"' AXFR"<<endl;
 643       di.backend->abortTransaction();
 644     }
 645   }
 646   catch(PDNSException &ae) {
 647     g_log<<Logger::Error<<"Unable to AXFR zone '"<<domain<<"' from remote '"<<remote<<"' (PDNSException): "<<ae.reason<<endl;
 648     if(di.backend && transaction) {
 649       g_log<<Logger::Error<<"Aborting possible open transaction for domain '"<<domain<<"' AXFR"<<endl;
 650       di.backend->abortTransaction();
 651     }
 652   }
 653 }
 654 namespace {
 655 struct DomainNotificationInfo
 656 {
 657   DomainInfo di;
 658   bool dnssecOk;
 659   ComboAddress localaddr;
 660   DNSName tsigkeyname, tsigalgname;
 661   string tsigsecret;
 662 };
 663 }
 664
 665
 666 struct SlaveSenderReceiver
 667 {
 668   typedef std::tuple<DNSName, ComboAddress, uint16_t> Identifier;
 669
 670   struct Answer {
 671     uint32_t theirSerial;
 672     uint32_t theirInception;
 673     uint32_t theirExpire;
 674   };
 675
 676   map<uint32_t, Answer> d_freshness;
 677
 678   SlaveSenderReceiver()
 679   {
 680   }
 681
 682   void deliverTimeout(const Identifier& i)
 683   {
 684   }
 685
 686   Identifier send(DomainNotificationInfo& dni)
 687   {
 688     random_shuffle(dni.di.masters.begin(), dni.di.masters.end());
 689     try {
 690       return std::make_tuple(dni.di.zone,
 691                              *dni.di.masters.begin(),
 692                              d_resolver.sendResolve(*dni.di.masters.begin(),
 693                                                     dni.localaddr,
 694                                                     dni.di.zone,
 695                                                     QType::SOA,
 696                                                     nullptr,
 697                                                     dni.dnssecOk, dni.tsigkeyname, dni.tsigalgname, dni.tsigsecret)
 698         );
 699     }
 700     catch(PDNSException& e) {
 701       throw runtime_error("While attempting to query freshness of '"+dni.di.zone.toLogString()+"': "+e.reason);
 702     }
 703   }
 704
 705   bool receive(Identifier& id, Answer& a)
 706   {
 707     if(d_resolver.tryGetSOASerial(&(std::get<0>(id)), &(std::get<1>(id)), &a.theirSerial, &a.theirInception, &a.theirExpire, &(std::get<2>(id)))) {
 708       return 1;
 709     }
 710     return 0;
 711   }
 712
 713   void deliverAnswer(DomainNotificationInfo& dni, const Answer& a, unsigned int usec)
 714   {
 715     d_freshness[dni.di.id]=a;
 716   }
 717
 718   Resolver d_resolver;
 719 };
 720
 721 void CommunicatorClass::addSlaveCheckRequest(const DomainInfo& di, const ComboAddress& remote)
 722 {
 723   Lock l(&d_lock);
 724   DomainInfo ours = di;
 725   ours.backend = 0;
 726
 727   // When adding a check, if the remote addr from which notification was
 728   // received is a master, clear all other masters so we can be sure the
 729   // query goes to that one.
 730   for (const auto& master : di.masters) {
 731     if (ComboAddress::addressOnlyEqual()(remote, master)) {
 732       ours.masters.clear();
 733       ours.masters.push_back(master);
 734       break;
 735     }
 736   }
 737   d_tocheck.erase(di);
 738   d_tocheck.insert(ours);
 739   d_any_sem.post(); // kick the loop!
 740 }
 741
 742 void CommunicatorClass::addTrySuperMasterRequest(DNSPacket *p)
 743 {
 744   Lock l(&d_lock);
 745   DNSPacket ours = *p;
 746   if(d_potentialsupermasters.insert(ours).second)
 747     d_any_sem.post(); // kick the loop!
 748 }
 749
 750 void CommunicatorClass::slaveRefresh(PacketHandler *P)
 751 {
 752   // not unless we are slave
 753   if (!::arg().mustDo("slave")) return;
 754
 755   UeberBackend *B=P->getBackend();
 756   vector<DomainInfo> rdomains;
 757   vector<DomainNotificationInfo> sdomains;
 758   set<DNSPacket, cmp> trysuperdomains;
 759   {
 760     Lock l(&d_lock);
 761     set<DomainInfo> requeue;
 762     for(const auto& di: d_tocheck) {
 763       if(d_inprogress.count(di.zone)) {
 764         g_log<<Logger::Debug<<"Got NOTIFY for "<<di.zone<<" while AXFR in progress, requeueing SOA check"<<endl;
 765         requeue.insert(di);
 766       }
 767       else {
 768         // We received a NOTIFY for a zone. This means at least one of the zone's master server is working.
 769         // Therefore we delete the zone from the list of failed slave-checks to allow immediate checking.
 770         const auto wasFailedDomain = d_failedSlaveRefresh.find(di.zone);
 771         if (wasFailedDomain != d_failedSlaveRefresh.end()) {
 772           g_log<<Logger::Debug<<"Got NOTIFY for "<<di.zone<<", removing zone from list of failed slave-checks and going to check SOA serial"<<endl;
 773           d_failedSlaveRefresh.erase(di.zone);
 774         } else {
 775           g_log<<Logger::Debug<<"Got NOTIFY for "<<di.zone<<", going to check SOA serial"<<endl;
 776         }
 777         rdomains.push_back(di);
 778       }
 779     }
 780     d_tocheck.swap(requeue);
 781
 782     trysuperdomains = d_potentialsupermasters;
 783     d_potentialsupermasters.clear();
 784   }
 785
 786   for(const DNSPacket& dp :  trysuperdomains) {
 787     // get the TSIG key name
 788     TSIGRecordContent trc;
 789     DNSName tsigkeyname;
 790     dp.getTSIGDetails(&trc, &tsigkeyname);
 791     P->trySuperMasterSynchronous(&dp, tsigkeyname); // FIXME could use some error loging
 792   }
 793   if(rdomains.empty()) { // if we have priority domains, check them first
 794     B->getUnfreshSlaveInfos(&rdomains);
 795   }
 796   DNSSECKeeper dk(B); // NOW HEAR THIS! This DK uses our B backend, so no interleaved access!
 797   {
 798     Lock l(&d_lock);
 799     domains_by_name_t& nameindex=boost::multi_index::get<IDTag>(d_suckdomains);
 800     time_t now = time(0);
 801
 802     for(DomainInfo& di :  rdomains) {
 803       const auto failed = d_failedSlaveRefresh.find(di.zone);
 804       if (failed != d_failedSlaveRefresh.end() && now < failed->second.second ) {
 805         // If the domain has failed before and the time before the next check has not expired, skip this domain
 806         g_log<<Logger::Debug<<"Zone '"<<di.zone<<"' is on the list of failed SOA checks. Skipping SOA checks until "<< failed->second.second<<endl;
 807         continue;
 808       }
 809       std::vector<std::string> localaddr;
 810       SuckRequest sr;
 811       sr.domain=di.zone;
 812       if(di.masters.empty()) // slave domains w/o masters are ignored
 813         continue;
 814       // remove unfresh domains already queued for AXFR, no sense polling them again
 815       sr.master=*di.masters.begin();
 816       if(nameindex.count(sr)) {  // this does NOT however protect us against AXFRs already in progress!
 817         continue;
 818       }
 819       if(d_inprogress.count(sr.domain)) // this does
 820         continue;
 821
 822       DomainNotificationInfo dni;
 823       dni.di=di;
 824       dni.dnssecOk = dk.doesDNSSEC();
 825
 826       if(dk.getTSIGForAccess(di.zone, sr.master, &dni.tsigkeyname)) {
 827         string secret64;
 828         if(!B->getTSIGKey(dni.tsigkeyname, &dni.tsigalgname, &secret64)) {
 829           g_log<<Logger::Error<<"TSIG key '"<<dni.tsigkeyname<<"' for domain '"<<di.zone<<"' not found, can not AXFR."<<endl;
 830           continue;
 831         }
 832         if (B64Decode(secret64, dni.tsigsecret) == -1) {
 833           g_log<<Logger::Error<<"Unable to Base-64 decode TSIG key '"<<dni.tsigkeyname<<"' for domain '"<<di.zone<<"', can not AXFR."<<endl;
 834           continue;
 835         }
 836       }
 837
 838       localaddr.clear();
 839       // check for AXFR-SOURCE
 840       if(B->getDomainMetadata(di.zone, "AXFR-SOURCE", localaddr) && !localaddr.empty()) {
 841         try {
 842           dni.localaddr = ComboAddress(localaddr[0]);
 843           g_log<<Logger::Info<<"Freshness check source (AXFR-SOURCE) for domain '"<<di.zone<<"' set to "<<localaddr[0]<<endl;
 844         }
 845         catch(std::exception& e) {
 846           g_log<<Logger::Error<<"Failed to load freshness check source '"<<localaddr[0]<<"' for '"<<di.zone<<"': "<<e.what()<<endl;
 847           return;
 848         }
 849       } else {
 850         dni.localaddr.sin4.sin_family = 0;
 851       }
 852
 853       sdomains.push_back(dni);
 854     }
 855   }
 856   if(sdomains.empty())
 857   {
 858     if(d_slaveschanged) {
 859       Lock l(&d_lock);
 860       g_log<<Logger::Warning<<"No new unfresh slave domains, "<<d_suckdomains.size()<<" queued for AXFR already, "<<d_inprogress.size()<<" in progress"<<endl;
 861     }
 862     d_slaveschanged = !rdomains.empty();
 863     return;
 864   }
 865   else {
 866     Lock l(&d_lock);
 867     g_log<<Logger::Warning<<sdomains.size()<<" slave domain"<<(sdomains.size()>1 ? "s" : "")<<" need"<<
 868       (sdomains.size()>1 ? "" : "s")<<
 869       " checking, "<<d_suckdomains.size()<<" queued for AXFR"<<endl;
 870   }
 871
 872   SlaveSenderReceiver ssr;
 873
 874   Inflighter<vector<DomainNotificationInfo>, SlaveSenderReceiver> ifl(sdomains, ssr);
 875
 876   ifl.d_maxInFlight = 200;
 877
 878   for(;;) {
 879     try {
 880       ifl.run();
 881       break;
 882     }
 883     catch(std::exception& e) {
 884       g_log<<Logger::Error<<"While checking domain freshness: " << e.what()<<endl;
 885     }
 886     catch(PDNSException &re) {
 887       g_log<<Logger::Error<<"While checking domain freshness: " << re.reason<<endl;
 888     }
 889   }
 890   g_log<<Logger::Warning<<"Received serial number updates for "<<ssr.d_freshness.size()<<" zone"<<addS(ssr.d_freshness.size())<<", had "<<ifl.getTimeouts()<<" timeout"<<addS(ifl.getTimeouts())<<endl;
 891
 892   typedef DomainNotificationInfo val_t;
 893   time_t now = time(0);
 894   for(val_t& val :  sdomains) {
 895     DomainInfo& di(val.di);
 896     DomainInfo tempdi;
 897     // might've come from the packethandler
 898     // Please do not overwrite received DI just to make sure it exists in backend.
 899     if(!di.backend) {
 900       if (!B->getDomainInfo(di.zone, tempdi)) {
 901         g_log<<Logger::Warning<<"Ignore domain "<< di.zone<<" since it has been removed from our backend"<<endl;
 902         continue;
 903       }
 904       // Backend for di still doesn't exist and this might cause us to
 905       // SEGFAULT on the setFresh command later on
 906       di.backend = tempdi.backend;
 907     }
 908
 909     if(!ssr.d_freshness.count(di.id)) { // If we don't have an answer for the domain
 910       uint64_t newCount = 1;
 911       Lock l(&d_lock);
 912       const auto failedEntry = d_failedSlaveRefresh.find(di.zone);
 913       if (failedEntry != d_failedSlaveRefresh.end())
 914         newCount = d_failedSlaveRefresh[di.zone].first + 1;
 915       time_t nextCheck = now + std::min(newCount * d_tickinterval, (uint64_t)::arg().asNum("soa-retry-default"));
 916       d_failedSlaveRefresh[di.zone] = {newCount, nextCheck};
 917       if (newCount == 1) {
 918         g_log<<Logger::Warning<<"Unable to retrieve SOA for "<<di.zone<<
 919           ", this was the first time. NOTE: For every subsequent failed SOA check the domain will be suspended from freshness checks for 'num-errors x "<<
 920           d_tickinterval<<" seconds', with a maximum of "<<(uint64_t)::arg().asNum("soa-retry-default")<<" seconds. Skipping SOA checks until "<<nextCheck<<endl;
 921       } else if (newCount % 10 == 0) {
 922         g_log<<Logger::Warning<<"Unable to retrieve SOA for "<<di.zone<<", this was the "<<std::to_string(newCount)<<"th time. Skipping SOA checks until "<<nextCheck<<endl;
 923       }
 924       continue;
 925     }
 926
 927     {
 928       Lock l(&d_lock);
 929       const auto wasFailedDomain = d_failedSlaveRefresh.find(di.zone);
 930       if (wasFailedDomain != d_failedSlaveRefresh.end())
 931         d_failedSlaveRefresh.erase(di.zone);
 932     }
 933
 934     bool hasSOA = false;
 935     SOAData sd;
 936     try{
 937       hasSOA = B->getSOA(di.zone, sd);
 938     }
 939     catch(...) {}
 940
 941     uint32_t theirserial = ssr.d_freshness[di.id].theirSerial, ourserial = sd.serial;
 942
 943     if(rfc1982LessThan(theirserial, ourserial) && ourserial != 0 && !::arg().mustDo("axfr-lower-serial"))  {
 944       g_log<<Logger::Error<<"Domain '"<<di.zone<<"' more recent than master, our serial " << ourserial << " > their serial "<< theirserial << endl;
 945       di.backend->setFresh(di.id);
 946     }
 947     else if(hasSOA && theirserial == ourserial) {
 948       uint32_t maxExpire=0, maxInception=0;
 949       if(dk.isPresigned(di.zone)) {
 950         B->lookup(QType(QType::RRSIG), di.zone); // can't use DK before we are done with this lookup!
 951         DNSZoneRecord zr;
 952         while(B->get(zr)) {
 953           auto rrsig = getRR<RRSIGRecordContent>(zr.dr);
 954           if(rrsig->d_type == QType::SOA) {
 955             maxInception = std::max(maxInception, rrsig->d_siginception);
 956             maxExpire = std::max(maxExpire, rrsig->d_sigexpire);
 957           }
 958         }
 959       }
 960       if(! maxInception && ! ssr.d_freshness[di.id].theirInception) {
 961         g_log<<Logger::Info<<"Domain '"<< di.zone<<"' is fresh (no DNSSEC), serial is "<<ourserial<<endl;
 962         di.backend->setFresh(di.id);
 963       }
 964       else if(maxInception == ssr.d_freshness[di.id].theirInception && maxExpire == ssr.d_freshness[di.id].theirExpire) {
 965         g_log<<Logger::Info<<"Domain '"<< di.zone<<"' is fresh and SOA RRSIGs match, serial is "<<ourserial<<endl;
 966         di.backend->setFresh(di.id);
 967       }
 968       else if(maxExpire >= now && ! ssr.d_freshness[di.id].theirInception ) {
 969         g_log<<Logger::Info<<"Domain '"<< di.zone<<"' is fresh, master is no longer signed but (some) signatures are still vallid, serial is "<<ourserial<<endl;
 970         di.backend->setFresh(di.id);
 971       }
 972       else if(maxInception && ! ssr.d_freshness[di.id].theirInception ) {
 973         g_log<<Logger::Warning<<"Domain '"<< di.zone<<"' is stale, master is no longer signed and all signatures have expired, serial is "<<ourserial<<endl;
 974         addSuckRequest(di.zone, *di.masters.begin());
 975       }
 976       else if(dk.doesDNSSEC() && ! maxInception && ssr.d_freshness[di.id].theirInception) {
 977         g_log<<Logger::Warning<<"Domain '"<< di.zone<<"' is stale, master has signed, serial is "<<ourserial<<endl;
 978         addSuckRequest(di.zone, *di.masters.begin());
 979       }
 980       else {
 981         g_log<<Logger::Warning<<"Domain '"<< di.zone<<"' is fresh, but RRSIGs differ, so DNSSEC is stale, serial is "<<ourserial<<endl;
 982         addSuckRequest(di.zone, *di.masters.begin());
 983       }
 984     }
 985     else {
 986       if(hasSOA) {
 987         g_log<<Logger::Warning<<"Domain '"<< di.zone<<"' is stale, master serial "<<theirserial<<", our serial "<< ourserial <<endl;
 988       }
 989       else {
 990         g_log<<Logger::Warning<<"Domain '"<< di.zone<<"' is empty, master serial "<<theirserial<<endl;
 991       }
 992       addSuckRequest(di.zone, *di.masters.begin());
 993     }
 994   }
 995 }