modules/lmdbbackend/lmdbbackend.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #ifdef HAVE_CONFIG_H
  23 #include "config.h"
  24 #endif
  25 #include "pdns/utility.hh"
  26 #include "pdns/dnsbackend.hh"
  27 #include "pdns/dns.hh"
  28 #include "pdns/dnspacket.hh"
  29 #include "pdns/base32.hh"
  30 #include "pdns/dnssecinfra.hh"
  31 #include "pdns/pdnsexception.hh"
  32 #include "pdns/logger.hh"
  33 #include "pdns/version.hh"
  34 #include "pdns/arguments.hh"
  35 #include <boost/archive/binary_oarchive.hpp>
  36 #include <boost/archive/binary_iarchive.hpp>
  37 #include <boost/serialization/vector.hpp>
  38 #include <boost/serialization/string.hpp>
  39 #include <boost/serialization/utility.hpp>
  40 // #include <boost/iostreams/stream.hpp>
  41 // #include <boost/iostreams/stream_buffer.hpp>
  42
  43 #include <boost/iostreams/device/back_inserter.hpp>
  44 // #include <sstream>
  45
  46
  47 #include "lmdbbackend.hh"
  48
  49 LMDBBackend::LMDBBackend(const std::string& suffix)
  50 {
  51   setArgPrefix("lmdb"+suffix);
  52
  53   string syncMode = toLower(getArg("sync-mode"));
  54
  55   if(syncMode == "nosync")
  56     d_asyncFlag = MDB_NOSYNC;
  57   else if(syncMode == "nometasync")
  58     d_asyncFlag = MDB_NOMETASYNC;
  59   else if(syncMode == "mapasync")
  60     d_asyncFlag = MDB_MAPASYNC;
  61   else if(syncMode.empty())
  62     d_asyncFlag = 0;
  63   else
  64     throw std::runtime_error("Unknown sync mode "+syncMode+" requested for LMDB backend");
  65
  66   d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | d_asyncFlag, 0600), "domains");
  67   d_tmeta = std::make_shared<tmeta_t>(d_tdomains->getEnv(), "metadata");
  68   d_tkdb = std::make_shared<tkdb_t>(d_tdomains->getEnv(), "keydata");
  69   d_ttsig = std::make_shared<ttsig_t>(d_tdomains->getEnv(), "tsig");
  70
  71   auto pdnsdbi = d_tdomains->getEnv()->openDB("pdns", MDB_CREATE);
  72   auto txn = d_tdomains->getEnv()->getRWTransaction();
  73   MDBOutVal shards;
  74   if(!txn.get(pdnsdbi, "shards", shards)) {
  75
  76     d_shards = shards.get<uint32_t>();
  77     if(d_shards != atoi(getArg("shards").c_str())) {
  78       g_log << Logger::Warning<<"Note: configured number of lmdb shards ("<<atoi(getArg("shards").c_str())<<") is different from on-disk ("<<d_shards<<"). Using on-disk shard number"<<endl;
  79     }
  80   }
  81   else {
  82     d_shards = atoi(getArg("shards").c_str());
  83     txn.put(pdnsdbi, "shards", d_shards);
  84     txn.commit();
  85   }
  86     d_trecords.resize(d_shards);
  87   d_dolog = ::arg().mustDo("query-logging");
  88 }
  89
  90
  91
  92 namespace boost {
  93 namespace serialization {
  94
  95 template<class Archive>
  96 void save(Archive & ar, const DNSName& g, const unsigned int version)
  97 {
  98   if(!g.empty()) {
  99     std::string tmp = g.toDNSStringLC(); // g++ 4.8 woes
 100     ar & tmp;
 101   }
 102   else
 103     ar & "";
 104 }
 105
 106 template<class Archive>
 107 void load(Archive & ar, DNSName& g, const unsigned int version)
 108 {
 109   string tmp;
 110   ar & tmp;
 111   if(tmp.empty())
 112     g = DNSName();
 113   else
 114     g = DNSName(tmp.c_str(), tmp.size(), 0, false);
 115 }
 116
 117 template<class Archive>
 118 void save(Archive & ar, const QType& g, const unsigned int version)
 119 {
 120   uint16_t tmp = g.getCode(); // g++ 4.8 woes
 121   ar & tmp;
 122 }
 123
 124 template<class Archive>
 125 void load(Archive & ar, QType& g, const unsigned int version)
 126 {
 127   uint16_t tmp;
 128   ar & tmp;
 129   g = QType(tmp);
 130 }
 131
 132 template<class Archive>
 133 void serialize(Archive & ar, DomainInfo& g, const unsigned int version)
 134 {
 135   ar & g.zone;
 136   ar & g.last_check;
 137   ar & g.account;
 138   ar & g.masters;
 139   ar & g.id;
 140   ar & g.notified_serial;
 141   ar & g.kind;
 142 }
 143
 144 template<class Archive>
 145 void serialize(Archive & ar, LMDBBackend::DomainMeta& g, const unsigned int version)
 146 {
 147   ar & g.domain & g.key & g.value;
 148 }
 149
 150 template<class Archive>
 151 void serialize(Archive & ar, LMDBBackend::KeyDataDB& g, const unsigned int version)
 152 {
 153   ar & g.domain & g.content & g.flags & g.active;
 154 }
 155
 156 template<class Archive>
 157 void serialize(Archive & ar, TSIGKey& g, const unsigned int version)
 158 {
 159   ar & g.name;
 160   ar & g.algorithm; // this is the ordername
 161   ar & g.key;
 162 }
 163
 164
 165
 166 } // namespace serialization
 167 } // namespace boost
 168
 169 BOOST_SERIALIZATION_SPLIT_FREE(DNSName);
 170 BOOST_SERIALIZATION_SPLIT_FREE(QType);
 171 BOOST_IS_BITWISE_SERIALIZABLE(ComboAddress);
 172
 173 template<>
 174 std::string serToString(const DNSResourceRecord& rr)
 175 {
 176   // only does content, ttl, auth
 177   std::string ret;
 178   uint16_t len = rr.content.length();
 179   ret.reserve(2+len+8);
 180
 181   ret.assign((const char*)&len, 2);
 182   ret += rr.content;
 183   ret.append((const char*)&rr.ttl, 4);
 184   ret.append(1, (char)rr.auth);
 185   ret.append(1, (char)false);
 186   ret.append(1, (char)rr.disabled);
 187   return ret;
 188 }
 189
 190 template<>
 191 void serFromString(const string_view& str, DNSResourceRecord& rr)
 192 {
 193   uint16_t len;
 194   memcpy(&len, &str[0], 2);
 195   rr.content.assign(&str[2], len);    // len bytes
 196   memcpy(&rr.ttl, &str[2] + len, 4);
 197   rr.auth = str[str.size()-3];
 198   rr.disabled = str[str.size()-1];
 199   rr.wildcardname.clear();
 200 }
 201
 202
 203 std::string serializeContent(uint16_t qtype, const DNSName& domain, const std::string& content)
 204 {
 205   auto drc = DNSRecordContent::mastermake(qtype, 1, content);
 206   return drc->serialize(domain, false);
 207 }
 208
 209 std::shared_ptr<DNSRecordContent> unserializeContentZR(uint16_t qtype, const DNSName& qname, const std::string& content)
 210 {
 211   if(qtype == QType::A && content.size() == 4) {
 212     return std::make_shared<ARecordContent>(*((uint32_t*)content.c_str()));
 213   }
 214   return DNSRecordContent::unserialize(qname, qtype, content);
 215 }
 216
 217
 218 /* design. If you ask a question without a zone id, we lookup the best
 219    zone id for you, and answer from that. This is different than other backends, but I can't see why it would not work.
 220
 221    The index we use is "zoneid,canonical relative name". This index is also used
 222    for AXFR.
 223
 224    Note - domain_id, name and type are ONLY present on the index!
 225 */
 226
 227 #if BOOST_VERSION <= 105400
 228 #define StringView string
 229 #else
 230 #define StringView string_view
 231 #endif
 232
 233 void LMDBBackend::deleteDomainRecords(RecordsRWTransaction& txn, uint32_t domain_id, uint16_t qtype)
 234 {
 235   compoundOrdername co;
 236   string match = co(domain_id);
 237
 238   auto cursor = txn.txn.getCursor(txn.db->dbi);
 239   MDBOutVal key, val;
 240   //  cout<<"Match: "<<makeHexDump(match);
 241   if(!cursor.lower_bound(match, key, val) ) {
 242     while(key.get<StringView>().rfind(match, 0) == 0) {
 243       if(qtype == QType::ANY || co.getQType(key.get<StringView>()) == qtype)
 244         cursor.del(MDB_NODUPDATA);
 245       if(cursor.next(key, val)) break;
 246     }
 247   }
 248 }
 249
 250 /* Here's the complicated story. Other backends have just one transaction, which is either
 251    on or not.
 252
 253    You can't call feedRecord without a transaction started with startTransaction.
 254
 255    However, other functions can be called after startTransaction() or without startTransaction()
 256      (like updateDNSSECOrderNameAndAuth)
 257
 258
 259
 260 */
 261
 262 bool LMDBBackend::startTransaction(const DNSName &domain, int domain_id)
 263 {
 264   // cout <<"startTransaction("<<domain<<", "<<domain_id<<")"<<endl;
 265   int real_id = domain_id;
 266   if(real_id < 0) {
 267     auto rotxn = d_tdomains->getROTransaction();
 268     DomainInfo di;
 269     real_id = rotxn.get<0>(domain, di);
 270     // cout<<"real_id = "<<real_id << endl;
 271     if(!real_id)
 272       return false;
 273   }
 274   if(d_rwtxn) {
 275     throw DBException("Attempt to start a transaction while one was open already");
 276   }
 277   d_rwtxn = getRecordsRWTransaction(real_id);
 278
 279   d_transactiondomain = domain;
 280   d_transactiondomainid = real_id;
 281   if(domain_id >= 0) {
 282     deleteDomainRecords(*d_rwtxn, domain_id);
 283   }
 284
 285   return true;
 286 }
 287
 288 bool LMDBBackend::commitTransaction()
 289 {
 290   // cout<<"Commit transaction" <<endl;
 291   d_rwtxn->txn.commit();
 292   d_rwtxn.reset();
 293   return true;
 294 }
 295
 296 bool LMDBBackend::abortTransaction()
 297 {
 298   // cout<<"Abort transaction"<<endl;
 299   d_rwtxn->txn.abort();
 300   d_rwtxn.reset();
 301
 302   return true;
 303 }
 304
 305 // d_rwtxn must be set here
 306 bool LMDBBackend::feedRecord(const DNSResourceRecord &r, const DNSName &ordername, bool ordernameIsNSEC3)
 307 {
 308   DNSResourceRecord rr(r);
 309   rr.qname.makeUsRelative(d_transactiondomain);
 310   rr.content = serializeContent(rr.qtype.getCode(), r.qname, rr.content);
 311   rr.disabled = false;
 312
 313   compoundOrdername co;
 314   d_rwtxn->txn.put(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, rr.qtype.getCode()), serToString(rr));
 315
 316   if(ordernameIsNSEC3 && !ordername.empty()) {
 317     MDBOutVal val;
 318     if(d_rwtxn->txn.get(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, QType::NSEC3), val)) {
 319       rr.ttl = 0;
 320       rr.content=rr.qname.toDNSStringLC();
 321       rr.auth = 0;
 322       string ser = serToString(rr);
 323       d_rwtxn->txn.put(d_rwtxn->db->dbi, co(r.domain_id, ordername, QType::NSEC3), ser);
 324
 325       rr.ttl = 1;
 326       rr.content = ordername.toDNSString();
 327       ser = serToString(rr);
 328       d_rwtxn->txn.put(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, QType::NSEC3), ser);
 329     }
 330   }
 331   return true;
 332 }
 333
 334 bool LMDBBackend::feedEnts(int domain_id, map<DNSName,bool>& nonterm)
 335 {
 336   DNSResourceRecord rr;
 337   rr.ttl = 0;
 338   compoundOrdername co;
 339   for(const auto& nt: nonterm) {
 340     rr.qname = nt.first.makeRelative(d_transactiondomain);
 341     rr.auth = nt.second;
 342     rr.disabled = true;
 343
 344     std::string ser = serToString(rr);
 345     d_rwtxn->txn.put(d_rwtxn->db->dbi, co(domain_id, rr.qname, 0), ser);
 346   }
 347   return true;
 348 }
 349
 350 bool LMDBBackend::feedEnts3(int domain_id, const DNSName &domain, map<DNSName,bool> &nonterm, const NSEC3PARAMRecordContent& ns3prc, bool narrow)
 351 {
 352   string ser;
 353   DNSName ordername;
 354   DNSResourceRecord rr;
 355   compoundOrdername co;
 356   for(const auto& nt: nonterm) {
 357     rr.qname = nt.first.makeRelative(domain);
 358     rr.ttl = 0;
 359     rr.auth = nt.second;
 360     rr.disabled = nt.second;
 361     ser = serToString(rr);
 362     d_rwtxn->txn.put(d_rwtxn->db->dbi, co(domain_id, rr.qname, 0), ser);
 363
 364     if(!narrow && rr.auth) {
 365       rr.content = rr.qname.toDNSString();
 366       rr.auth = false;
 367       rr.disabled = false;
 368       ser = serToString(rr);
 369
 370       ordername=DNSName(toBase32Hex(hashQNameWithSalt(ns3prc, nt.first)));
 371       d_rwtxn->txn.put(d_rwtxn->db->dbi, co(domain_id, ordername, QType::NSEC3), ser);
 372
 373       rr.ttl = 1;
 374       rr.content = ordername.toDNSString();
 375       ser = serToString(rr);
 376       d_rwtxn->txn.put(d_rwtxn->db->dbi, co(domain_id, rr.qname, QType::NSEC3), ser);
 377     }
 378   }
 379   return true;
 380 }
 381
 382
 383 // might be called within a transaction, might also be called alone
 384 bool LMDBBackend::replaceRRSet(uint32_t domain_id, const DNSName& qname, const QType& qt, const vector<DNSResourceRecord>& rrset)
 385 {
 386   // zonk qname/qtype within domain_id (go through qname, check domain_id && qtype)
 387   shared_ptr<RecordsRWTransaction> txn;
 388   bool needCommit = false;
 389   if(d_rwtxn && d_transactiondomainid==domain_id) {
 390     txn = d_rwtxn;
 391     //    cout<<"Reusing open transaction"<<endl;
 392   }
 393   else {
 394     //    cout<<"Making a new RW txn for replace rrset"<<endl;
 395     txn = getRecordsRWTransaction(domain_id);
 396     needCommit = true;
 397   }
 398
 399   DomainInfo di;
 400   d_tdomains->getROTransaction().get(domain_id, di); // XX error checking
 401
 402   compoundOrdername co;
 403   auto cursor = txn->txn.getCursor(txn->db->dbi);
 404   MDBOutVal key, val;
 405   string match =co(domain_id, qname.makeRelative(di.zone), qt.getCode());
 406   if(!cursor.find(match, key, val)) {
 407     do {
 408       cursor.del(MDB_NODUPDATA);
 409     } while(!cursor.next(key, val) && key.get<StringView>().rfind(match, 0) == 0);
 410   }
 411
 412   for(auto rr : rrset) {
 413     rr.content = serializeContent(rr.qtype.getCode(), rr.qname, rr.content);
 414     rr.qname.makeUsRelative(di.zone);
 415     txn->txn.put(txn->db->dbi, match, serToString(rr));
 416   }
 417
 418   if(needCommit)
 419     txn->txn.commit();
 420
 421   return true;
 422 }
 423
 424 // tempting to templatize these two functions but the pain is not worth it
 425 std::shared_ptr<LMDBBackend::RecordsRWTransaction> LMDBBackend::getRecordsRWTransaction(uint32_t id)
 426 {
 427   auto& shard =d_trecords[id % d_shards];
 428   if(!shard.env) {
 429     shard.env = getMDBEnv( (getArg("filename")+"-"+std::to_string(id % d_shards)).c_str(),
 430                            MDB_NOSUBDIR | d_asyncFlag, 0600);
 431     shard.dbi = shard.env->openDB("records", MDB_CREATE | MDB_DUPSORT);
 432   }
 433   auto ret = std::make_shared<RecordsRWTransaction>(shard.env->getRWTransaction());
 434   ret->db = std::make_shared<RecordsDB>(shard);
 435
 436   return ret;
 437 }
 438
 439 std::shared_ptr<LMDBBackend::RecordsROTransaction> LMDBBackend::getRecordsROTransaction(uint32_t id)
 440 {
 441   auto& shard =d_trecords[id % d_shards];
 442   if(!shard.env) {
 443     shard.env = getMDBEnv( (getArg("filename")+"-"+std::to_string(id % d_shards)).c_str(),
 444                            MDB_NOSUBDIR | d_asyncFlag, 0600);
 445     shard.dbi = shard.env->openDB("records", MDB_CREATE | MDB_DUPSORT);
 446   }
 447
 448   auto ret = std::make_shared<RecordsROTransaction>(shard.env->getROTransaction());
 449   ret->db = std::make_shared<RecordsDB>(shard);
 450   return ret;
 451 }
 452
 453
 454 bool LMDBBackend::deleteDomain(const DNSName &domain)
 455 {
 456   auto doms = d_tdomains->getRWTransaction();
 457
 458   DomainInfo di;
 459   auto id = doms.get<0>(domain, di);
 460   if(!id)
 461     return false;
 462
 463   shared_ptr<RecordsRWTransaction> txn;
 464   bool needCommit = false;
 465   if(d_rwtxn && d_transactiondomainid == id) {
 466     txn = d_rwtxn;
 467     //    cout<<"Reusing open transaction"<<endl;
 468   }
 469   else {
 470     //    cout<<"Making a new RW txn for delete domain"<<endl;
 471     txn = getRecordsRWTransaction(id);
 472     needCommit = true;
 473   }
 474
 475
 476   doms.del(id);
 477   compoundOrdername co;
 478   string match=co(id);
 479
 480   auto cursor = txn->txn.getCursor(txn->db->dbi);
 481   MDBOutVal key, val;
 482   if(!cursor.find(match, key, val)) {
 483     do {
 484       cursor.del(MDB_NODUPDATA);
 485     } while(!cursor.next(key, val) && key.get<StringView>().rfind(match, 0) == 0);
 486   }
 487
 488   if(needCommit)
 489     txn->txn.commit();
 490
 491   doms.commit();
 492
 493   return true;
 494 }
 495
 496 bool LMDBBackend::list(const DNSName &target, int id, bool include_disabled)
 497 {
 498   d_inlist=true;
 499   DomainInfo di;
 500   {
 501     auto dtxn = d_tdomains->getROTransaction();
 502
 503     if((di.id = dtxn.get<0>(target, di)))
 504       ; //      cout<<"Found domain "<<target<<" on domain_id "<<di.id <<", list requested "<<id<<endl;
 505     else {
 506       // cout<<"Did not find "<<target<<endl;
 507       return false;
 508     }
 509   }
 510
 511   d_rotxn = getRecordsROTransaction(di.id);
 512   compoundOrdername co;
 513   d_matchkey = co(di.id);
 514   d_getcursor = std::make_shared<MDBROCursor>(d_rotxn->txn.getCursor(d_rotxn->db->dbi));
 515   MDBOutVal key, val;
 516   d_inlist = true;
 517
 518   if(d_getcursor->lower_bound(d_matchkey, key, val) || key.get<StringView>().rfind(d_matchkey, 0) != 0) {
 519     // cout<<"Found nothing for list"<<endl;
 520     d_getcursor.reset();
 521     return true;
 522   }
 523
 524   d_lookupqname = target;
 525
 526   return true;
 527 }
 528
 529 void LMDBBackend::lookup(const QType &type, const DNSName &qdomain, DNSPacket *p, int zoneId)
 530 {
 531   if(d_dolog) {
 532     g_log << Logger::Warning << "Got lookup for "<<qdomain<<"|"<<type.getName()<<" in zone "<< zoneId<<endl;
 533     d_dtime.set();
 534   }
 535   DNSName hunt(qdomain);
 536   if(zoneId < 0) {
 537     auto rotxn = d_tdomains->getROTransaction();
 538
 539     for(;;) {
 540       DomainInfo di;
 541       if((zoneId = rotxn.get<0>(hunt, di))) {
 542         break;
 543       }
 544       if(!hunt.chopOff())
 545         break;
 546     }
 547     if(zoneId <= 0) {
 548       //      cout << "Did not find zone for "<< qdomain<<endl;
 549       d_getcursor.reset();
 550       return;
 551     }
 552   }
 553   else {
 554     DomainInfo di;
 555     if(!d_tdomains->getROTransaction().get(zoneId, di)) {
 556       // cout<<"Could not find a zone with id "<<zoneId<<endl;
 557       d_getcursor.reset();
 558       return;
 559     }
 560     hunt = di.zone;
 561   }
 562
 563   DNSName relqname = qdomain.makeRelative(hunt);
 564   //  cout<<"get will look for "<<relqname<< " in zone "<<hunt<<" with id "<<zoneId<<endl;
 565   d_rotxn = getRecordsROTransaction(zoneId);
 566
 567   compoundOrdername co;
 568   d_getcursor = std::make_shared<MDBROCursor>(d_rotxn->txn.getCursor(d_rotxn->db->dbi));
 569   MDBOutVal key, val;
 570   if(type.getCode() == QType::ANY) {
 571     d_matchkey = co(zoneId,relqname);
 572   }
 573   else {
 574     d_matchkey= co(zoneId,relqname, type.getCode());
 575   }
 576   d_inlist=false;
 577
 578   if(d_getcursor->lower_bound(d_matchkey, key, val) || key.get<StringView>().rfind(d_matchkey, 0) != 0) {
 579     d_getcursor.reset();
 580     if(d_dolog) {
 581       g_log<<Logger::Warning<< "Query "<<((long)(void*)this)<<": "<<d_dtime.udiffNoReset()<<" usec to execute (found nothing)"<<endl;
 582     }
 583     return;
 584   }
 585
 586   if(d_dolog) {
 587     g_log<<Logger::Warning<< "Query "<<((long)(void*)this)<<": "<<d_dtime.udiffNoReset()<<" usec to execute"<<endl;
 588   }
 589
 590   d_lookuptype=type;
 591   d_lookupqname = qdomain;
 592   d_lookupdomain = hunt;
 593   d_lookupdomainid = zoneId;
 594 }
 595
 596 bool LMDBBackend::get(DNSZoneRecord& rr)
 597 {
 598   if(d_inlist)
 599     return get_list(rr);
 600   else
 601     return get_lookup(rr);
 602 }
 603
 604 bool LMDBBackend::get(DNSResourceRecord& rr)
 605 {
 606   //  cout <<"Old-school get called"<<endl;
 607   DNSZoneRecord dzr;
 608   if(d_inlist) {
 609     if(!get_list(dzr))
 610       return false;
 611   }
 612   else {
 613     if(!get_lookup(dzr))
 614       return false;
 615   }
 616   rr.qname = dzr.dr.d_name;
 617   rr.ttl = dzr.dr.d_ttl;
 618   rr.qtype =dzr.dr.d_type;
 619   rr.content = dzr.dr.d_content->getZoneRepresentation();
 620   rr.domain_id = dzr.domain_id;
 621   //  cout<<"old school called for "<<rr.qname<<", "<<rr.qtype.getName()<<endl;
 622   return true;
 623 }
 624
 625 bool LMDBBackend::getSOA(const DNSName &domain, SOAData &sd)
 626 {
 627   //  cout <<"Native getSOA called"<<endl;
 628   lookup(QType(QType::SOA), domain, 0, -1);
 629   DNSZoneRecord dzr;
 630   bool found=false;
 631   while(get(dzr)) {
 632     auto src = getRR<SOARecordContent>(dzr.dr);
 633     sd.domain_id = dzr.domain_id;
 634     sd.ttl = dzr.dr.d_ttl;
 635     sd.qname = dzr.dr.d_name;
 636
 637     sd.nameserver = src->d_mname;
 638     sd.hostmaster = src->d_rname;
 639     sd.serial = src->d_st.serial;
 640     sd.refresh = src->d_st.refresh;
 641     sd.retry = src->d_st.retry;
 642     sd.expire = src->d_st.expire;
 643     sd.default_ttl = src->d_st.minimum;
 644
 645     sd.db = this;
 646     found=true;
 647   }
 648   return found;
 649 }
 650 bool LMDBBackend::get_list(DNSZoneRecord& rr)
 651 {
 652   for(;;) {
 653     if(!d_getcursor)  {
 654       d_rotxn.reset();
 655       return false;
 656     }
 657
 658     MDBOutVal keyv, val;
 659
 660     d_getcursor->current(keyv, val);
 661     DNSResourceRecord drr;
 662     serFromString(val.get<string>(), drr);
 663
 664     auto key = keyv.get<string_view>();
 665     rr.dr.d_name = compoundOrdername::getQName(key) + d_lookupqname;
 666     rr.domain_id = compoundOrdername::getDomainID(key);
 667     rr.dr.d_type = compoundOrdername::getQType(key).getCode();
 668     rr.dr.d_ttl = drr.ttl;
 669     rr.auth = drr.auth;
 670
 671     if(rr.dr.d_type == QType::NSEC3) {
 672       //      cout << "Had a magic NSEC3, skipping it" << endl;
 673       if(d_getcursor->next(keyv, val) || keyv.get<StringView>().rfind(d_matchkey, 0) != 0) {
 674         d_getcursor.reset();
 675       }
 676       continue;
 677     }
 678     rr.dr.d_content = unserializeContentZR(rr.dr.d_type, rr.dr.d_name, drr.content);
 679
 680     if(d_getcursor->next(keyv, val) || keyv.get<StringView>().rfind(d_matchkey, 0) != 0) {
 681       d_getcursor.reset();
 682     }
 683     break;
 684   }
 685   return true;
 686 }
 687
 688
 689 bool LMDBBackend::get_lookup(DNSZoneRecord& rr)
 690 {
 691   for(;;) {
 692     if(!d_getcursor) {
 693       d_rotxn.reset();
 694       return false;
 695     }
 696     MDBOutVal keyv, val;
 697     d_getcursor->current(keyv, val);
 698     DNSResourceRecord drr;
 699     serFromString(val.get<string>(), drr);
 700
 701     auto key = keyv.get<string_view>();
 702
 703     rr.dr.d_name = compoundOrdername::getQName(key) + d_lookupdomain;
 704
 705     rr.domain_id = compoundOrdername::getDomainID(key);
 706     //  cout << "We found "<<rr.qname<< " in zone id "<<rr.domain_id <<endl;
 707     rr.dr.d_type = compoundOrdername::getQType(key).getCode();
 708     rr.dr.d_ttl = drr.ttl;
 709     if(rr.dr.d_type == QType::NSEC3) {
 710       //      cout << "Hit a magic NSEC3 skipping" << endl;
 711       if(d_getcursor->next(keyv, val) || keyv.get<StringView>().rfind(d_matchkey, 0) != 0) {
 712         d_getcursor.reset();
 713         d_rotxn.reset();
 714       }
 715       continue;
 716     }
 717
 718     rr.dr.d_content = unserializeContentZR(rr.dr.d_type, rr.dr.d_name, drr.content);
 719     rr.auth = drr.auth;
 720     if(d_getcursor->next(keyv, val) || keyv.get<StringView>().rfind(d_matchkey, 0) != 0) {
 721       d_getcursor.reset();
 722       d_rotxn.reset();
 723     }
 724     break;
 725   }
 726
 727
 728   return true;
 729 }
 730
 731
 732 bool LMDBBackend::getDomainInfo(const DNSName &domain, DomainInfo &di, bool getSerial)
 733 {
 734   auto txn = d_tdomains->getROTransaction();
 735
 736   if(!(di.id=txn.get<0>(domain, di)))
 737     return false;
 738   di.backend = this;
 739   return true;
 740 }
 741
 742
 743 int LMDBBackend::genChangeDomain(const DNSName& domain, std::function<void(DomainInfo&)> func)
 744 {
 745   auto txn = d_tdomains->getRWTransaction();
 746
 747   DomainInfo di;
 748
 749   auto id = txn.get<0>(domain, di);
 750   func(di);
 751   txn.put(di, id);
 752
 753   txn.commit();
 754   return true;
 755 }
 756
 757 int LMDBBackend::genChangeDomain(uint32_t id, std::function<void(DomainInfo&)> func)
 758 {
 759   DomainInfo di;
 760
 761   auto txn = d_tdomains->getRWTransaction();
 762
 763   if(!txn.get(id , di))
 764     return false;
 765
 766   func(di);
 767
 768   txn.put(di, id);
 769
 770   txn.commit();
 771   return true;
 772 }
 773
 774
 775 bool LMDBBackend::setKind(const DNSName &domain, const DomainInfo::DomainKind kind)
 776 {
 777   return genChangeDomain(domain, [kind](DomainInfo& di) {
 778       di.kind = kind;
 779     });
 780 }
 781
 782 bool LMDBBackend::setAccount(const DNSName &domain, const std::string& account)
 783 {
 784   return genChangeDomain(domain, [account](DomainInfo& di) {
 785       di.account = account;
 786     });
 787 }
 788
 789
 790 void LMDBBackend::setFresh(uint32_t domain_id)
 791 {
 792   genChangeDomain(domain_id, [](DomainInfo& di) {
 793       di.last_check = time(0);
 794     });
 795 }
 796
 797 void LMDBBackend::setNotified(uint32_t domain_id, uint32_t serial)
 798 {
 799   genChangeDomain(domain_id, [serial](DomainInfo& di) {
 800       di.serial = serial;
 801     });
 802 }
 803
 804
 805 bool LMDBBackend::setMaster(const DNSName &domain, const std::string& ips)
 806 {
 807   vector<ComboAddress> masters;
 808   vector<string> parts;
 809   stringtok(parts, ips, " \t;,");
 810   for(const auto& ip : parts)
 811     masters.push_back(ComboAddress(ip));
 812
 813   return genChangeDomain(domain, [&masters](DomainInfo& di) {
 814       di.masters = masters;
 815     });
 816 }
 817
 818 bool LMDBBackend::createDomain(const DNSName &domain)
 819 {
 820   return createDomain(domain, "NATIVE", "", "");
 821 }
 822
 823 bool LMDBBackend::createDomain(const DNSName &domain, const string &type, const string &masters, const string &account)
 824 {
 825   DomainInfo di;
 826
 827   auto txn = d_tdomains->getRWTransaction();
 828   if(txn.get<0>(domain, di)) {
 829     throw DBException("Domain '"+domain.toLogString()+"' exists already");
 830   }
 831
 832   di.zone = domain;
 833   if(pdns_iequals(type, "master"))
 834     di.kind = DomainInfo::Master;
 835   else if(pdns_iequals(type, "slave"))
 836     di.kind = DomainInfo::Slave;
 837   else if(pdns_iequals(type, "native"))
 838     di.kind = DomainInfo::Native;
 839   else
 840     throw DBException("Unable to create domain of unknown type '"+type+"'");
 841   di.account = account;
 842
 843   txn.put(di);
 844   txn.commit();
 845
 846   return true;
 847 }
 848
 849 void LMDBBackend::getAllDomains(vector<DomainInfo> *domains, bool include_disabled)
 850 {
 851   compoundOrdername co;
 852   MDBOutVal val;
 853   domains->clear();
 854   auto txn = d_tdomains->getROTransaction();
 855   for(auto iter = txn.begin(); iter != txn.end(); ++iter) {
 856     DomainInfo di=*iter;
 857     di.id = iter.getID();
 858
 859     auto txn = getRecordsROTransaction(iter.getID());
 860     if(!txn->txn.get(txn->db->dbi, co(di.id, g_rootdnsname, QType::SOA), val)) {
 861       domains->push_back(di);
 862     }
 863   }
 864 }
 865
 866 void LMDBBackend::getUnfreshSlaveInfos(vector<DomainInfo>* domains)
 867 {
 868   //  cout<<"Start of getUnfreshSlaveInfos"<<endl;
 869   domains->clear();
 870   auto txn = d_tdomains->getROTransaction();
 871
 872   time_t now = time(0);
 873   for(auto iter = txn.begin(); iter != txn.end(); ++iter) {
 874     if(iter->kind != DomainInfo::Slave)
 875       continue;
 876
 877     auto txn2 = getRecordsROTransaction(iter.getID());
 878     compoundOrdername co;
 879     MDBOutVal val;
 880     uint32_t serial = 0;
 881     if(!txn2->txn.get(txn2->db->dbi, co(iter.getID(), g_rootdnsname, QType::SOA), val)) {
 882       DNSResourceRecord rr;
 883       serFromString(val.get<string_view>(), rr);
 884       struct soatimes
 885       {
 886         uint32_t serial;
 887         uint32_t refresh;
 888         uint32_t retry;
 889         uint32_t expire;
 890         uint32_t minimum;
 891       } st;
 892
 893       memcpy(&st, &rr.content[rr.content.size()-sizeof(soatimes)], sizeof(soatimes));
 894
 895       if((time_t)(iter->last_check + ntohl(st.refresh)) >= now) { // still fresh
 896         continue; // try next domain
 897       }
 898       //      cout << di.last_check <<" + " <<sdata.refresh<<" > = " << now << "\n";
 899       serial = ntohl(st.serial);
 900     }
 901     else {
 902       //      cout << "Could not find SOA for "<<iter->zone<<" with id "<<iter.getID()<<endl;
 903       serial=0;
 904     }
 905     DomainInfo di=*iter;
 906     di.id = iter.getID();
 907     di.serial = serial;
 908
 909     domains->push_back(di);
 910   }
 911   //  cout<<"END of getUnfreshSlaveInfos"<<endl;
 912 }
 913
 914 bool LMDBBackend::getAllDomainMetadata(const DNSName& name, std::map<std::string, std::vector<std::string> >& meta)
 915 {
 916   meta.clear();
 917   auto txn = d_tmeta->getROTransaction();
 918   auto range = txn.equal_range<0>(name);
 919
 920   for(auto& iter = range.first; iter != range.second; ++iter) {
 921     meta[iter->key].push_back(iter->value);
 922   }
 923   return true;
 924 }
 925
 926 bool LMDBBackend::setDomainMetadata(const DNSName& name, const std::string& kind, const std::vector<std::string>& meta)
 927 {
 928   auto txn = d_tmeta->getRWTransaction();
 929
 930   auto range = txn.equal_range<0>(name);
 931
 932   for(auto& iter = range.first; iter != range.second; ++iter) {
 933     if(iter-> key == kind)
 934       iter.del();
 935   }
 936
 937   for(const auto& m : meta) {
 938     DomainMeta dm{name, kind, m};
 939     txn.put(dm);
 940   }
 941   txn.commit();
 942   return true;
 943
 944 }
 945
 946 bool LMDBBackend::getDomainKeys(const DNSName& name, std::vector<KeyData>& keys)
 947 {
 948   auto txn = d_tkdb->getROTransaction();
 949   auto range = txn.equal_range<0>(name);
 950   for(auto& iter = range.first; iter != range.second; ++iter) {
 951     KeyData kd{iter->content, iter.getID(), iter->flags, iter->active};
 952     keys.push_back(kd);
 953   }
 954
 955   return true;
 956 }
 957
 958 bool LMDBBackend::removeDomainKey(const DNSName& name, unsigned int id)
 959 {
 960   auto txn = d_tkdb->getRWTransaction();
 961   KeyDataDB kdb;
 962   if(txn.get(id, kdb)) {
 963     if(kdb.domain == name) {
 964       txn.del(id);
 965       txn.commit();
 966       return true;
 967     }
 968   }
 969   // cout << "??? wanted to remove domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
 970   return true;
 971 }
 972
 973 bool LMDBBackend::addDomainKey(const DNSName& name, const KeyData& key, int64_t& id)
 974 {
 975   auto txn = d_tkdb->getRWTransaction();
 976   KeyDataDB kdb{name, key.content, key.flags, key.active};
 977   id = txn.put(kdb);
 978   txn.commit();
 979
 980   return true;
 981 }
 982
 983 bool LMDBBackend::activateDomainKey(const DNSName& name, unsigned int id)
 984 {
 985   auto txn = d_tkdb->getRWTransaction();
 986   KeyDataDB kdb;
 987   if(txn.get(id, kdb)) {
 988     if(kdb.domain == name) {
 989       txn.modify(id, [](KeyDataDB& kdb)
 990                  {
 991                    kdb.active = true;
 992                  });
 993       txn.commit();
 994       return true;
 995     }
 996   }
 997
 998   // cout << "??? wanted to activate domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
 999   return true;
1000 }
1001
1002 bool LMDBBackend::deactivateDomainKey(const DNSName& name, unsigned int id)
1003 {
1004   auto txn = d_tkdb->getRWTransaction();
1005   KeyDataDB kdb;
1006   if(txn.get(id, kdb)) {
1007     if(kdb.domain == name) {
1008       txn.modify(id, [](KeyDataDB& kdb)
1009                  {
1010                    kdb.active = false;
1011                  });
1012       txn.commit();
1013       return true;
1014     }
1015   }
1016   // cout << "??? wanted to activate domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
1017   return true;
1018 }
1019
1020 bool LMDBBackend::getBeforeAndAfterNamesAbsolute(uint32_t id, const DNSName& qname, DNSName& unhashed, DNSName& before, DNSName& after)
1021 {
1022   //  cout << __PRETTY_FUNCTION__<< ": "<<id <<", "<<qname << " " << unhashed<<endl;
1023
1024   DomainInfo di;
1025   if(!d_tdomains->getROTransaction().get(id, di)) {
1026     // domain does not exist, tough luck
1027     return false;
1028   }
1029   // cout <<"Zone: "<<di.zone<<endl;
1030
1031   compoundOrdername co;
1032   auto txn = getRecordsROTransaction(id);
1033
1034   auto cursor = txn->txn.getCursor(txn->db->dbi);
1035   MDBOutVal key, val;
1036
1037   DNSResourceRecord rr;
1038
1039   string matchkey = co(id, qname, QType::NSEC3);
1040   if(cursor.lower_bound(matchkey, key, val)) {
1041     // this is beyond the end of the database
1042     // cout << "Beyond end of database!" << endl;
1043     cursor.last(key, val);
1044
1045     for(;;) {
1046       if(co.getDomainID(key.get<StringView>()) != id) {
1047         //cout<<"Last record also not part of this zone!"<<endl;
1048         // this implies something is wrong in the database, nothing we can do
1049         return false;
1050       }
1051
1052       if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1053         serFromString(val.get<StringView>(), rr);
1054         if(!rr.ttl) // the kind of NSEC3 we need
1055           break;
1056       }
1057       if(cursor.prev(key, val)) {
1058         // hit beginning of database, again means something is wrong with it
1059         return false;
1060       }
1061     }
1062     before = co.getQName(key.get<StringView>());
1063     unhashed = DNSName(rr.content.c_str(), rr.content.size(), 0, false) + di.zone;
1064
1065     // now to find after .. at the beginning of the zone
1066     if(cursor.lower_bound(co(id), key, val)) {
1067       // cout<<"hit end of zone find when we shouldn't"<<endl;
1068       return false;
1069     }
1070     for(;;) {
1071       if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1072         serFromString(val.get<StringView>(), rr);
1073         if(!rr.ttl)
1074           break;
1075       }
1076
1077       if(cursor.next(key, val) || co.getDomainID(key.get<StringView>()) != id) {
1078         // cout<<"hit end of zone or database when we shouldn't"<<endl;
1079         return false;
1080       }
1081     }
1082     after = co.getQName(key.get<StringView>());
1083     // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1084     return true;
1085   }
1086
1087   // cout<<"Ended up at "<<co.getQName(key.get<StringView>()) <<endl;
1088
1089   before = co.getQName(key.get<StringView>());
1090   if(before == qname) {
1091     // cout << "Ended up on exact right node" << endl;
1092     before = co.getQName(key.get<StringView>());
1093     // unhashed should be correct now, maybe check?
1094     if(cursor.next(key, val)) {
1095       // xxx should find first hash now
1096
1097       if(cursor.lower_bound(co(id), key, val)) {
1098         // cout<<"hit end of zone find when we shouldn't for id "<<id<< __LINE__<<endl;
1099         return false;
1100       }
1101       for(;;) {
1102         if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1103           serFromString(val.get<StringView>(), rr);
1104           if(!rr.ttl)
1105             break;
1106         }
1107
1108         if(cursor.next(key, val) || co.getDomainID(key.get<StringView>()) != id) {
1109           // cout<<"hit end of zone or database when we shouldn't" << __LINE__<<endl;
1110           return false;
1111         }
1112       }
1113       after = co.getQName(key.get<StringView>());
1114       // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1115       return true;
1116     }
1117   }
1118   else {
1119     // cout <<"Going backwards to find 'before'"<<endl;
1120     int count=0;
1121     for(;;) {
1122       if(co.getQName(key.get<StringView>()).canonCompare(qname) && co.getQType(key.get<StringView>()) == QType::NSEC3) {
1123         // cout<<"Potentially stopping traverse at "<< co.getQName(key.get<StringView>()) <<", " << (co.getQName(key.get<StringView>()).canonCompare(qname))<<endl;
1124         // cout<<"qname = "<<qname<<endl;
1125         // cout<<"here  = "<<co.getQName(key.get<StringView>())<<endl;
1126         serFromString(val.get<StringView>(), rr);
1127         if(!rr.ttl)
1128           break;
1129       }
1130
1131       if(cursor.prev(key, val) || co.getDomainID(key.get<StringView>()) != id ) {
1132         // cout <<"XXX Hit *beginning* of zone or database"<<endl;
1133         // this can happen, must deal with it
1134         // should now find the last hash of the zone
1135
1136         if(cursor.lower_bound(co(id+1), key, val)) {
1137           // cout << "Could not find the next higher zone, going to the end of the database then"<<endl;
1138           cursor.last(key, val);
1139         }
1140         else
1141           cursor.prev(key, val);
1142
1143         for(;;) {
1144           if(co.getDomainID(key.get<StringView>()) != id) {
1145             //cout<<"Last record also not part of this zone!"<<endl;
1146             // this implies something is wrong in the database, nothing we can do
1147             return false;
1148           }
1149
1150           if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1151             serFromString(val.get<StringView>(), rr);
1152             if(!rr.ttl) // the kind of NSEC3 we need
1153               break;
1154           }
1155           if(cursor.prev(key, val)) {
1156             // hit beginning of database, again means something is wrong with it
1157             return false;
1158           }
1159         }
1160         before = co.getQName(key.get<StringView>());
1161         unhashed = DNSName(rr.content.c_str(), rr.content.size(), 0, false) + di.zone;
1162         // cout <<"Should still find 'after'!"<<endl;
1163         // for 'after', we need to find the first hash of this zone
1164
1165         if(cursor.lower_bound(co(id), key, val)) {
1166           // cout<<"hit end of zone find when we shouldn't"<<endl;
1167           // means database is wrong, nothing we can do
1168           return false;
1169         }
1170         for(;;) {
1171           if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1172             serFromString(val.get<StringView>(), rr);
1173             if(!rr.ttl)
1174               break;
1175           }
1176
1177           if(cursor.next(key, val)) {
1178             // means database is wrong, nothing we can do
1179             // cout<<"hit end of zone when we shouldn't 2"<<endl;
1180             return false;
1181           }
1182         }
1183         after = co.getQName(key.get<StringView>());
1184
1185
1186         // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1187         return true;
1188       }
1189       ++count;
1190     }
1191     before = co.getQName(key.get<StringView>());
1192     unhashed = DNSName(rr.content.c_str(), rr.content.size(), 0, false) + di.zone;
1193     // cout<<"Went backwards, found "<<before<<endl;
1194     // return us to starting point
1195     while(count--)
1196       cursor.next(key, val);
1197   }
1198   //  cout<<"Now going forward"<<endl;
1199   for(int count = 0 ;;++count) {
1200     if((count && cursor.next(key, val)) || co.getDomainID(key.get<StringView>()) != id ) {
1201       // cout <<"Hit end of database or zone, finding first hash then in zone "<<id<<endl;
1202       if(cursor.lower_bound(co(id), key, val)) {
1203         // cout<<"hit end of zone find when we shouldn't"<<endl;
1204         // means database is wrong, nothing we can do
1205         return false;
1206       }
1207       for(;;) {
1208         if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1209           serFromString(val.get<StringView>(), rr);
1210           if(!rr.ttl)
1211             break;
1212         }
1213
1214         if(cursor.next(key, val)) {
1215           // means database is wrong, nothing we can do
1216           // cout<<"hit end of zone when we shouldn't 2"<<endl;
1217           return false;
1218         }
1219         // cout << "Next.. "<<endl;
1220       }
1221       after = co.getQName(key.get<StringView>());
1222
1223       // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1224       return true;
1225     }
1226
1227     // cout<<"After "<<co.getQName(key.get<StringView>()) <<endl;
1228     if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1229       serFromString(val.get<StringView>(), rr);
1230       if(!rr.ttl) {
1231         break;
1232       }
1233     }
1234   }
1235   after = co.getQName(key.get<StringView>());
1236   // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1237   return true;
1238 }
1239
1240 bool LMDBBackend::getBeforeAndAfterNames(uint32_t id, const DNSName& zonenameU, const DNSName& qname, DNSName& before, DNSName& after)
1241 {
1242   DNSName zonename = zonenameU.makeLowerCase();
1243   //  cout << __PRETTY_FUNCTION__<< ": "<<id <<", "<<zonename << ", '"<<qname<<"'"<<endl;
1244
1245   auto txn = getRecordsROTransaction(id);
1246   compoundOrdername co;
1247   DNSName qname2 = qname.makeRelative(zonename);
1248   string matchkey=co(id,qname2);
1249   auto cursor = txn->txn.getCursor(txn->db->dbi);
1250   MDBOutVal key, val;
1251   // cout<<"Lower_bound for "<<qname2<<endl;
1252   if(cursor.lower_bound(matchkey, key, val)) {
1253     // cout << "Hit end of database, bummer"<<endl;
1254     cursor.last(key, val);
1255     if(co.getDomainID(key.get<string_view>()) == id) {
1256       before = co.getQName(key.get<string_view>()) + zonename;
1257       after = zonename;
1258     }
1259     // else
1260       // cout << "We were at end of database, but this zone is not there?!"<<endl;
1261     return true;
1262   }
1263   // cout<<"Cursor is at "<<co.getQName(key.get<string_view>()) <<", in zone id "<<co.getDomainID(key.get<string_view>())<< endl;
1264
1265   if(co.getQType(key.get<string_view>()).getCode() && co.getDomainID(key.get<string_view>()) ==id && co.getQName(key.get<string_view>()) == qname2) { // don't match ENTs
1266     // cout << "Had an exact match!"<<endl;
1267     before = qname2 + zonename;
1268     int rc;
1269     for(;;) {
1270       rc=cursor.next(key, val);
1271       if(rc) break;
1272
1273       if(co.getDomainID(key.get<string_view>()) == id && key.get<StringView>().rfind(matchkey, 0)==0)
1274         continue;
1275       DNSResourceRecord rr;
1276       serFromString(val.get<StringView>(), rr);
1277       if(co.getQType(key.get<string_view>()).getCode() && (rr.auth || co.getQType(key.get<string_view>()).getCode() == QType::NS))
1278         break;
1279     }
1280     if(rc || co.getDomainID(key.get<string_view>()) != id) {
1281       // cout << "We hit the end of the zone or database. 'after' is apex" << endl;
1282       after=zonename;
1283       return false;
1284     }
1285     after = co.getQName(key.get<string_view>()) + zonename;
1286     return true;
1287   }
1288
1289
1290   if(co.getDomainID(key.get<string_view>()) != id) {
1291     // cout << "Ended up in next zone, 'after' is zonename" <<endl;
1292     after = zonename;
1293     // cout << "Now hunting for previous" << endl;
1294     int rc;
1295     for(;;) {
1296       rc=cursor.prev(key, val);
1297       if(rc) {
1298         // cout<<"Reversed into zone, but got not found from lmdb" <<endl;
1299         return false;
1300       }
1301
1302       if(co.getDomainID(key.get<string_view>()) != id) {
1303         // cout<<"Reversed into zone, but found wrong zone id " << co.getDomainID(key.get<string_view>()) << " != "<<id<<endl;
1304         // "this can't happen"
1305         return false;
1306       }
1307       DNSResourceRecord rr;
1308       serFromString(val.get<StringView>(), rr);
1309       if(co.getQType(key.get<string_view>()).getCode() && (rr.auth || co.getQType(key.get<string_view>()).getCode() == QType::NS))
1310         break;
1311     }
1312
1313     before = co.getQName(key.get<string_view>()) + zonename;
1314     // cout<<"Found: "<< before<<endl;
1315     return true;
1316   }
1317
1318   // cout <<"We ended up after "<<qname<<", on "<<co.getQName(key.get<string_view>())<<endl;
1319
1320   int skips = 0;
1321   for(; ;) {
1322     DNSResourceRecord rr;
1323     serFromString(val.get<StringView>(), rr);
1324     if(co.getQType(key.get<string_view>()).getCode() && (rr.auth || co.getQType(key.get<string_view>()).getCode() == QType::NS)) {
1325       after = co.getQName(key.get<string_view>()) + zonename;
1326       // cout <<"Found auth ("<<rr.auth<<") or an NS record "<<after<<", type: "<<co.getQType(key.get<string_view>()).getName()<<", ttl = "<<rr.ttl<<endl;
1327       // cout << makeHexDump(val.get<string>()) << endl;
1328       break;
1329     }
1330     // cout <<"  oops, " << co.getQName(key.get<string_view>()) << " was not auth "<<rr.auth<< " type=" << rr.qtype.getName()<<" or NS, so need to skip ahead a bit more" << endl;
1331     int rc = cursor.next(key, val);
1332     if(!rc)
1333       ++skips;
1334     if(rc || co.getDomainID(key.get<string_view>()) != id ) {
1335       // cout << "  oops, hit end of database or zone. This means after is apex" <<endl;
1336       after = zonename;
1337       break;
1338     }
1339   }
1340   // go back to where we were
1341   while(skips--)
1342     cursor.prev(key,val);
1343
1344   for(;;) {
1345     int rc = cursor.prev(key, val);
1346     if(rc || co.getDomainID(key.get<string_view>()) != id) {
1347       // XX I don't think this case can happen
1348       // cout << "We hit the beginning of the zone or database.. now what" << endl;
1349       return false;
1350     }
1351     before = co.getQName(key.get<string_view>()) + zonename;
1352     DNSResourceRecord rr;
1353     serFromString(val.get<string_view>(), rr);
1354     // cout<<"And before to "<<before<<", auth = "<<rr.auth<<endl;
1355     if(co.getQType(key.get<string_view>()).getCode() && (rr.auth || co.getQType(key.get<string_view>()) == QType::NS))
1356       break;
1357     // cout << "Oops, that was wrong, go back one more"<<endl;
1358   }
1359
1360   return true;
1361
1362 }
1363
1364 bool LMDBBackend::updateDNSSECOrderNameAndAuth(uint32_t domain_id, const DNSName& qname, const DNSName& ordername, bool auth, const uint16_t qtype)
1365 {
1366   //  cout << __PRETTY_FUNCTION__<< ": "<< domain_id <<", '"<<qname <<"', '"<<ordername<<"', "<<auth<< ", " << qtype << endl;
1367   shared_ptr<RecordsRWTransaction> txn;
1368   bool needCommit = false;
1369   if(d_rwtxn && d_transactiondomainid==domain_id) {
1370     txn = d_rwtxn;
1371     //    cout<<"Reusing open transaction"<<endl;
1372   }
1373   else {
1374     //    cout<<"Making a new RW txn for " << __PRETTY_FUNCTION__ <<endl;
1375     txn = getRecordsRWTransaction(domain_id);
1376     needCommit = true;
1377   }
1378
1379   DomainInfo di;
1380   if(!d_tdomains->getROTransaction().get(domain_id, di)) {
1381     //    cout<<"Could not find domain_id "<<domain_id <<endl;
1382     return false;
1383   }
1384
1385   DNSName rel = qname.makeRelative(di.zone);
1386
1387   compoundOrdername co;
1388   string matchkey = co(domain_id, rel);
1389
1390   auto cursor = txn->txn.getCursor(txn->db->dbi);
1391   MDBOutVal key, val;
1392   if(cursor.lower_bound(matchkey, key, val)) {
1393     // cout << "Could not find anything"<<endl;
1394     return false;
1395   }
1396
1397   bool hasOrderName = !ordername.empty();
1398   bool needNSEC3 = hasOrderName;
1399
1400   for(; key.get<StringView>().rfind(matchkey,0) == 0; ) {
1401     DNSResourceRecord rr;
1402     rr.qtype = co.getQType(key.get<StringView>());
1403
1404     if(rr.qtype != QType::NSEC3) {
1405       serFromString(val.get<StringView>(), rr);
1406       if(!needNSEC3 && qtype != QType::ANY) {
1407         needNSEC3 = (rr.disabled && QType(qtype) != rr.qtype);
1408       }
1409
1410       if((qtype == QType::ANY || QType(qtype) == rr.qtype) && (rr.disabled != hasOrderName || rr.auth != auth)) {
1411         rr.auth = auth;
1412         rr.disabled = hasOrderName;
1413         string repl = serToString(rr);
1414         cursor.put(key, repl);
1415       }
1416     }
1417
1418     if(cursor.next(key, val))
1419       break;
1420   }
1421
1422   bool del = false;
1423   DNSResourceRecord rr;
1424   matchkey = co(domain_id,rel,QType::NSEC3);
1425   if(!txn->txn.get(txn->db->dbi, matchkey, val)) {
1426     serFromString(val.get<string_view>(), rr);
1427
1428     if(needNSEC3) {
1429       if(hasOrderName && rr.content != ordername.toDNSStringLC()) {
1430           del = true;
1431       }
1432     } else {
1433       del = true;
1434     }
1435     if(del) {
1436       txn->txn.del(txn->db->dbi, co(domain_id, DNSName(rr.content.c_str(), rr.content.size(), 0, false), QType::NSEC3));
1437       txn->txn.del(txn->db->dbi, matchkey);
1438     }
1439   } else {
1440     del = true;
1441   }
1442
1443   if(hasOrderName && del) {
1444     matchkey = co(domain_id,rel,QType::NSEC3);
1445
1446     rr.ttl=0;
1447     rr.auth=0;
1448     rr.content=rel.toDNSStringLC();
1449
1450     string str = serToString(rr);
1451     txn->txn.put(txn->db->dbi, co(domain_id,ordername,QType::NSEC3), str);
1452     rr.ttl = 1;
1453     rr.content = ordername.toDNSStringLC();
1454     str = serToString(rr);
1455     txn->txn.put(txn->db->dbi, matchkey, str);  // 2
1456   }
1457
1458   if(needCommit)
1459     txn->txn.commit();
1460   return false;
1461 }
1462
1463 bool LMDBBackend::updateEmptyNonTerminals(uint32_t domain_id, set<DNSName>& insert, set<DNSName>& erase, bool remove)
1464 {
1465   // cout << __PRETTY_FUNCTION__<< ": "<< domain_id << ", insert.size() "<<insert.size()<<", "<<erase.size()<<", " <<remove<<endl;
1466
1467   bool needCommit = false;
1468   shared_ptr<RecordsRWTransaction> txn;
1469   if(d_rwtxn && d_transactiondomainid == domain_id) {
1470     txn = d_rwtxn;
1471     //    cout<<"Reusing open transaction"<<endl;
1472   }
1473   else {
1474     //    cout<<"Making a new RW txn for delete domain"<<endl;
1475     txn = getRecordsRWTransaction(domain_id);
1476     needCommit = true;
1477   }
1478
1479
1480   // if remove is set, all ENTs should be removed & nothing else should be done
1481   if(remove) {
1482     deleteDomainRecords(*txn, domain_id, 0);
1483   }
1484   else {
1485     DomainInfo di;
1486     auto rotxn = d_tdomains->getROTransaction();
1487     if(!rotxn.get(domain_id, di)) {
1488       // cout <<"No such domain with id "<<domain_id<<endl;
1489       return false;
1490     }
1491     compoundOrdername co;
1492     for(const auto& n : insert) {
1493       DNSResourceRecord rr;
1494       rr.qname = n.makeRelative(di.zone);
1495       rr.ttl = 0;
1496       rr.auth = true;
1497
1498       std::string ser = serToString(rr);
1499
1500       txn->txn.put(txn->db->dbi, co(domain_id, rr.qname, 0), ser);
1501
1502       DNSResourceRecord rr2;
1503       serFromString(ser, rr2);
1504
1505       // cout <<" +"<<n<<endl;
1506     }
1507     for(auto n : erase) {
1508       // cout <<" -"<<n<<endl;
1509       n.makeUsRelative(di.zone);
1510       txn->txn.del(txn->db->dbi, co(domain_id, n, 0));
1511     }
1512   }
1513   if(needCommit)
1514     txn->txn.commit();
1515   return false;
1516 }
1517
1518 /* TSIG */
1519 bool LMDBBackend::getTSIGKey(const DNSName& name, DNSName* algorithm, string* content)
1520 {
1521   auto txn = d_ttsig->getROTransaction();
1522
1523   TSIGKey tk;
1524   if(!txn.get<0>(name, tk))
1525     return false;
1526   if(algorithm)
1527     *algorithm = tk.algorithm;
1528   if(content)
1529     *content = tk.key;
1530   return true;
1531
1532 }
1533 // this deletes an old key if it has the same algorithm
1534 bool LMDBBackend::setTSIGKey(const DNSName& name, const DNSName& algorithm, const string& content)
1535 {
1536   auto txn = d_ttsig->getRWTransaction();
1537
1538   for(auto range = txn.equal_range<0>(name); range.first != range.second; ++range.first) {
1539     if(range.first->algorithm == algorithm)
1540       range.first.del();
1541   }
1542
1543   TSIGKey tk;
1544   tk.name = name;
1545   tk.algorithm = algorithm;
1546   tk.key=content;
1547
1548   txn.put(tk);
1549   txn.commit();
1550
1551   return true;
1552 }
1553 bool LMDBBackend::deleteTSIGKey(const DNSName& name)
1554 {
1555   auto txn = d_ttsig->getRWTransaction();
1556   TSIGKey tk;
1557
1558   for(auto range = txn.equal_range<0>(name); range.first != range.second; ++range.first) {
1559     range.first.del();
1560   }
1561   txn.commit();
1562   return true;
1563 }
1564 bool LMDBBackend::getTSIGKeys(std::vector< struct TSIGKey > &keys)
1565 {
1566   auto txn = d_ttsig->getROTransaction();
1567
1568   keys.clear();
1569   for(auto iter = txn.begin(); iter != txn.end(); ++iter) {
1570     keys.push_back(*iter);
1571   }
1572   return false;
1573 }
1574
1575
1576
1577
1578 class LMDBFactory : public BackendFactory
1579 {
1580 public:
1581   LMDBFactory() : BackendFactory("lmdb") {}
1582   void declareArguments(const string &suffix="")
1583   {
1584     declare(suffix,"filename","Filename for lmdb","./pdns.lmdb");
1585     declare(suffix,"sync-mode","Synchronisation mode: nosync, nometasync, mapasync","mapasync");
1586     declare(suffix,"shards","Records database will be split into this number of shards","64");
1587   }
1588   DNSBackend *make(const string &suffix="")
1589   {
1590     return new LMDBBackend(suffix);
1591   }
1592 };
1593
1594
1595
1596
1597 /* THIRD PART */
1598
1599 class LMDBLoader
1600 {
1601 public:
1602   LMDBLoader()
1603   {
1604     BackendMakers().report(new LMDBFactory);
1605     g_log << Logger::Info << "[lmdbbackend] This is the lmdb backend version " VERSION
1606 #ifndef REPRODUCIBLE
1607       << " (" __DATE__ " " __TIME__ ")"
1608 #endif
1609       << " reporting" << endl;
1610   }
1611 };
1612
1613 static LMDBLoader randomLoader;