modules/lmdbbackend/lmdbbackend.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #ifdef HAVE_CONFIG_H
  23 #include "config.h"
  24 #endif
  25 #include "pdns/utility.hh"
  26 #include "pdns/dnsbackend.hh"
  27 #include "pdns/dns.hh"
  28 #include "pdns/dnspacket.hh"
  29 #include "pdns/base32.hh"
  30 #include "pdns/dnssecinfra.hh"
  31 #include "pdns/pdnsexception.hh"
  32 #include "pdns/logger.hh"
  33 #include "pdns/version.hh"
  34 #include "pdns/arguments.hh"
  35 #include <boost/archive/binary_oarchive.hpp>
  36 #include <boost/archive/binary_iarchive.hpp>
  37 #include <boost/serialization/vector.hpp>
  38 #include <boost/serialization/string.hpp>
  39 #include <boost/serialization/utility.hpp>
  40 // #include <boost/iostreams/stream.hpp>
  41 // #include <boost/iostreams/stream_buffer.hpp>
  42
  43 #include <boost/iostreams/device/back_inserter.hpp>
  44 // #include <sstream>
  45
  46
  47 #include "lmdbbackend.hh"
  48
  49 LMDBBackend::LMDBBackend(const std::string& suffix)
  50 {
  51   setArgPrefix("lmdb"+suffix);
  52
  53   string syncMode = toLower(getArg("sync-mode"));
  54
  55   if(syncMode == "nosync")
  56     d_asyncFlag = MDB_NOSYNC;
  57   else if(syncMode == "nometasync")
  58     d_asyncFlag = MDB_NOMETASYNC;
  59   else if(syncMode == "mapasync")
  60     d_asyncFlag = MDB_MAPASYNC;
  61   else if(syncMode.empty())
  62     d_asyncFlag = 0;
  63   else
  64     throw std::runtime_error("Unknown sync mode "+syncMode+" requested for LMDB backend");
  65
  66   d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | d_asyncFlag, 0600), "domains");
  67   d_tmeta = std::make_shared<tmeta_t>(d_tdomains->getEnv(), "metadata");
  68   d_tkdb = std::make_shared<tkdb_t>(d_tdomains->getEnv(), "keydata");
  69   d_ttsig = std::make_shared<ttsig_t>(d_tdomains->getEnv(), "tsig");
  70
  71   auto pdnsdbi = d_tdomains->getEnv()->openDB("pdns", MDB_CREATE);
  72   auto txn = d_tdomains->getEnv()->getRWTransaction();
  73   MDBOutVal shards;
  74   if(!txn.get(pdnsdbi, "shards", shards)) {
  75
  76     d_shards = shards.get<uint32_t>();
  77     if(d_shards != atoi(getArg("shards").c_str())) {
  78       g_log << Logger::Warning<<"Note: configured number of lmdb shards ("<<atoi(getArg("shards").c_str())<<") is different from on-disk ("<<d_shards<<"). Using on-disk shard number"<<endl;
  79     }
  80   }
  81   else {
  82     d_shards = atoi(getArg("shards").c_str());
  83     txn.put(pdnsdbi, "shards", d_shards);
  84     txn.commit();
  85   }
  86     d_trecords.resize(d_shards);
  87   d_dolog = ::arg().mustDo("query-logging");
  88 }
  89
  90
  91
  92 namespace boost {
  93 namespace serialization {
  94
  95 template<class Archive>
  96 void save(Archive & ar, const DNSName& g, const unsigned int version)
  97 {
  98   if(!g.empty()) {
  99     std::string tmp = g.toDNSStringLC(); // g++ 4.8 woes
 100     ar & tmp;
 101   }
 102   else
 103     ar & "";
 104 }
 105
 106 template<class Archive>
 107 void load(Archive & ar, DNSName& g, const unsigned int version)
 108 {
 109   string tmp;
 110   ar & tmp;
 111   if(tmp.empty())
 112     g = DNSName();
 113   else
 114     g = DNSName(tmp.c_str(), tmp.size(), 0, false);
 115 }
 116
 117 template<class Archive>
 118 void save(Archive & ar, const QType& g, const unsigned int version)
 119 {
 120   uint16_t tmp = g.getCode(); // g++ 4.8 woes
 121   ar & tmp;
 122 }
 123
 124 template<class Archive>
 125 void load(Archive & ar, QType& g, const unsigned int version)
 126 {
 127   uint16_t tmp;
 128   ar & tmp;
 129   g = QType(tmp);
 130 }
 131
 132 template<class Archive>
 133 void serialize(Archive & ar, DomainInfo& g, const unsigned int version)
 134 {
 135   ar & g.zone;
 136   ar & g.last_check;
 137   ar & g.account;
 138   ar & g.masters;
 139   ar & g.id;
 140   ar & g.notified_serial;
 141   ar & g.kind;
 142 }
 143
 144 template<class Archive>
 145 void serialize(Archive & ar, LMDBBackend::DomainMeta& g, const unsigned int version)
 146 {
 147   ar & g.domain & g.key & g.value;
 148 }
 149
 150 template<class Archive>
 151 void serialize(Archive & ar, LMDBBackend::KeyDataDB& g, const unsigned int version)
 152 {
 153   ar & g.domain & g.content & g.flags & g.active;
 154 }
 155
 156 template<class Archive>
 157 void serialize(Archive & ar, TSIGKey& g, const unsigned int version)
 158 {
 159   ar & g.name;
 160   ar & g.algorithm; // this is the ordername
 161   ar & g.key;
 162 }
 163
 164
 165
 166 } // namespace serialization
 167 } // namespace boost
 168
 169 BOOST_SERIALIZATION_SPLIT_FREE(DNSName);
 170 BOOST_SERIALIZATION_SPLIT_FREE(QType);
 171 BOOST_IS_BITWISE_SERIALIZABLE(ComboAddress);
 172
 173 template<>
 174 std::string serToString(const DNSResourceRecord& rr)
 175 {
 176   // only does content, ttl, auth
 177   std::string ret;
 178   uint16_t len = rr.content.length();
 179   ret.reserve(2+len+8);
 180
 181   ret.assign((const char*)&len, 2);
 182   ret += rr.content;
 183   ret.append((const char*)&rr.ttl, 4);
 184   ret.append(1, (char)rr.auth);
 185   ret.append(1, (char)false);
 186   ret.append(1, (char)rr.disabled);
 187   return ret;
 188 }
 189
 190 template<>
 191 void serFromString(const string_view& str, DNSResourceRecord& rr)
 192 {
 193   uint16_t len;
 194   memcpy(&len, &str[0], 2);
 195   rr.content.assign(&str[2], len);    // len bytes
 196   memcpy(&rr.ttl, &str[2] + len, 4);
 197   rr.auth = str[str.size()-3];
 198   rr.disabled = str[str.size()-1];
 199   rr.wildcardname.clear();
 200 }
 201
 202
 203 std::string serializeContent(uint16_t qtype, const DNSName& domain, const std::string& content)
 204 {
 205   auto drc = DNSRecordContent::mastermake(qtype, 1, content);
 206   return drc->serialize(domain, false);
 207 }
 208
 209 std::shared_ptr<DNSRecordContent> unserializeContentZR(uint16_t qtype, const DNSName& qname, const std::string& content)
 210 {
 211   if(qtype == QType::A && content.size() == 4) {
 212     return std::make_shared<ARecordContent>(*((uint32_t*)content.c_str()));
 213   }
 214   return DNSRecordContent::unserialize(qname, qtype, content);
 215 }
 216
 217
 218 /* design. If you ask a question without a zone id, we lookup the best
 219    zone id for you, and answer from that. This is different than other backends, but I can't see why it would not work.
 220
 221    The index we use is "zoneid,canonical relative name". This index is also used
 222    for AXFR.
 223
 224    Note - domain_id, name and type are ONLY present on the index!
 225 */
 226
 227 #if BOOST_VERSION >= 106100
 228 #define StringView string_view
 229 #else
 230 #define StringView string
 231 #endif
 232
 233 void LMDBBackend::deleteDomainRecords(RecordsRWTransaction& txn, uint32_t domain_id, uint16_t qtype)
 234 {
 235   compoundOrdername co;
 236   string match = co(domain_id);
 237
 238   auto cursor = txn.txn.getCursor(txn.db->dbi);
 239   MDBOutVal key, val;
 240   //  cout<<"Match: "<<makeHexDump(match);
 241   if(!cursor.lower_bound(match, key, val) ) {
 242     while(key.get<StringView>().rfind(match, 0) == 0) {
 243       if(qtype == QType::ANY || co.getQType(key.get<StringView>()) == qtype)
 244         cursor.del(MDB_NODUPDATA);
 245       if(cursor.next(key, val)) break;
 246     }
 247   }
 248 }
 249
 250 /* Here's the complicated story. Other backends have just one transaction, which is either
 251    on or not.
 252
 253    You can't call feedRecord without a transaction started with startTransaction.
 254
 255    However, other functions can be called after startTransaction() or without startTransaction()
 256      (like updateDNSSECOrderNameAndAuth)
 257
 258
 259
 260 */
 261
 262 bool LMDBBackend::startTransaction(const DNSName &domain, int domain_id)
 263 {
 264   // cout <<"startTransaction("<<domain<<", "<<domain_id<<")"<<endl;
 265   int real_id = domain_id;
 266   if(real_id < 0) {
 267     auto rotxn = d_tdomains->getROTransaction();
 268     DomainInfo di;
 269     real_id = rotxn.get<0>(domain, di);
 270     // cout<<"real_id = "<<real_id << endl;
 271     if(!real_id)
 272       return false;
 273   }
 274   if(d_rwtxn) {
 275     throw DBException("Attempt to start a transaction while one was open already");
 276   }
 277   d_rwtxn = getRecordsRWTransaction(real_id);
 278
 279   d_transactiondomain = domain;
 280   d_transactiondomainid = real_id;
 281   if(domain_id >= 0) {
 282     deleteDomainRecords(*d_rwtxn, domain_id);
 283   }
 284
 285   return true;
 286 }
 287
 288 bool LMDBBackend::commitTransaction()
 289 {
 290   // cout<<"Commit transaction" <<endl;
 291   d_rwtxn->txn.commit();
 292   d_rwtxn.reset();
 293   return true;
 294 }
 295
 296 bool LMDBBackend::abortTransaction()
 297 {
 298   // cout<<"Abort transaction"<<endl;
 299   d_rwtxn->txn.abort();
 300   d_rwtxn.reset();
 301
 302   return true;
 303 }
 304
 305 // d_rwtxn must be set here
 306 bool LMDBBackend::feedRecord(const DNSResourceRecord &r, const DNSName &ordername, bool ordernameIsNSEC3)
 307 {
 308   DNSResourceRecord rr(r);
 309   rr.qname.makeUsRelative(d_transactiondomain);
 310   rr.content = serializeContent(rr.qtype.getCode(), r.qname, rr.content);
 311   rr.disabled = false;
 312
 313   compoundOrdername co;
 314   d_rwtxn->txn.put(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, rr.qtype.getCode()), serToString(rr));
 315
 316   if(ordernameIsNSEC3 && !ordername.empty()) {
 317     MDBOutVal val;
 318     if(d_rwtxn->txn.get(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, QType::NSEC3), val)) {
 319       rr.ttl = 0;
 320       rr.content=rr.qname.toDNSStringLC();
 321       rr.auth = 0;
 322       string ser = serToString(rr);
 323       d_rwtxn->txn.put(d_rwtxn->db->dbi, co(r.domain_id, ordername, QType::NSEC3), ser);
 324
 325       rr.ttl = 1;
 326       rr.content = ordername.toDNSString();
 327       ser = serToString(rr);
 328       d_rwtxn->txn.put(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, QType::NSEC3), ser);
 329     }
 330   }
 331   return true;
 332 }
 333
 334 bool LMDBBackend::feedEnts(int domain_id, map<DNSName,bool>& nonterm)
 335 {
 336   DNSResourceRecord rr;
 337   rr.ttl = 0;
 338   compoundOrdername co;
 339   for(const auto& nt: nonterm) {
 340     rr.qname = nt.first.makeRelative(d_transactiondomain);
 341     rr.auth = nt.second;
 342     rr.disabled = true;
 343
 344     std::string ser = serToString(rr);
 345     d_rwtxn->txn.put(d_rwtxn->db->dbi, co(domain_id, rr.qname, 0), ser);
 346   }
 347   return true;
 348 }
 349
 350 bool LMDBBackend::feedEnts3(int domain_id, const DNSName &domain, map<DNSName,bool> &nonterm, const NSEC3PARAMRecordContent& ns3prc, bool narrow)
 351 {
 352   string ser;
 353   DNSName ordername;
 354   DNSResourceRecord rr;
 355   compoundOrdername co;
 356   for(const auto& nt: nonterm) {
 357     rr.qname = nt.first.makeRelative(domain);
 358     rr.ttl = 0;
 359     rr.auth = nt.second;
 360     rr.disabled = nt.second;
 361     ser = serToString(rr);
 362     d_rwtxn->txn.put(d_rwtxn->db->dbi, co(domain_id, rr.qname, 0), ser);
 363
 364     if(!narrow && rr.auth) {
 365       rr.content = rr.qname.toDNSString();
 366       rr.auth = false;
 367       rr.disabled = false;
 368       ser = serToString(rr);
 369
 370       ordername=DNSName(toBase32Hex(hashQNameWithSalt(ns3prc, nt.first)));
 371       d_rwtxn->txn.put(d_rwtxn->db->dbi, co(domain_id, ordername, QType::NSEC3), ser);
 372
 373       rr.ttl = 1;
 374       rr.content = ordername.toDNSString();
 375       ser = serToString(rr);
 376       d_rwtxn->txn.put(d_rwtxn->db->dbi, co(domain_id, rr.qname, QType::NSEC3), ser);
 377     }
 378   }
 379   return true;
 380 }
 381
 382
 383 // might be called within a transaction, might also be called alone
 384 bool LMDBBackend::replaceRRSet(uint32_t domain_id, const DNSName& qname, const QType& qt, const vector<DNSResourceRecord>& rrset)
 385 {
 386   // zonk qname/qtype within domain_id (go through qname, check domain_id && qtype)
 387   shared_ptr<RecordsRWTransaction> txn;
 388   bool needCommit = false;
 389   if(d_rwtxn && d_transactiondomainid==domain_id) {
 390     txn = d_rwtxn;
 391     //    cout<<"Reusing open transaction"<<endl;
 392   }
 393   else {
 394     //    cout<<"Making a new RW txn for replace rrset"<<endl;
 395     txn = getRecordsRWTransaction(domain_id);
 396     needCommit = true;
 397   }
 398
 399   DomainInfo di;
 400   d_tdomains->getROTransaction().get(domain_id, di); // XX error checking
 401
 402   compoundOrdername co;
 403   auto cursor = txn->txn.getCursor(txn->db->dbi);
 404   MDBOutVal key, val;
 405   string match =co(domain_id, qname.makeRelative(di.zone), qt.getCode());
 406   if(!cursor.find(match, key, val)) {
 407     do {
 408       cursor.del(MDB_NODUPDATA);
 409     } while(!cursor.next(key, val) && key.get<StringView>().rfind(match, 0) == 0);
 410   }
 411
 412   for(auto rr : rrset) {
 413     rr.content = serializeContent(rr.qtype.getCode(), rr.qname, rr.content);
 414     rr.qname.makeUsRelative(di.zone);
 415     txn->txn.put(txn->db->dbi, match, serToString(rr));
 416   }
 417
 418   if(needCommit)
 419     txn->txn.commit();
 420
 421   return true;
 422 }
 423
 424 // tempting to templatize these two functions but the pain is not worth it
 425 std::shared_ptr<LMDBBackend::RecordsRWTransaction> LMDBBackend::getRecordsRWTransaction(uint32_t id)
 426 {
 427   auto& shard =d_trecords[id % d_shards];
 428   if(!shard.env) {
 429     shard.env = getMDBEnv( (getArg("filename")+"-"+std::to_string(id % d_shards)).c_str(),
 430                            MDB_NOSUBDIR | d_asyncFlag, 0600);
 431     shard.dbi = shard.env->openDB("records", MDB_CREATE | MDB_DUPSORT);
 432   }
 433   auto ret = std::make_shared<RecordsRWTransaction>(shard.env->getRWTransaction());
 434   ret->db = std::make_shared<RecordsDB>(shard);
 435
 436   return ret;
 437 }
 438
 439 std::shared_ptr<LMDBBackend::RecordsROTransaction> LMDBBackend::getRecordsROTransaction(uint32_t id)
 440 {
 441   auto& shard =d_trecords[id % d_shards];
 442   if(!shard.env) {
 443     shard.env = getMDBEnv( (getArg("filename")+"-"+std::to_string(id % d_shards)).c_str(),
 444                            MDB_NOSUBDIR | d_asyncFlag, 0600);
 445     shard.dbi = shard.env->openDB("records", MDB_CREATE | MDB_DUPSORT);
 446   }
 447
 448   auto ret = std::make_shared<RecordsROTransaction>(shard.env->getROTransaction());
 449   ret->db = std::make_shared<RecordsDB>(shard);
 450   return ret;
 451 }
 452
 453
 454 bool LMDBBackend::deleteDomain(const DNSName &domain)
 455 {
 456   auto doms = d_tdomains->getRWTransaction();
 457
 458   DomainInfo di;
 459   auto id = doms.get<0>(domain, di);
 460   if(!id)
 461     return false;
 462
 463   shared_ptr<RecordsRWTransaction> txn;
 464   bool needCommit = false;
 465   if(d_rwtxn && d_transactiondomainid == id) {
 466     txn = d_rwtxn;
 467     //    cout<<"Reusing open transaction"<<endl;
 468   }
 469   else {
 470     //    cout<<"Making a new RW txn for delete domain"<<endl;
 471     txn = getRecordsRWTransaction(id);
 472     needCommit = true;
 473   }
 474
 475
 476   doms.del(id);
 477   compoundOrdername co;
 478   string match=co(id);
 479
 480   auto cursor = txn->txn.getCursor(txn->db->dbi);
 481   MDBOutVal key, val;
 482   if(!cursor.find(match, key, val)) {
 483     do {
 484       cursor.del(MDB_NODUPDATA);
 485     } while(!cursor.next(key, val) && key.get<StringView>().rfind(match, 0) == 0);
 486   }
 487
 488   if(needCommit)
 489     txn->txn.commit();
 490
 491   doms.commit();
 492
 493   return true;
 494 }
 495
 496 bool LMDBBackend::list(const DNSName &target, int id, bool include_disabled)
 497 {
 498   d_inlist=true;
 499   DomainInfo di;
 500   {
 501     auto dtxn = d_tdomains->getROTransaction();
 502
 503     if((di.id = dtxn.get<0>(target, di)))
 504       ; //      cout<<"Found domain "<<target<<" on domain_id "<<di.id <<", list requested "<<id<<endl;
 505     else {
 506       // cout<<"Did not find "<<target<<endl;
 507       return false;
 508     }
 509   }
 510
 511   d_rotxn = getRecordsROTransaction(di.id);
 512   compoundOrdername co;
 513   d_matchkey = co(di.id);
 514   d_getcursor = std::make_shared<MDBROCursor>(d_rotxn->txn.getCursor(d_rotxn->db->dbi));
 515   MDBOutVal key, val;
 516   d_inlist = true;
 517
 518   if(d_getcursor->lower_bound(d_matchkey, key, val) || key.get<StringView>().rfind(d_matchkey, 0) != 0) {
 519     // cout<<"Found nothing for list"<<endl;
 520     d_getcursor.reset();
 521     return true;
 522   }
 523
 524   d_lookupqname = target;
 525
 526   return true;
 527 }
 528
 529 void LMDBBackend::lookup(const QType &type, const DNSName &qdomain, DNSPacket *p, int zoneId)
 530 {
 531   if(d_dolog) {
 532     g_log << Logger::Warning << "Got lookup for "<<qdomain<<"|"<<type.getName()<<" in zone "<< zoneId<<endl;
 533     d_dtime.set();
 534   }
 535   DNSName hunt(qdomain);
 536   if(zoneId < 0) {
 537     auto rotxn = d_tdomains->getROTransaction();
 538
 539     for(;;) {
 540       DomainInfo di;
 541       if((zoneId = rotxn.get<0>(hunt, di))) {
 542         break;
 543       }
 544       if(!hunt.chopOff())
 545         break;
 546     }
 547     if(zoneId <= 0) {
 548       //      cout << "Did not find zone for "<< qdomain<<endl;
 549       d_getcursor.reset();
 550       return;
 551     }
 552   }
 553   else {
 554     DomainInfo di;
 555     if(!d_tdomains->getROTransaction().get(zoneId, di)) {
 556       // cout<<"Could not find a zone with id "<<zoneId<<endl;
 557       d_getcursor.reset();
 558       return;
 559     }
 560     hunt = di.zone;
 561   }
 562
 563   DNSName relqname = qdomain.makeRelative(hunt);
 564   //  cout<<"get will look for "<<relqname<< " in zone "<<hunt<<" with id "<<zoneId<<endl;
 565   d_rotxn = getRecordsROTransaction(zoneId);
 566
 567   compoundOrdername co;
 568   d_getcursor = std::make_shared<MDBROCursor>(d_rotxn->txn.getCursor(d_rotxn->db->dbi));
 569   MDBOutVal key, val;
 570   if(type.getCode() == QType::ANY) {
 571     d_matchkey = co(zoneId,relqname);
 572   }
 573   else {
 574     d_matchkey= co(zoneId,relqname, type.getCode());
 575   }
 576   d_inlist=false;
 577
 578   if(d_getcursor->lower_bound(d_matchkey, key, val) || key.get<StringView>().rfind(d_matchkey, 0) != 0) {
 579     d_getcursor.reset();
 580     if(d_dolog) {
 581       g_log<<Logger::Warning<< "Query "<<((long)(void*)this)<<": "<<d_dtime.udiffNoReset()<<" usec to execute (found nothing)"<<endl;
 582     }
 583     return;
 584   }
 585
 586   if(d_dolog) {
 587     g_log<<Logger::Warning<< "Query "<<((long)(void*)this)<<": "<<d_dtime.udiffNoReset()<<" usec to execute"<<endl;
 588   }
 589
 590   d_lookuptype=type;
 591   d_lookupqname = qdomain;
 592   d_lookupdomain = hunt;
 593   d_lookupdomainid = zoneId;
 594 }
 595
 596 bool LMDBBackend::get(DNSZoneRecord& rr)
 597 {
 598   if(d_inlist)
 599     return get_list(rr);
 600   else
 601     return get_lookup(rr);
 602 }
 603
 604 bool LMDBBackend::get(DNSResourceRecord& rr)
 605 {
 606   //  cout <<"Old-school get called"<<endl;
 607   DNSZoneRecord dzr;
 608   if(d_inlist) {
 609     if(!get_list(dzr))
 610       return false;
 611   }
 612   else {
 613     if(!get_lookup(dzr))
 614       return false;
 615   }
 616   rr.qname = dzr.dr.d_name;
 617   rr.ttl = dzr.dr.d_ttl;
 618   rr.qtype =dzr.dr.d_type;
 619   rr.content = dzr.dr.d_content->getZoneRepresentation();
 620   rr.domain_id = dzr.domain_id;
 621   //  cout<<"old school called for "<<rr.qname<<", "<<rr.qtype.getName()<<endl;
 622   return true;
 623 }
 624
 625 bool LMDBBackend::getSOA(const DNSName &domain, SOAData &sd)
 626 {
 627   //  cout <<"Native getSOA called"<<endl;
 628   lookup(QType(QType::SOA), domain, 0, -1);
 629   DNSZoneRecord dzr;
 630   bool found=false;
 631   while(get(dzr)) {
 632     auto src = getRR<SOARecordContent>(dzr.dr);
 633     sd.domain_id = dzr.domain_id;
 634     sd.ttl = dzr.dr.d_ttl;
 635     sd.qname = dzr.dr.d_name;
 636
 637     sd.nameserver = src->d_mname;
 638     sd.hostmaster = src->d_rname;
 639     sd.serial = src->d_st.serial;
 640     sd.refresh = src->d_st.refresh;
 641     sd.retry = src->d_st.retry;
 642     sd.expire = src->d_st.expire;
 643     sd.default_ttl = src->d_st.minimum;
 644
 645     sd.db = this;
 646     found=true;
 647   }
 648   return found;
 649 }
 650 bool LMDBBackend::get_list(DNSZoneRecord& rr)
 651 {
 652   for(;;) {
 653     if(!d_getcursor)  {
 654       d_rotxn.reset();
 655       return false;
 656     }
 657
 658     MDBOutVal keyv, val;
 659
 660     d_getcursor->current(keyv, val);
 661     DNSResourceRecord drr;
 662     serFromString(val.get<string>(), drr);
 663
 664     auto key = keyv.get<string_view>();
 665     rr.dr.d_name = compoundOrdername::getQName(key) + d_lookupqname;
 666     rr.domain_id = compoundOrdername::getDomainID(key);
 667     rr.dr.d_type = compoundOrdername::getQType(key).getCode();
 668     rr.dr.d_ttl = drr.ttl;
 669     rr.auth = drr.auth;
 670
 671     if(rr.dr.d_type == QType::NSEC3) {
 672       //      cout << "Had a magic NSEC3, skipping it" << endl;
 673       if(d_getcursor->next(keyv, val) || keyv.get<StringView>().rfind(d_matchkey, 0) != 0) {
 674         d_getcursor.reset();
 675       }
 676       continue;
 677     }
 678     rr.dr.d_content = unserializeContentZR(rr.dr.d_type, rr.dr.d_name, drr.content);
 679
 680     if(d_getcursor->next(keyv, val) || keyv.get<StringView>().rfind(d_matchkey, 0) != 0) {
 681       d_getcursor.reset();
 682     }
 683     break;
 684   }
 685   return true;
 686 }
 687
 688
 689 bool LMDBBackend::get_lookup(DNSZoneRecord& rr)
 690 {
 691   for(;;) {
 692     if(!d_getcursor) {
 693       d_rotxn.reset();
 694       return false;
 695     }
 696     MDBOutVal keyv, val;
 697     d_getcursor->current(keyv, val);
 698     DNSResourceRecord drr;
 699     serFromString(val.get<string>(), drr);
 700
 701     auto key = keyv.get<string_view>();
 702
 703     rr.dr.d_name = compoundOrdername::getQName(key) + d_lookupdomain;
 704
 705     rr.domain_id = compoundOrdername::getDomainID(key);
 706     //  cout << "We found "<<rr.qname<< " in zone id "<<rr.domain_id <<endl;
 707     rr.dr.d_type = compoundOrdername::getQType(key).getCode();
 708     rr.dr.d_ttl = drr.ttl;
 709     if(rr.dr.d_type == QType::NSEC3) {
 710       //      cout << "Hit a magic NSEC3 skipping" << endl;
 711       if(d_getcursor->next(keyv, val) || keyv.get<StringView>().rfind(d_matchkey, 0) != 0) {
 712         d_getcursor.reset();
 713         d_rotxn.reset();
 714       }
 715       continue;
 716     }
 717
 718     rr.dr.d_content = unserializeContentZR(rr.dr.d_type, rr.dr.d_name, drr.content);
 719     rr.auth = drr.auth;
 720     if(d_getcursor->next(keyv, val) || keyv.get<StringView>().rfind(d_matchkey, 0) != 0) {
 721       d_getcursor.reset();
 722       d_rotxn.reset();
 723     }
 724     break;
 725   }
 726
 727
 728   return true;
 729 }
 730
 731
 732 bool LMDBBackend::getDomainInfo(const DNSName &domain, DomainInfo &di, bool getSerial)
 733 {
 734   auto txn = d_tdomains->getROTransaction();
 735
 736   if(!(di.id=txn.get<0>(domain, di)))
 737     return false;
 738   di.backend = this;
 739   return true;
 740 }
 741
 742
 743 int LMDBBackend::genChangeDomain(const DNSName& domain, std::function<void(DomainInfo&)> func)
 744 {
 745   auto txn = d_tdomains->getRWTransaction();
 746
 747   DomainInfo di;
 748
 749   auto id = txn.get<0>(domain, di);
 750   func(di);
 751   txn.put(di, id);
 752
 753   txn.commit();
 754   return true;
 755 }
 756
 757 int LMDBBackend::genChangeDomain(uint32_t id, std::function<void(DomainInfo&)> func)
 758 {
 759   DomainInfo di;
 760
 761   auto txn = d_tdomains->getRWTransaction();
 762
 763   if(!txn.get(id , di))
 764     return false;
 765
 766   func(di);
 767
 768   txn.put(di, id);
 769
 770   txn.commit();
 771   return true;
 772 }
 773
 774
 775 bool LMDBBackend::setKind(const DNSName &domain, const DomainInfo::DomainKind kind)
 776 {
 777   return genChangeDomain(domain, [kind](DomainInfo& di) {
 778       di.kind = kind;
 779     });
 780 }
 781
 782 bool LMDBBackend::setAccount(const DNSName &domain, const std::string& account)
 783 {
 784   return genChangeDomain(domain, [account](DomainInfo& di) {
 785       di.account = account;
 786     });
 787 }
 788
 789
 790 void LMDBBackend::setFresh(uint32_t domain_id)
 791 {
 792   genChangeDomain(domain_id, [](DomainInfo& di) {
 793       di.last_check = time(0);
 794     });
 795 }
 796
 797 void LMDBBackend::setNotified(uint32_t domain_id, uint32_t serial)
 798 {
 799   genChangeDomain(domain_id, [serial](DomainInfo& di) {
 800       di.serial = serial;
 801     });
 802 }
 803
 804
 805 bool LMDBBackend::setMaster(const DNSName &domain, const std::string& ips)
 806 {
 807   vector<ComboAddress> masters;
 808   vector<string> parts;
 809   stringtok(parts, ips, " \t;,");
 810   for(const auto& ip : parts)
 811     masters.push_back(ComboAddress(ip, 53));
 812
 813   return genChangeDomain(domain, [&masters](DomainInfo& di) {
 814       di.masters = masters;
 815     });
 816 }
 817
 818 bool LMDBBackend::createDomain(const DNSName &domain)
 819 {
 820   return createDomain(domain, "NATIVE", "", "");
 821 }
 822
 823 bool LMDBBackend::createDomain(const DNSName &domain, const string &type, const string &masters, const string &account)
 824 {
 825   DomainInfo di;
 826
 827   auto txn = d_tdomains->getRWTransaction();
 828   if(txn.get<0>(domain, di)) {
 829     throw DBException("Domain '"+domain.toLogString()+"' exists already");
 830   }
 831
 832   di.zone = domain;
 833   if(pdns_iequals(type, "master"))
 834     di.kind = DomainInfo::Master;
 835   else if(pdns_iequals(type, "slave"))
 836     di.kind = DomainInfo::Slave;
 837   else if(pdns_iequals(type, "native"))
 838     di.kind = DomainInfo::Native;
 839   else
 840     throw DBException("Unable to create domain of unknown type '"+type+"'");
 841   di.account = account;
 842
 843   txn.put(di);
 844   txn.commit();
 845
 846   return true;
 847 }
 848
 849 void LMDBBackend::getAllDomains(vector<DomainInfo> *domains, bool include_disabled)
 850 {
 851   compoundOrdername co;
 852   MDBOutVal val;
 853   domains->clear();
 854   auto txn = d_tdomains->getROTransaction();
 855   for(auto iter = txn.begin(); iter != txn.end(); ++iter) {
 856     DomainInfo di=*iter;
 857     di.id = iter.getID();
 858
 859     auto txn = getRecordsROTransaction(iter.getID());
 860     if(!txn->txn.get(txn->db->dbi, co(di.id, g_rootdnsname, QType::SOA), val)) {
 861       DNSResourceRecord rr;
 862       serFromString(val.get<string_view>(), rr);
 863
 864       if(rr.content.size() >= 5 * sizeof(uint32_t)) {
 865         uint32_t serial = *reinterpret_cast<uint32_t*>(&rr.content[rr.content.size() - (5 * sizeof(uint32_t))]);
 866         di.serial = ntohl(serial);
 867       }
 868     } else if(!include_disabled) {
 869       continue;
 870     }
 871     domains->push_back(di);
 872   }
 873 }
 874
 875 void LMDBBackend::getUnfreshSlaveInfos(vector<DomainInfo>* domains)
 876 {
 877   //  cout<<"Start of getUnfreshSlaveInfos"<<endl;
 878   domains->clear();
 879   auto txn = d_tdomains->getROTransaction();
 880
 881   time_t now = time(0);
 882   for(auto iter = txn.begin(); iter != txn.end(); ++iter) {
 883     if(iter->kind != DomainInfo::Slave)
 884       continue;
 885
 886     auto txn2 = getRecordsROTransaction(iter.getID());
 887     compoundOrdername co;
 888     MDBOutVal val;
 889     uint32_t serial = 0;
 890     if(!txn2->txn.get(txn2->db->dbi, co(iter.getID(), g_rootdnsname, QType::SOA), val)) {
 891       DNSResourceRecord rr;
 892       serFromString(val.get<string_view>(), rr);
 893       struct soatimes
 894       {
 895         uint32_t serial;
 896         uint32_t refresh;
 897         uint32_t retry;
 898         uint32_t expire;
 899         uint32_t minimum;
 900       } st;
 901
 902       memcpy(&st, &rr.content[rr.content.size()-sizeof(soatimes)], sizeof(soatimes));
 903
 904       if((time_t)(iter->last_check + ntohl(st.refresh)) >= now) { // still fresh
 905         continue; // try next domain
 906       }
 907       //      cout << di.last_check <<" + " <<sdata.refresh<<" > = " << now << "\n";
 908       serial = ntohl(st.serial);
 909     }
 910     else {
 911       //      cout << "Could not find SOA for "<<iter->zone<<" with id "<<iter.getID()<<endl;
 912       serial=0;
 913     }
 914     DomainInfo di=*iter;
 915     di.id = iter.getID();
 916     di.serial = serial;
 917
 918     domains->push_back(di);
 919   }
 920   //  cout<<"END of getUnfreshSlaveInfos"<<endl;
 921 }
 922
 923 bool LMDBBackend::getAllDomainMetadata(const DNSName& name, std::map<std::string, std::vector<std::string> >& meta)
 924 {
 925   meta.clear();
 926   auto txn = d_tmeta->getROTransaction();
 927   auto range = txn.equal_range<0>(name);
 928
 929   for(auto& iter = range.first; iter != range.second; ++iter) {
 930     meta[iter->key].push_back(iter->value);
 931   }
 932   return true;
 933 }
 934
 935 bool LMDBBackend::setDomainMetadata(const DNSName& name, const std::string& kind, const std::vector<std::string>& meta)
 936 {
 937   auto txn = d_tmeta->getRWTransaction();
 938
 939   auto range = txn.equal_range<0>(name);
 940
 941   for(auto& iter = range.first; iter != range.second; ++iter) {
 942     if(iter-> key == kind)
 943       iter.del();
 944   }
 945
 946   for(const auto& m : meta) {
 947     DomainMeta dm{name, kind, m};
 948     txn.put(dm);
 949   }
 950   txn.commit();
 951   return true;
 952
 953 }
 954
 955 bool LMDBBackend::getDomainKeys(const DNSName& name, std::vector<KeyData>& keys)
 956 {
 957   auto txn = d_tkdb->getROTransaction();
 958   auto range = txn.equal_range<0>(name);
 959   for(auto& iter = range.first; iter != range.second; ++iter) {
 960     KeyData kd{iter->content, iter.getID(), iter->flags, iter->active};
 961     keys.push_back(kd);
 962   }
 963
 964   return true;
 965 }
 966
 967 bool LMDBBackend::removeDomainKey(const DNSName& name, unsigned int id)
 968 {
 969   auto txn = d_tkdb->getRWTransaction();
 970   KeyDataDB kdb;
 971   if(txn.get(id, kdb)) {
 972     if(kdb.domain == name) {
 973       txn.del(id);
 974       txn.commit();
 975       return true;
 976     }
 977   }
 978   // cout << "??? wanted to remove domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
 979   return true;
 980 }
 981
 982 bool LMDBBackend::addDomainKey(const DNSName& name, const KeyData& key, int64_t& id)
 983 {
 984   auto txn = d_tkdb->getRWTransaction();
 985   KeyDataDB kdb{name, key.content, key.flags, key.active};
 986   id = txn.put(kdb);
 987   txn.commit();
 988
 989   return true;
 990 }
 991
 992 bool LMDBBackend::activateDomainKey(const DNSName& name, unsigned int id)
 993 {
 994   auto txn = d_tkdb->getRWTransaction();
 995   KeyDataDB kdb;
 996   if(txn.get(id, kdb)) {
 997     if(kdb.domain == name) {
 998       txn.modify(id, [](KeyDataDB& kdb)
 999                  {
1000                    kdb.active = true;
1001                  });
1002       txn.commit();
1003       return true;
1004     }
1005   }
1006
1007   // cout << "??? wanted to activate domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
1008   return true;
1009 }
1010
1011 bool LMDBBackend::deactivateDomainKey(const DNSName& name, unsigned int id)
1012 {
1013   auto txn = d_tkdb->getRWTransaction();
1014   KeyDataDB kdb;
1015   if(txn.get(id, kdb)) {
1016     if(kdb.domain == name) {
1017       txn.modify(id, [](KeyDataDB& kdb)
1018                  {
1019                    kdb.active = false;
1020                  });
1021       txn.commit();
1022       return true;
1023     }
1024   }
1025   // cout << "??? wanted to activate domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
1026   return true;
1027 }
1028
1029 bool LMDBBackend::getBeforeAndAfterNamesAbsolute(uint32_t id, const DNSName& qname, DNSName& unhashed, DNSName& before, DNSName& after)
1030 {
1031   //  cout << __PRETTY_FUNCTION__<< ": "<<id <<", "<<qname << " " << unhashed<<endl;
1032
1033   DomainInfo di;
1034   if(!d_tdomains->getROTransaction().get(id, di)) {
1035     // domain does not exist, tough luck
1036     return false;
1037   }
1038   // cout <<"Zone: "<<di.zone<<endl;
1039
1040   compoundOrdername co;
1041   auto txn = getRecordsROTransaction(id);
1042
1043   auto cursor = txn->txn.getCursor(txn->db->dbi);
1044   MDBOutVal key, val;
1045
1046   DNSResourceRecord rr;
1047
1048   string matchkey = co(id, qname, QType::NSEC3);
1049   if(cursor.lower_bound(matchkey, key, val)) {
1050     // this is beyond the end of the database
1051     // cout << "Beyond end of database!" << endl;
1052     cursor.last(key, val);
1053
1054     for(;;) {
1055       if(co.getDomainID(key.get<StringView>()) != id) {
1056         //cout<<"Last record also not part of this zone!"<<endl;
1057         // this implies something is wrong in the database, nothing we can do
1058         return false;
1059       }
1060
1061       if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1062         serFromString(val.get<StringView>(), rr);
1063         if(!rr.ttl) // the kind of NSEC3 we need
1064           break;
1065       }
1066       if(cursor.prev(key, val)) {
1067         // hit beginning of database, again means something is wrong with it
1068         return false;
1069       }
1070     }
1071     before = co.getQName(key.get<StringView>());
1072     unhashed = DNSName(rr.content.c_str(), rr.content.size(), 0, false) + di.zone;
1073
1074     // now to find after .. at the beginning of the zone
1075     if(cursor.lower_bound(co(id), key, val)) {
1076       // cout<<"hit end of zone find when we shouldn't"<<endl;
1077       return false;
1078     }
1079     for(;;) {
1080       if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1081         serFromString(val.get<StringView>(), rr);
1082         if(!rr.ttl)
1083           break;
1084       }
1085
1086       if(cursor.next(key, val) || co.getDomainID(key.get<StringView>()) != id) {
1087         // cout<<"hit end of zone or database when we shouldn't"<<endl;
1088         return false;
1089       }
1090     }
1091     after = co.getQName(key.get<StringView>());
1092     // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1093     return true;
1094   }
1095
1096   // cout<<"Ended up at "<<co.getQName(key.get<StringView>()) <<endl;
1097
1098   before = co.getQName(key.get<StringView>());
1099   if(before == qname) {
1100     // cout << "Ended up on exact right node" << endl;
1101     before = co.getQName(key.get<StringView>());
1102     // unhashed should be correct now, maybe check?
1103     if(cursor.next(key, val)) {
1104       // xxx should find first hash now
1105
1106       if(cursor.lower_bound(co(id), key, val)) {
1107         // cout<<"hit end of zone find when we shouldn't for id "<<id<< __LINE__<<endl;
1108         return false;
1109       }
1110       for(;;) {
1111         if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1112           serFromString(val.get<StringView>(), rr);
1113           if(!rr.ttl)
1114             break;
1115         }
1116
1117         if(cursor.next(key, val) || co.getDomainID(key.get<StringView>()) != id) {
1118           // cout<<"hit end of zone or database when we shouldn't" << __LINE__<<endl;
1119           return false;
1120         }
1121       }
1122       after = co.getQName(key.get<StringView>());
1123       // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1124       return true;
1125     }
1126   }
1127   else {
1128     // cout <<"Going backwards to find 'before'"<<endl;
1129     int count=0;
1130     for(;;) {
1131       if(co.getQName(key.get<StringView>()).canonCompare(qname) && co.getQType(key.get<StringView>()) == QType::NSEC3) {
1132         // cout<<"Potentially stopping traverse at "<< co.getQName(key.get<StringView>()) <<", " << (co.getQName(key.get<StringView>()).canonCompare(qname))<<endl;
1133         // cout<<"qname = "<<qname<<endl;
1134         // cout<<"here  = "<<co.getQName(key.get<StringView>())<<endl;
1135         serFromString(val.get<StringView>(), rr);
1136         if(!rr.ttl)
1137           break;
1138       }
1139
1140       if(cursor.prev(key, val) || co.getDomainID(key.get<StringView>()) != id ) {
1141         // cout <<"XXX Hit *beginning* of zone or database"<<endl;
1142         // this can happen, must deal with it
1143         // should now find the last hash of the zone
1144
1145         if(cursor.lower_bound(co(id+1), key, val)) {
1146           // cout << "Could not find the next higher zone, going to the end of the database then"<<endl;
1147           cursor.last(key, val);
1148         }
1149         else
1150           cursor.prev(key, val);
1151
1152         for(;;) {
1153           if(co.getDomainID(key.get<StringView>()) != id) {
1154             //cout<<"Last record also not part of this zone!"<<endl;
1155             // this implies something is wrong in the database, nothing we can do
1156             return false;
1157           }
1158
1159           if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1160             serFromString(val.get<StringView>(), rr);
1161             if(!rr.ttl) // the kind of NSEC3 we need
1162               break;
1163           }
1164           if(cursor.prev(key, val)) {
1165             // hit beginning of database, again means something is wrong with it
1166             return false;
1167           }
1168         }
1169         before = co.getQName(key.get<StringView>());
1170         unhashed = DNSName(rr.content.c_str(), rr.content.size(), 0, false) + di.zone;
1171         // cout <<"Should still find 'after'!"<<endl;
1172         // for 'after', we need to find the first hash of this zone
1173
1174         if(cursor.lower_bound(co(id), key, val)) {
1175           // cout<<"hit end of zone find when we shouldn't"<<endl;
1176           // means database is wrong, nothing we can do
1177           return false;
1178         }
1179         for(;;) {
1180           if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1181             serFromString(val.get<StringView>(), rr);
1182             if(!rr.ttl)
1183               break;
1184           }
1185
1186           if(cursor.next(key, val)) {
1187             // means database is wrong, nothing we can do
1188             // cout<<"hit end of zone when we shouldn't 2"<<endl;
1189             return false;
1190           }
1191         }
1192         after = co.getQName(key.get<StringView>());
1193
1194
1195         // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1196         return true;
1197       }
1198       ++count;
1199     }
1200     before = co.getQName(key.get<StringView>());
1201     unhashed = DNSName(rr.content.c_str(), rr.content.size(), 0, false) + di.zone;
1202     // cout<<"Went backwards, found "<<before<<endl;
1203     // return us to starting point
1204     while(count--)
1205       cursor.next(key, val);
1206   }
1207   //  cout<<"Now going forward"<<endl;
1208   for(int count = 0 ;;++count) {
1209     if((count && cursor.next(key, val)) || co.getDomainID(key.get<StringView>()) != id ) {
1210       // cout <<"Hit end of database or zone, finding first hash then in zone "<<id<<endl;
1211       if(cursor.lower_bound(co(id), key, val)) {
1212         // cout<<"hit end of zone find when we shouldn't"<<endl;
1213         // means database is wrong, nothing we can do
1214         return false;
1215       }
1216       for(;;) {
1217         if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1218           serFromString(val.get<StringView>(), rr);
1219           if(!rr.ttl)
1220             break;
1221         }
1222
1223         if(cursor.next(key, val)) {
1224           // means database is wrong, nothing we can do
1225           // cout<<"hit end of zone when we shouldn't 2"<<endl;
1226           return false;
1227         }
1228         // cout << "Next.. "<<endl;
1229       }
1230       after = co.getQName(key.get<StringView>());
1231
1232       // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1233       return true;
1234     }
1235
1236     // cout<<"After "<<co.getQName(key.get<StringView>()) <<endl;
1237     if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1238       serFromString(val.get<StringView>(), rr);
1239       if(!rr.ttl) {
1240         break;
1241       }
1242     }
1243   }
1244   after = co.getQName(key.get<StringView>());
1245   // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1246   return true;
1247 }
1248
1249 bool LMDBBackend::getBeforeAndAfterNames(uint32_t id, const DNSName& zonenameU, const DNSName& qname, DNSName& before, DNSName& after)
1250 {
1251   DNSName zonename = zonenameU.makeLowerCase();
1252   //  cout << __PRETTY_FUNCTION__<< ": "<<id <<", "<<zonename << ", '"<<qname<<"'"<<endl;
1253
1254   auto txn = getRecordsROTransaction(id);
1255   compoundOrdername co;
1256   DNSName qname2 = qname.makeRelative(zonename);
1257   string matchkey=co(id,qname2);
1258   auto cursor = txn->txn.getCursor(txn->db->dbi);
1259   MDBOutVal key, val;
1260   // cout<<"Lower_bound for "<<qname2<<endl;
1261   if(cursor.lower_bound(matchkey, key, val)) {
1262     // cout << "Hit end of database, bummer"<<endl;
1263     cursor.last(key, val);
1264     if(co.getDomainID(key.get<string_view>()) == id) {
1265       before = co.getQName(key.get<string_view>()) + zonename;
1266       after = zonename;
1267     }
1268     // else
1269       // cout << "We were at end of database, but this zone is not there?!"<<endl;
1270     return true;
1271   }
1272   // cout<<"Cursor is at "<<co.getQName(key.get<string_view>()) <<", in zone id "<<co.getDomainID(key.get<string_view>())<< endl;
1273
1274   if(co.getQType(key.get<string_view>()).getCode() && co.getDomainID(key.get<string_view>()) ==id && co.getQName(key.get<string_view>()) == qname2) { // don't match ENTs
1275     // cout << "Had an exact match!"<<endl;
1276     before = qname2 + zonename;
1277     int rc;
1278     for(;;) {
1279       rc=cursor.next(key, val);
1280       if(rc) break;
1281
1282       if(co.getDomainID(key.get<string_view>()) == id && key.get<StringView>().rfind(matchkey, 0)==0)
1283         continue;
1284       DNSResourceRecord rr;
1285       serFromString(val.get<StringView>(), rr);
1286       if(co.getQType(key.get<string_view>()).getCode() && (rr.auth || co.getQType(key.get<string_view>()).getCode() == QType::NS))
1287         break;
1288     }
1289     if(rc || co.getDomainID(key.get<string_view>()) != id) {
1290       // cout << "We hit the end of the zone or database. 'after' is apex" << endl;
1291       after=zonename;
1292       return false;
1293     }
1294     after = co.getQName(key.get<string_view>()) + zonename;
1295     return true;
1296   }
1297
1298
1299   if(co.getDomainID(key.get<string_view>()) != id) {
1300     // cout << "Ended up in next zone, 'after' is zonename" <<endl;
1301     after = zonename;
1302     // cout << "Now hunting for previous" << endl;
1303     int rc;
1304     for(;;) {
1305       rc=cursor.prev(key, val);
1306       if(rc) {
1307         // cout<<"Reversed into zone, but got not found from lmdb" <<endl;
1308         return false;
1309       }
1310
1311       if(co.getDomainID(key.get<string_view>()) != id) {
1312         // cout<<"Reversed into zone, but found wrong zone id " << co.getDomainID(key.get<string_view>()) << " != "<<id<<endl;
1313         // "this can't happen"
1314         return false;
1315       }
1316       DNSResourceRecord rr;
1317       serFromString(val.get<StringView>(), rr);
1318       if(co.getQType(key.get<string_view>()).getCode() && (rr.auth || co.getQType(key.get<string_view>()).getCode() == QType::NS))
1319         break;
1320     }
1321
1322     before = co.getQName(key.get<string_view>()) + zonename;
1323     // cout<<"Found: "<< before<<endl;
1324     return true;
1325   }
1326
1327   // cout <<"We ended up after "<<qname<<", on "<<co.getQName(key.get<string_view>())<<endl;
1328
1329   int skips = 0;
1330   for(; ;) {
1331     DNSResourceRecord rr;
1332     serFromString(val.get<StringView>(), rr);
1333     if(co.getQType(key.get<string_view>()).getCode() && (rr.auth || co.getQType(key.get<string_view>()).getCode() == QType::NS)) {
1334       after = co.getQName(key.get<string_view>()) + zonename;
1335       // cout <<"Found auth ("<<rr.auth<<") or an NS record "<<after<<", type: "<<co.getQType(key.get<string_view>()).getName()<<", ttl = "<<rr.ttl<<endl;
1336       // cout << makeHexDump(val.get<string>()) << endl;
1337       break;
1338     }
1339     // cout <<"  oops, " << co.getQName(key.get<string_view>()) << " was not auth "<<rr.auth<< " type=" << rr.qtype.getName()<<" or NS, so need to skip ahead a bit more" << endl;
1340     int rc = cursor.next(key, val);
1341     if(!rc)
1342       ++skips;
1343     if(rc || co.getDomainID(key.get<string_view>()) != id ) {
1344       // cout << "  oops, hit end of database or zone. This means after is apex" <<endl;
1345       after = zonename;
1346       break;
1347     }
1348   }
1349   // go back to where we were
1350   while(skips--)
1351     cursor.prev(key,val);
1352
1353   for(;;) {
1354     int rc = cursor.prev(key, val);
1355     if(rc || co.getDomainID(key.get<string_view>()) != id) {
1356       // XX I don't think this case can happen
1357       // cout << "We hit the beginning of the zone or database.. now what" << endl;
1358       return false;
1359     }
1360     before = co.getQName(key.get<string_view>()) + zonename;
1361     DNSResourceRecord rr;
1362     serFromString(val.get<string_view>(), rr);
1363     // cout<<"And before to "<<before<<", auth = "<<rr.auth<<endl;
1364     if(co.getQType(key.get<string_view>()).getCode() && (rr.auth || co.getQType(key.get<string_view>()) == QType::NS))
1365       break;
1366     // cout << "Oops, that was wrong, go back one more"<<endl;
1367   }
1368
1369   return true;
1370
1371 }
1372
1373 bool LMDBBackend::updateDNSSECOrderNameAndAuth(uint32_t domain_id, const DNSName& qname, const DNSName& ordername, bool auth, const uint16_t qtype)
1374 {
1375   //  cout << __PRETTY_FUNCTION__<< ": "<< domain_id <<", '"<<qname <<"', '"<<ordername<<"', "<<auth<< ", " << qtype << endl;
1376   shared_ptr<RecordsRWTransaction> txn;
1377   bool needCommit = false;
1378   if(d_rwtxn && d_transactiondomainid==domain_id) {
1379     txn = d_rwtxn;
1380     //    cout<<"Reusing open transaction"<<endl;
1381   }
1382   else {
1383     //    cout<<"Making a new RW txn for " << __PRETTY_FUNCTION__ <<endl;
1384     txn = getRecordsRWTransaction(domain_id);
1385     needCommit = true;
1386   }
1387
1388   DomainInfo di;
1389   if(!d_tdomains->getROTransaction().get(domain_id, di)) {
1390     //    cout<<"Could not find domain_id "<<domain_id <<endl;
1391     return false;
1392   }
1393
1394   DNSName rel = qname.makeRelative(di.zone);
1395
1396   compoundOrdername co;
1397   string matchkey = co(domain_id, rel);
1398
1399   auto cursor = txn->txn.getCursor(txn->db->dbi);
1400   MDBOutVal key, val;
1401   if(cursor.lower_bound(matchkey, key, val)) {
1402     // cout << "Could not find anything"<<endl;
1403     return false;
1404   }
1405
1406   bool hasOrderName = !ordername.empty();
1407   bool needNSEC3 = hasOrderName;
1408
1409   for(; key.get<StringView>().rfind(matchkey,0) == 0; ) {
1410     DNSResourceRecord rr;
1411     rr.qtype = co.getQType(key.get<StringView>());
1412
1413     if(rr.qtype != QType::NSEC3) {
1414       serFromString(val.get<StringView>(), rr);
1415       if(!needNSEC3 && qtype != QType::ANY) {
1416         needNSEC3 = (rr.disabled && QType(qtype) != rr.qtype);
1417       }
1418
1419       if((qtype == QType::ANY || QType(qtype) == rr.qtype) && (rr.disabled != hasOrderName || rr.auth != auth)) {
1420         rr.auth = auth;
1421         rr.disabled = hasOrderName;
1422         string repl = serToString(rr);
1423         cursor.put(key, repl);
1424       }
1425     }
1426
1427     if(cursor.next(key, val))
1428       break;
1429   }
1430
1431   bool del = false;
1432   DNSResourceRecord rr;
1433   matchkey = co(domain_id,rel,QType::NSEC3);
1434   if(!txn->txn.get(txn->db->dbi, matchkey, val)) {
1435     serFromString(val.get<string_view>(), rr);
1436
1437     if(needNSEC3) {
1438       if(hasOrderName && rr.content != ordername.toDNSStringLC()) {
1439           del = true;
1440       }
1441     } else {
1442       del = true;
1443     }
1444     if(del) {
1445       txn->txn.del(txn->db->dbi, co(domain_id, DNSName(rr.content.c_str(), rr.content.size(), 0, false), QType::NSEC3));
1446       txn->txn.del(txn->db->dbi, matchkey);
1447     }
1448   } else {
1449     del = true;
1450   }
1451
1452   if(hasOrderName && del) {
1453     matchkey = co(domain_id,rel,QType::NSEC3);
1454
1455     rr.ttl=0;
1456     rr.auth=0;
1457     rr.content=rel.toDNSStringLC();
1458
1459     string str = serToString(rr);
1460     txn->txn.put(txn->db->dbi, co(domain_id,ordername,QType::NSEC3), str);
1461     rr.ttl = 1;
1462     rr.content = ordername.toDNSStringLC();
1463     str = serToString(rr);
1464     txn->txn.put(txn->db->dbi, matchkey, str);  // 2
1465   }
1466
1467   if(needCommit)
1468     txn->txn.commit();
1469   return false;
1470 }
1471
1472 bool LMDBBackend::updateEmptyNonTerminals(uint32_t domain_id, set<DNSName>& insert, set<DNSName>& erase, bool remove)
1473 {
1474   // cout << __PRETTY_FUNCTION__<< ": "<< domain_id << ", insert.size() "<<insert.size()<<", "<<erase.size()<<", " <<remove<<endl;
1475
1476   bool needCommit = false;
1477   shared_ptr<RecordsRWTransaction> txn;
1478   if(d_rwtxn && d_transactiondomainid == domain_id) {
1479     txn = d_rwtxn;
1480     //    cout<<"Reusing open transaction"<<endl;
1481   }
1482   else {
1483     //    cout<<"Making a new RW txn for delete domain"<<endl;
1484     txn = getRecordsRWTransaction(domain_id);
1485     needCommit = true;
1486   }
1487
1488
1489   // if remove is set, all ENTs should be removed & nothing else should be done
1490   if(remove) {
1491     deleteDomainRecords(*txn, domain_id, 0);
1492   }
1493   else {
1494     DomainInfo di;
1495     auto rotxn = d_tdomains->getROTransaction();
1496     if(!rotxn.get(domain_id, di)) {
1497       // cout <<"No such domain with id "<<domain_id<<endl;
1498       return false;
1499     }
1500     compoundOrdername co;
1501     for(const auto& n : insert) {
1502       DNSResourceRecord rr;
1503       rr.qname = n.makeRelative(di.zone);
1504       rr.ttl = 0;
1505       rr.auth = true;
1506
1507       std::string ser = serToString(rr);
1508
1509       txn->txn.put(txn->db->dbi, co(domain_id, rr.qname, 0), ser);
1510
1511       DNSResourceRecord rr2;
1512       serFromString(ser, rr2);
1513
1514       // cout <<" +"<<n<<endl;
1515     }
1516     for(auto n : erase) {
1517       // cout <<" -"<<n<<endl;
1518       n.makeUsRelative(di.zone);
1519       txn->txn.del(txn->db->dbi, co(domain_id, n, 0));
1520     }
1521   }
1522   if(needCommit)
1523     txn->txn.commit();
1524   return false;
1525 }
1526
1527 /* TSIG */
1528 bool LMDBBackend::getTSIGKey(const DNSName& name, DNSName* algorithm, string* content)
1529 {
1530   auto txn = d_ttsig->getROTransaction();
1531
1532   TSIGKey tk;
1533   if(!txn.get<0>(name, tk))
1534     return false;
1535   if(algorithm)
1536     *algorithm = tk.algorithm;
1537   if(content)
1538     *content = tk.key;
1539   return true;
1540
1541 }
1542 // this deletes an old key if it has the same algorithm
1543 bool LMDBBackend::setTSIGKey(const DNSName& name, const DNSName& algorithm, const string& content)
1544 {
1545   auto txn = d_ttsig->getRWTransaction();
1546
1547   for(auto range = txn.equal_range<0>(name); range.first != range.second; ++range.first) {
1548     if(range.first->algorithm == algorithm)
1549       range.first.del();
1550   }
1551
1552   TSIGKey tk;
1553   tk.name = name;
1554   tk.algorithm = algorithm;
1555   tk.key=content;
1556
1557   txn.put(tk);
1558   txn.commit();
1559
1560   return true;
1561 }
1562 bool LMDBBackend::deleteTSIGKey(const DNSName& name)
1563 {
1564   auto txn = d_ttsig->getRWTransaction();
1565   TSIGKey tk;
1566
1567   for(auto range = txn.equal_range<0>(name); range.first != range.second; ++range.first) {
1568     range.first.del();
1569   }
1570   txn.commit();
1571   return true;
1572 }
1573 bool LMDBBackend::getTSIGKeys(std::vector< struct TSIGKey > &keys)
1574 {
1575   auto txn = d_ttsig->getROTransaction();
1576
1577   keys.clear();
1578   for(auto iter = txn.begin(); iter != txn.end(); ++iter) {
1579     keys.push_back(*iter);
1580   }
1581   return false;
1582 }
1583
1584
1585
1586
1587 class LMDBFactory : public BackendFactory
1588 {
1589 public:
1590   LMDBFactory() : BackendFactory("lmdb") {}
1591   void declareArguments(const string &suffix="")
1592   {
1593     declare(suffix,"filename","Filename for lmdb","./pdns.lmdb");
1594     declare(suffix,"sync-mode","Synchronisation mode: nosync, nometasync, mapasync","mapasync");
1595     declare(suffix,"shards","Records database will be split into this number of shards","64");
1596   }
1597   DNSBackend *make(const string &suffix="")
1598   {
1599     return new LMDBBackend(suffix);
1600   }
1601 };
1602
1603
1604
1605
1606 /* THIRD PART */
1607
1608 class LMDBLoader
1609 {
1610 public:
1611   LMDBLoader()
1612   {
1613     BackendMakers().report(new LMDBFactory);
1614     g_log << Logger::Info << "[lmdbbackend] This is the lmdb backend version " VERSION
1615 #ifndef REPRODUCIBLE
1616       << " (" __DATE__ " " __TIME__ ")"
1617 #endif
1618       << " reporting" << endl;
1619   }
1620 };
1621
1622 static LMDBLoader randomLoader;