modules/lmdbbackend/lmdbbackend.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #ifdef HAVE_CONFIG_H
  23 #include "config.h"
  24 #endif
  25 #include "pdns/utility.hh"
  26 #include "pdns/dnsbackend.hh"
  27 #include "pdns/dns.hh"
  28 #include "pdns/dnspacket.hh"
  29 #include "pdns/base32.hh"
  30 #include "pdns/dnssecinfra.hh"
  31 #include "pdns/pdnsexception.hh"
  32 #include "pdns/logger.hh"
  33 #include "pdns/version.hh"
  34 #include "pdns/arguments.hh"
  35 #include <boost/archive/binary_oarchive.hpp>
  36 #include <boost/archive/binary_iarchive.hpp>
  37 #include <boost/serialization/vector.hpp>
  38 #include <boost/serialization/string.hpp>
  39 #include <boost/serialization/utility.hpp>
  40 // #include <boost/iostreams/stream.hpp>
  41 // #include <boost/iostreams/stream_buffer.hpp>
  42
  43 #include <boost/iostreams/device/back_inserter.hpp>
  44 // #include <sstream>
  45
  46
  47 #include "lmdbbackend.hh"
  48
  49 #define SCHEMAVERSION 1
  50
  51 LMDBBackend::LMDBBackend(const std::string& suffix)
  52 {
  53   setArgPrefix("lmdb"+suffix);
  54
  55   string syncMode = toLower(getArg("sync-mode"));
  56
  57   if(syncMode == "nosync")
  58     d_asyncFlag = MDB_NOSYNC;
  59   else if(syncMode == "nometasync")
  60     d_asyncFlag = MDB_NOMETASYNC;
  61   else if(syncMode == "mapasync")
  62     d_asyncFlag = MDB_MAPASYNC;
  63   else if(syncMode.empty() || syncMode == "sync")
  64     d_asyncFlag = 0;
  65   else
  66     throw std::runtime_error("Unknown sync mode "+syncMode+" requested for LMDB backend");
  67
  68   d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | d_asyncFlag, 0600), "domains");
  69   d_tmeta = std::make_shared<tmeta_t>(d_tdomains->getEnv(), "metadata");
  70   d_tkdb = std::make_shared<tkdb_t>(d_tdomains->getEnv(), "keydata");
  71   d_ttsig = std::make_shared<ttsig_t>(d_tdomains->getEnv(), "tsig");
  72
  73   auto pdnsdbi = d_tdomains->getEnv()->openDB("pdns", MDB_CREATE);
  74   auto txn = d_tdomains->getEnv()->getRWTransaction();
  75   MDBOutVal _schemaversion;
  76   if(!txn->get(pdnsdbi, "schemaversion", _schemaversion)) {
  77     auto schemaversion = _schemaversion.get<uint32_t>();
  78     if (schemaversion != SCHEMAVERSION) {
  79       throw std::runtime_error("Expected LMDB schema version "+std::to_string(SCHEMAVERSION)+" but got "+std::to_string(schemaversion));
  80     }
  81   }
  82   else {
  83     txn->put(pdnsdbi, "schemaversion", SCHEMAVERSION);
  84   }
  85   MDBOutVal shards;
  86   if(!txn->get(pdnsdbi, "shards", shards)) {
  87
  88     d_shards = shards.get<uint32_t>();
  89     if(d_shards != atoi(getArg("shards").c_str())) {
  90       g_log << Logger::Warning<<"Note: configured number of lmdb shards ("<<atoi(getArg("shards").c_str())<<") is different from on-disk ("<<d_shards<<"). Using on-disk shard number"<<endl;
  91     }
  92   }
  93   else {
  94     d_shards = atoi(getArg("shards").c_str());
  95     txn->put(pdnsdbi, "shards", d_shards);
  96   }
  97   txn->commit();
  98   d_trecords.resize(d_shards);
  99   d_dolog = ::arg().mustDo("query-logging");
 100 }
 101
 102
 103
 104 namespace boost {
 105 namespace serialization {
 106
 107 template<class Archive>
 108 void save(Archive & ar, const DNSName& g, const unsigned int version)
 109 {
 110   if(!g.empty()) {
 111     std::string tmp = g.toDNSStringLC(); // g++ 4.8 woes
 112     ar & tmp;
 113   }
 114   else
 115     ar & "";
 116 }
 117
 118 template<class Archive>
 119 void load(Archive & ar, DNSName& g, const unsigned int version)
 120 {
 121   string tmp;
 122   ar & tmp;
 123   if(tmp.empty())
 124     g = DNSName();
 125   else
 126     g = DNSName(tmp.c_str(), tmp.size(), 0, false);
 127 }
 128
 129 template<class Archive>
 130 void save(Archive & ar, const QType& g, const unsigned int version)
 131 {
 132   uint16_t tmp = g.getCode(); // g++ 4.8 woes
 133   ar & tmp;
 134 }
 135
 136 template<class Archive>
 137 void load(Archive & ar, QType& g, const unsigned int version)
 138 {
 139   uint16_t tmp;
 140   ar & tmp;
 141   g = QType(tmp);
 142 }
 143
 144 template<class Archive>
 145 void serialize(Archive & ar, DomainInfo& g, const unsigned int version)
 146 {
 147   ar & g.zone;
 148   ar & g.last_check;
 149   ar & g.account;
 150   ar & g.masters;
 151   ar & g.id;
 152   ar & g.notified_serial;
 153   ar & g.kind;
 154 }
 155
 156 template<class Archive>
 157 void serialize(Archive & ar, LMDBBackend::DomainMeta& g, const unsigned int version)
 158 {
 159   ar & g.domain & g.key & g.value;
 160 }
 161
 162 template<class Archive>
 163 void serialize(Archive & ar, LMDBBackend::KeyDataDB& g, const unsigned int version)
 164 {
 165   ar & g.domain & g.content & g.flags & g.active;
 166 }
 167
 168 template<class Archive>
 169 void serialize(Archive & ar, TSIGKey& g, const unsigned int version)
 170 {
 171   ar & g.name;
 172   ar & g.algorithm; // this is the ordername
 173   ar & g.key;
 174 }
 175
 176
 177
 178 } // namespace serialization
 179 } // namespace boost
 180
 181 BOOST_SERIALIZATION_SPLIT_FREE(DNSName);
 182 BOOST_SERIALIZATION_SPLIT_FREE(QType);
 183 BOOST_IS_BITWISE_SERIALIZABLE(ComboAddress);
 184
 185 template<>
 186 std::string serToString(const DNSResourceRecord& rr)
 187 {
 188   // only does content, ttl, auth
 189   std::string ret;
 190   uint16_t len = rr.content.length();
 191   ret.reserve(2+len+8);
 192
 193   ret.assign((const char*)&len, 2);
 194   ret += rr.content;
 195   ret.append((const char*)&rr.ttl, 4);
 196   ret.append(1, (char)rr.auth);
 197   ret.append(1, (char)false);
 198   ret.append(1, (char)rr.disabled);
 199   return ret;
 200 }
 201
 202 template<>
 203 void serFromString(const string_view& str, DNSResourceRecord& rr)
 204 {
 205   uint16_t len;
 206   memcpy(&len, &str[0], 2);
 207   rr.content.assign(&str[2], len);    // len bytes
 208   memcpy(&rr.ttl, &str[2] + len, 4);
 209   rr.auth = str[str.size()-3];
 210   rr.disabled = str[str.size()-1];
 211   rr.wildcardname.clear();
 212 }
 213
 214
 215 std::string serializeContent(uint16_t qtype, const DNSName& domain, const std::string& content)
 216 {
 217   auto drc = DNSRecordContent::mastermake(qtype, 1, content);
 218   return drc->serialize(domain, false);
 219 }
 220
 221 std::shared_ptr<DNSRecordContent> unserializeContentZR(uint16_t qtype, const DNSName& qname, const std::string& content)
 222 {
 223   if(qtype == QType::A && content.size() == 4) {
 224     return std::make_shared<ARecordContent>(*((uint32_t*)content.c_str()));
 225   }
 226   return DNSRecordContent::unserialize(qname, qtype, content);
 227 }
 228
 229
 230 /* design. If you ask a question without a zone id, we lookup the best
 231    zone id for you, and answer from that. This is different than other backends, but I can't see why it would not work.
 232
 233    The index we use is "zoneid,canonical relative name". This index is also used
 234    for AXFR.
 235
 236    Note - domain_id, name and type are ONLY present on the index!
 237 */
 238
 239 #if BOOST_VERSION >= 106100
 240 #define StringView string_view
 241 #else
 242 #define StringView string
 243 #endif
 244
 245 void LMDBBackend::deleteDomainRecords(RecordsRWTransaction& txn, uint32_t domain_id, uint16_t qtype)
 246 {
 247   compoundOrdername co;
 248   string match = co(domain_id);
 249
 250   auto cursor = txn.txn->getCursor(txn.db->dbi);
 251   MDBOutVal key, val;
 252   //  cout<<"Match: "<<makeHexDump(match);
 253   if(!cursor.lower_bound(match, key, val) ) {
 254     while(key.get<StringView>().rfind(match, 0) == 0) {
 255       if(qtype == QType::ANY || co.getQType(key.get<StringView>()) == qtype)
 256         cursor.del(MDB_NODUPDATA);
 257       if(cursor.next(key, val)) break;
 258     }
 259   }
 260 }
 261
 262 /* Here's the complicated story. Other backends have just one transaction, which is either
 263    on or not.
 264
 265    You can't call feedRecord without a transaction started with startTransaction.
 266
 267    However, other functions can be called after startTransaction() or without startTransaction()
 268      (like updateDNSSECOrderNameAndAuth)
 269
 270
 271
 272 */
 273
 274 bool LMDBBackend::startTransaction(const DNSName &domain, int domain_id)
 275 {
 276   // cout <<"startTransaction("<<domain<<", "<<domain_id<<")"<<endl;
 277   int real_id = domain_id;
 278   if(real_id < 0) {
 279     auto rotxn = d_tdomains->getROTransaction();
 280     DomainInfo di;
 281     real_id = rotxn.get<0>(domain, di);
 282     // cout<<"real_id = "<<real_id << endl;
 283     if(!real_id)
 284       return false;
 285   }
 286   if(d_rwtxn) {
 287     throw DBException("Attempt to start a transaction while one was open already");
 288   }
 289   d_rwtxn = getRecordsRWTransaction(real_id);
 290
 291   d_transactiondomain = domain;
 292   d_transactiondomainid = real_id;
 293   if(domain_id >= 0) {
 294     deleteDomainRecords(*d_rwtxn, domain_id);
 295   }
 296
 297   return true;
 298 }
 299
 300 bool LMDBBackend::commitTransaction()
 301 {
 302   // cout<<"Commit transaction" <<endl;
 303   d_rwtxn->txn->commit();
 304   d_rwtxn.reset();
 305   return true;
 306 }
 307
 308 bool LMDBBackend::abortTransaction()
 309 {
 310   // cout<<"Abort transaction"<<endl;
 311   d_rwtxn->txn->abort();
 312   d_rwtxn.reset();
 313
 314   return true;
 315 }
 316
 317 // d_rwtxn must be set here
 318 bool LMDBBackend::feedRecord(const DNSResourceRecord &r, const DNSName &ordername, bool ordernameIsNSEC3)
 319 {
 320   DNSResourceRecord rr(r);
 321   rr.qname.makeUsRelative(d_transactiondomain);
 322   rr.content = serializeContent(rr.qtype.getCode(), r.qname, rr.content);
 323   rr.disabled = false;
 324
 325   compoundOrdername co;
 326   d_rwtxn->txn->put(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, rr.qtype.getCode()), serToString(rr));
 327
 328   if(ordernameIsNSEC3 && !ordername.empty()) {
 329     MDBOutVal val;
 330     if(d_rwtxn->txn->get(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, QType::NSEC3), val)) {
 331       rr.ttl = 0;
 332       rr.content=rr.qname.toDNSStringLC();
 333       rr.auth = 0;
 334       string ser = serToString(rr);
 335       d_rwtxn->txn->put(d_rwtxn->db->dbi, co(r.domain_id, ordername, QType::NSEC3), ser);
 336
 337       rr.ttl = 1;
 338       rr.content = ordername.toDNSString();
 339       ser = serToString(rr);
 340       d_rwtxn->txn->put(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, QType::NSEC3), ser);
 341     }
 342   }
 343   return true;
 344 }
 345
 346 bool LMDBBackend::feedEnts(int domain_id, map<DNSName,bool>& nonterm)
 347 {
 348   DNSResourceRecord rr;
 349   rr.ttl = 0;
 350   compoundOrdername co;
 351   for(const auto& nt: nonterm) {
 352     rr.qname = nt.first.makeRelative(d_transactiondomain);
 353     rr.auth = nt.second;
 354     rr.disabled = true;
 355
 356     std::string ser = serToString(rr);
 357     d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, rr.qname, 0), ser);
 358   }
 359   return true;
 360 }
 361
 362 bool LMDBBackend::feedEnts3(int domain_id, const DNSName &domain, map<DNSName,bool> &nonterm, const NSEC3PARAMRecordContent& ns3prc, bool narrow)
 363 {
 364   string ser;
 365   DNSName ordername;
 366   DNSResourceRecord rr;
 367   compoundOrdername co;
 368   for(const auto& nt: nonterm) {
 369     rr.qname = nt.first.makeRelative(domain);
 370     rr.ttl = 0;
 371     rr.auth = nt.second;
 372     rr.disabled = nt.second;
 373     ser = serToString(rr);
 374     d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, rr.qname, 0), ser);
 375
 376     if(!narrow && rr.auth) {
 377       rr.content = rr.qname.toDNSString();
 378       rr.auth = false;
 379       rr.disabled = false;
 380       ser = serToString(rr);
 381
 382       ordername=DNSName(toBase32Hex(hashQNameWithSalt(ns3prc, nt.first)));
 383       d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, ordername, QType::NSEC3), ser);
 384
 385       rr.ttl = 1;
 386       rr.content = ordername.toDNSString();
 387       ser = serToString(rr);
 388       d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, rr.qname, QType::NSEC3), ser);
 389     }
 390   }
 391   return true;
 392 }
 393
 394
 395 // might be called within a transaction, might also be called alone
 396 bool LMDBBackend::replaceRRSet(uint32_t domain_id, const DNSName& qname, const QType& qt, const vector<DNSResourceRecord>& rrset)
 397 {
 398   // zonk qname/qtype within domain_id (go through qname, check domain_id && qtype)
 399   shared_ptr<RecordsRWTransaction> txn;
 400   bool needCommit = false;
 401   if(d_rwtxn && d_transactiondomainid==domain_id) {
 402     txn = d_rwtxn;
 403     //    cout<<"Reusing open transaction"<<endl;
 404   }
 405   else {
 406     //    cout<<"Making a new RW txn for replace rrset"<<endl;
 407     txn = getRecordsRWTransaction(domain_id);
 408     needCommit = true;
 409   }
 410
 411   DomainInfo di;
 412   if (!d_tdomains->getROTransaction().get(domain_id, di)) {
 413     return false;
 414   }
 415
 416   compoundOrdername co;
 417   auto cursor = txn->txn->getCursor(txn->db->dbi);
 418   MDBOutVal key, val;
 419   string match =co(domain_id, qname.makeRelative(di.zone), qt.getCode());
 420   if(!cursor.find(match, key, val)) {
 421     do {
 422       cursor.del(MDB_NODUPDATA);
 423     } while(!cursor.next(key, val) && key.get<StringView>().rfind(match, 0) == 0);
 424   }
 425
 426   for(auto rr : rrset) {
 427     rr.content = serializeContent(rr.qtype.getCode(), rr.qname, rr.content);
 428     rr.qname.makeUsRelative(di.zone);
 429     txn->txn->put(txn->db->dbi, match, serToString(rr));
 430   }
 431
 432   if(needCommit)
 433     txn->txn->commit();
 434
 435   return true;
 436 }
 437
 438 // tempting to templatize these two functions but the pain is not worth it
 439 std::shared_ptr<LMDBBackend::RecordsRWTransaction> LMDBBackend::getRecordsRWTransaction(uint32_t id)
 440 {
 441   auto& shard =d_trecords[id % d_shards];
 442   if(!shard.env) {
 443     shard.env = getMDBEnv( (getArg("filename")+"-"+std::to_string(id % d_shards)).c_str(),
 444                            MDB_NOSUBDIR | d_asyncFlag, 0600);
 445     shard.dbi = shard.env->openDB("records", MDB_CREATE | MDB_DUPSORT);
 446   }
 447   auto ret = std::make_shared<RecordsRWTransaction>(shard.env->getRWTransaction());
 448   ret->db = std::make_shared<RecordsDB>(shard);
 449
 450   return ret;
 451 }
 452
 453 std::shared_ptr<LMDBBackend::RecordsROTransaction> LMDBBackend::getRecordsROTransaction(uint32_t id, std::shared_ptr<LMDBBackend::RecordsRWTransaction> rwtxn)
 454 {
 455   auto& shard =d_trecords[id % d_shards];
 456   if(!shard.env) {
 457     if (rwtxn) {
 458       throw DBException("attempting to start nested transaction without open parent env");
 459     }
 460     shard.env = getMDBEnv( (getArg("filename")+"-"+std::to_string(id % d_shards)).c_str(),
 461                            MDB_NOSUBDIR | d_asyncFlag, 0600);
 462     shard.dbi = shard.env->openDB("records", MDB_CREATE | MDB_DUPSORT);
 463   }
 464
 465   if (rwtxn) {
 466     auto ret = std::make_shared<RecordsROTransaction>(rwtxn->txn->getROTransaction());
 467     ret->db = std::make_shared<RecordsDB>(shard);
 468     return ret;
 469   } else {
 470     auto ret = std::make_shared<RecordsROTransaction>(shard.env->getROTransaction());
 471     ret->db = std::make_shared<RecordsDB>(shard);
 472     return ret;
 473   }
 474 }
 475
 476
 477 bool LMDBBackend::deleteDomain(const DNSName &domain)
 478 {
 479   auto doms = d_tdomains->getRWTransaction();
 480
 481   DomainInfo di;
 482   auto id = doms.get<0>(domain, di);
 483   if(!id)
 484     return false;
 485
 486   shared_ptr<RecordsRWTransaction> txn;
 487   bool needCommit = false;
 488   if(d_rwtxn && d_transactiondomainid == id) {
 489     txn = d_rwtxn;
 490     //    cout<<"Reusing open transaction"<<endl;
 491   }
 492   else {
 493     //    cout<<"Making a new RW txn for delete domain"<<endl;
 494     txn = getRecordsRWTransaction(id);
 495     needCommit = true;
 496   }
 497
 498
 499   doms.del(id);
 500   compoundOrdername co;
 501   string match=co(id);
 502
 503   auto cursor = txn->txn->getCursor(txn->db->dbi);
 504   MDBOutVal key, val;
 505   if(!cursor.find(match, key, val)) {
 506     do {
 507       cursor.del(MDB_NODUPDATA);
 508     } while(!cursor.next(key, val) && key.get<StringView>().rfind(match, 0) == 0);
 509   }
 510
 511   if(needCommit)
 512     txn->txn->commit();
 513
 514   doms.commit();
 515
 516   return true;
 517 }
 518
 519 bool LMDBBackend::list(const DNSName &target, int id, bool include_disabled)
 520 {
 521   d_inlist=true;
 522   DomainInfo di;
 523   {
 524     auto dtxn = d_tdomains->getROTransaction();
 525
 526     if((di.id = dtxn.get<0>(target, di)))
 527       ; //      cout<<"Found domain "<<target<<" on domain_id "<<di.id <<", list requested "<<id<<endl;
 528     else {
 529       // cout<<"Did not find "<<target<<endl;
 530       return false;
 531     }
 532   }
 533
 534   d_rotxn = getRecordsROTransaction(di.id, d_rwtxn);
 535   compoundOrdername co;
 536   d_matchkey = co(di.id);
 537   d_getcursor = std::make_shared<MDBROCursor>(d_rotxn->txn->getCursor(d_rotxn->db->dbi));
 538   MDBOutVal key, val;
 539   d_inlist = true;
 540
 541   if(d_getcursor->lower_bound(d_matchkey, key, val) || key.get<StringView>().rfind(d_matchkey, 0) != 0) {
 542     // cout<<"Found nothing for list"<<endl;
 543     d_getcursor.reset();
 544     return true;
 545   }
 546
 547   d_lookupqname = target;
 548
 549   return true;
 550 }
 551
 552 void LMDBBackend::lookup(const QType &type, const DNSName &qdomain, int zoneId, DNSPacket *p)
 553 {
 554   if(d_dolog) {
 555     g_log << Logger::Warning << "Got lookup for "<<qdomain<<"|"<<type.getName()<<" in zone "<< zoneId<<endl;
 556     d_dtime.set();
 557   }
 558   DNSName hunt(qdomain);
 559   DomainInfo di;
 560   if(zoneId < 0) {
 561     auto rotxn = d_tdomains->getROTransaction();
 562
 563     do {
 564       zoneId = rotxn.get<0>(hunt, di);
 565     } while (!zoneId && type != QType::SOA && hunt.chopOff());
 566     if(zoneId <= 0) {
 567       //      cout << "Did not find zone for "<< qdomain<<endl;
 568       d_getcursor.reset();
 569       return;
 570     }
 571   }
 572   else {
 573     if(!d_tdomains->getROTransaction().get(zoneId, di)) {
 574       // cout<<"Could not find a zone with id "<<zoneId<<endl;
 575       d_getcursor.reset();
 576       return;
 577     }
 578     hunt = di.zone;
 579   }
 580
 581   DNSName relqname = qdomain.makeRelative(hunt);
 582   //  cout<<"get will look for "<<relqname<< " in zone "<<hunt<<" with id "<<zoneId<<endl;
 583   d_rotxn = getRecordsROTransaction(zoneId, d_rwtxn);
 584
 585   compoundOrdername co;
 586   d_getcursor = std::make_shared<MDBROCursor>(d_rotxn->txn->getCursor(d_rotxn->db->dbi));
 587   MDBOutVal key, val;
 588   if(type.getCode() == QType::ANY) {
 589     d_matchkey = co(zoneId,relqname);
 590   }
 591   else {
 592     d_matchkey= co(zoneId,relqname, type.getCode());
 593   }
 594   d_inlist=false;
 595
 596   if(d_getcursor->lower_bound(d_matchkey, key, val) || key.get<StringView>().rfind(d_matchkey, 0) != 0) {
 597     d_getcursor.reset();
 598     if(d_dolog) {
 599       g_log<<Logger::Warning<< "Query "<<((long)(void*)this)<<": "<<d_dtime.udiffNoReset()<<" usec to execute (found nothing)"<<endl;
 600     }
 601     return;
 602   }
 603
 604   if(d_dolog) {
 605     g_log<<Logger::Warning<< "Query "<<((long)(void*)this)<<": "<<d_dtime.udiffNoReset()<<" usec to execute"<<endl;
 606   }
 607
 608   d_lookuptype=type;
 609   d_lookupqname = qdomain;
 610   d_lookupdomain = hunt;
 611   d_lookupdomainid = zoneId;
 612 }
 613
 614 bool LMDBBackend::get(DNSZoneRecord& rr)
 615 {
 616   if(d_inlist)
 617     return get_list(rr);
 618   else
 619     return get_lookup(rr);
 620 }
 621
 622 bool LMDBBackend::get(DNSResourceRecord& rr)
 623 {
 624   //  cout <<"Old-school get called"<<endl;
 625   DNSZoneRecord dzr;
 626   if(d_inlist) {
 627     if(!get_list(dzr))
 628       return false;
 629   }
 630   else {
 631     if(!get_lookup(dzr))
 632       return false;
 633   }
 634   rr.qname = dzr.dr.d_name;
 635   rr.ttl = dzr.dr.d_ttl;
 636   rr.qtype =dzr.dr.d_type;
 637   rr.content = dzr.dr.d_content->getZoneRepresentation(true);
 638   rr.domain_id = dzr.domain_id;
 639   rr.auth = dzr.auth;
 640   //  cout<<"old school called for "<<rr.qname<<", "<<rr.qtype.getName()<<endl;
 641   return true;
 642 }
 643
 644 bool LMDBBackend::getSOA(const DNSName &domain, SOAData &sd)
 645 {
 646   //  cout <<"Native getSOA called"<<endl;
 647   lookup(QType(QType::SOA), domain, -1);
 648   DNSZoneRecord dzr;
 649   bool found=false;
 650   while(get(dzr)) {
 651     auto src = getRR<SOARecordContent>(dzr.dr);
 652     sd.domain_id = dzr.domain_id;
 653     sd.ttl = dzr.dr.d_ttl;
 654     sd.qname = dzr.dr.d_name;
 655
 656     sd.nameserver = src->d_mname;
 657     sd.hostmaster = src->d_rname;
 658     sd.serial = src->d_st.serial;
 659     sd.refresh = src->d_st.refresh;
 660     sd.retry = src->d_st.retry;
 661     sd.expire = src->d_st.expire;
 662     sd.default_ttl = src->d_st.minimum;
 663
 664     sd.db = this;
 665     found=true;
 666   }
 667   return found;
 668 }
 669 bool LMDBBackend::get_list(DNSZoneRecord& rr)
 670 {
 671   for(;;) {
 672     if(!d_getcursor)  {
 673       d_rotxn.reset();
 674       return false;
 675     }
 676
 677     MDBOutVal keyv, val;
 678
 679     d_getcursor->current(keyv, val);
 680     DNSResourceRecord drr;
 681     serFromString(val.get<string>(), drr);
 682
 683     auto key = keyv.get<string_view>();
 684     rr.dr.d_name = compoundOrdername::getQName(key) + d_lookupqname;
 685     rr.domain_id = compoundOrdername::getDomainID(key);
 686     rr.dr.d_type = compoundOrdername::getQType(key).getCode();
 687     rr.dr.d_ttl = drr.ttl;
 688     rr.auth = drr.auth;
 689
 690     if(rr.dr.d_type == QType::NSEC3) {
 691       //      cout << "Had a magic NSEC3, skipping it" << endl;
 692       if(d_getcursor->next(keyv, val) || keyv.get<StringView>().rfind(d_matchkey, 0) != 0) {
 693         d_getcursor.reset();
 694       }
 695       continue;
 696     }
 697     rr.dr.d_content = unserializeContentZR(rr.dr.d_type, rr.dr.d_name, drr.content);
 698
 699     if(d_getcursor->next(keyv, val) || keyv.get<StringView>().rfind(d_matchkey, 0) != 0) {
 700       d_getcursor.reset();
 701     }
 702     break;
 703   }
 704   return true;
 705 }
 706
 707
 708 bool LMDBBackend::get_lookup(DNSZoneRecord& rr)
 709 {
 710   for(;;) {
 711     if(!d_getcursor) {
 712       d_rotxn.reset();
 713       return false;
 714     }
 715     MDBOutVal keyv, val;
 716     d_getcursor->current(keyv, val);
 717     DNSResourceRecord drr;
 718     serFromString(val.get<string>(), drr);
 719
 720     auto key = keyv.get<string_view>();
 721
 722     rr.dr.d_name = compoundOrdername::getQName(key) + d_lookupdomain;
 723
 724     rr.domain_id = compoundOrdername::getDomainID(key);
 725     //  cout << "We found "<<rr.qname<< " in zone id "<<rr.domain_id <<endl;
 726     rr.dr.d_type = compoundOrdername::getQType(key).getCode();
 727     rr.dr.d_ttl = drr.ttl;
 728     if(rr.dr.d_type == QType::NSEC3) {
 729       //      cout << "Hit a magic NSEC3 skipping" << endl;
 730       if(d_getcursor->next(keyv, val) || keyv.get<StringView>().rfind(d_matchkey, 0) != 0) {
 731         d_getcursor.reset();
 732         d_rotxn.reset();
 733       }
 734       continue;
 735     }
 736
 737     rr.dr.d_content = unserializeContentZR(rr.dr.d_type, rr.dr.d_name, drr.content);
 738     rr.auth = drr.auth;
 739     if(d_getcursor->next(keyv, val) || keyv.get<StringView>().rfind(d_matchkey, 0) != 0) {
 740       d_getcursor.reset();
 741       d_rotxn.reset();
 742     }
 743     break;
 744   }
 745
 746
 747   return true;
 748 }
 749
 750
 751 bool LMDBBackend::getDomainInfo(const DNSName &domain, DomainInfo &di, bool getSerial)
 752 {
 753   auto txn = d_tdomains->getROTransaction();
 754
 755   if(!(di.id=txn.get<0>(domain, di)))
 756     return false;
 757   di.backend = this;
 758   return true;
 759 }
 760
 761
 762 int LMDBBackend::genChangeDomain(const DNSName& domain, std::function<void(DomainInfo&)> func)
 763 {
 764   auto txn = d_tdomains->getRWTransaction();
 765
 766   DomainInfo di;
 767
 768   auto id = txn.get<0>(domain, di);
 769   func(di);
 770   txn.put(di, id);
 771
 772   txn.commit();
 773   return true;
 774 }
 775
 776 int LMDBBackend::genChangeDomain(uint32_t id, std::function<void(DomainInfo&)> func)
 777 {
 778   DomainInfo di;
 779
 780   auto txn = d_tdomains->getRWTransaction();
 781
 782   if(!txn.get(id , di))
 783     return false;
 784
 785   func(di);
 786
 787   txn.put(di, id);
 788
 789   txn.commit();
 790   return true;
 791 }
 792
 793
 794 bool LMDBBackend::setKind(const DNSName &domain, const DomainInfo::DomainKind kind)
 795 {
 796   return genChangeDomain(domain, [kind](DomainInfo& di) {
 797       di.kind = kind;
 798     });
 799 }
 800
 801 bool LMDBBackend::setAccount(const DNSName &domain, const std::string& account)
 802 {
 803   return genChangeDomain(domain, [account](DomainInfo& di) {
 804       di.account = account;
 805     });
 806 }
 807
 808
 809 void LMDBBackend::setFresh(uint32_t domain_id)
 810 {
 811   genChangeDomain(domain_id, [](DomainInfo& di) {
 812       di.last_check = time(0);
 813     });
 814 }
 815
 816 void LMDBBackend::setNotified(uint32_t domain_id, uint32_t serial)
 817 {
 818   genChangeDomain(domain_id, [serial](DomainInfo& di) {
 819       di.serial = serial;
 820     });
 821 }
 822
 823
 824 bool LMDBBackend::setMaster(const DNSName &domain, const std::string& ips)
 825 {
 826   vector<ComboAddress> masters;
 827   vector<string> parts;
 828   stringtok(parts, ips, " \t;,");
 829   for(const auto& ip : parts)
 830     masters.push_back(ComboAddress(ip, 53));
 831
 832   return genChangeDomain(domain, [&masters](DomainInfo& di) {
 833       di.masters = masters;
 834     });
 835 }
 836
 837 bool LMDBBackend::createDomain(const DNSName &domain)
 838 {
 839   return createDomain(domain, "NATIVE", "", "");
 840 }
 841
 842 bool LMDBBackend::createDomain(const DNSName &domain, const string &type, const string &masters, const string &account)
 843 {
 844   DomainInfo di;
 845
 846   auto txn = d_tdomains->getRWTransaction();
 847   if(txn.get<0>(domain, di)) {
 848     throw DBException("Domain '"+domain.toLogString()+"' exists already");
 849   }
 850
 851   di.zone = domain;
 852   if(pdns_iequals(type, "master"))
 853     di.kind = DomainInfo::Master;
 854   else if(pdns_iequals(type, "slave"))
 855     di.kind = DomainInfo::Slave;
 856   else if(pdns_iequals(type, "native"))
 857     di.kind = DomainInfo::Native;
 858   else
 859     throw DBException("Unable to create domain of unknown type '"+type+"'");
 860   di.account = account;
 861
 862   txn.put(di);
 863   txn.commit();
 864
 865   return true;
 866 }
 867
 868 void LMDBBackend::getAllDomains(vector<DomainInfo> *domains, bool include_disabled)
 869 {
 870   compoundOrdername co;
 871   MDBOutVal val;
 872   domains->clear();
 873   auto txn = d_tdomains->getROTransaction();
 874   for(auto iter = txn.begin(); iter != txn.end(); ++iter) {
 875     DomainInfo di=*iter;
 876     di.id = iter.getID();
 877
 878     auto txn2 = getRecordsROTransaction(iter.getID());
 879     if(!txn2->txn->get(txn2->db->dbi, co(di.id, g_rootdnsname, QType::SOA), val)) {
 880       DNSResourceRecord rr;
 881       serFromString(val.get<string_view>(), rr);
 882
 883       if(rr.content.size() >= 5 * sizeof(uint32_t)) {
 884         uint32_t serial = *reinterpret_cast<uint32_t*>(&rr.content[rr.content.size() - (5 * sizeof(uint32_t))]);
 885         di.serial = ntohl(serial);
 886       }
 887     } else if(!include_disabled) {
 888       continue;
 889     }
 890     domains->push_back(di);
 891   }
 892 }
 893
 894 void LMDBBackend::getUnfreshSlaveInfos(vector<DomainInfo>* domains)
 895 {
 896   //  cout<<"Start of getUnfreshSlaveInfos"<<endl;
 897   domains->clear();
 898   auto txn = d_tdomains->getROTransaction();
 899
 900   time_t now = time(0);
 901   for(auto iter = txn.begin(); iter != txn.end(); ++iter) {
 902     if(iter->kind != DomainInfo::Slave)
 903       continue;
 904
 905     auto txn2 = getRecordsROTransaction(iter.getID());
 906     compoundOrdername co;
 907     MDBOutVal val;
 908     uint32_t serial = 0;
 909     if(!txn2->txn->get(txn2->db->dbi, co(iter.getID(), g_rootdnsname, QType::SOA), val)) {
 910       DNSResourceRecord rr;
 911       serFromString(val.get<string_view>(), rr);
 912       struct soatimes st;
 913
 914       memcpy(&st, &rr.content[rr.content.size()-sizeof(soatimes)], sizeof(soatimes));
 915
 916       if((time_t)(iter->last_check + ntohl(st.refresh)) >= now) { // still fresh
 917         continue; // try next domain
 918       }
 919       //      cout << di.last_check <<" + " <<sdata.refresh<<" > = " << now << "\n";
 920       serial = ntohl(st.serial);
 921     }
 922     else {
 923       //      cout << "Could not find SOA for "<<iter->zone<<" with id "<<iter.getID()<<endl;
 924       serial=0;
 925     }
 926     DomainInfo di=*iter;
 927     di.id = iter.getID();
 928     di.serial = serial;
 929
 930     domains->push_back(di);
 931   }
 932   //  cout<<"END of getUnfreshSlaveInfos"<<endl;
 933 }
 934
 935 bool LMDBBackend::getAllDomainMetadata(const DNSName& name, std::map<std::string, std::vector<std::string> >& meta)
 936 {
 937   meta.clear();
 938   auto txn = d_tmeta->getROTransaction();
 939   auto range = txn.equal_range<0>(name);
 940
 941   for(auto& iter = range.first; iter != range.second; ++iter) {
 942     meta[iter->key].push_back(iter->value);
 943   }
 944   return true;
 945 }
 946
 947 bool LMDBBackend::setDomainMetadata(const DNSName& name, const std::string& kind, const std::vector<std::string>& meta)
 948 {
 949   auto txn = d_tmeta->getRWTransaction();
 950
 951   auto range = txn.equal_range<0>(name);
 952
 953   for(auto& iter = range.first; iter != range.second; ++iter) {
 954     if(iter-> key == kind)
 955       iter.del();
 956   }
 957
 958   for(const auto& m : meta) {
 959     DomainMeta dm{name, kind, m};
 960     txn.put(dm);
 961   }
 962   txn.commit();
 963   return true;
 964
 965 }
 966
 967 bool LMDBBackend::getDomainKeys(const DNSName& name, std::vector<KeyData>& keys)
 968 {
 969   auto txn = d_tkdb->getROTransaction();
 970   auto range = txn.equal_range<0>(name);
 971   for(auto& iter = range.first; iter != range.second; ++iter) {
 972     KeyData kd{iter->content, iter.getID(), iter->flags, iter->active};
 973     keys.push_back(kd);
 974   }
 975
 976   return true;
 977 }
 978
 979 bool LMDBBackend::removeDomainKey(const DNSName& name, unsigned int id)
 980 {
 981   auto txn = d_tkdb->getRWTransaction();
 982   KeyDataDB kdb;
 983   if(txn.get(id, kdb)) {
 984     if(kdb.domain == name) {
 985       txn.del(id);
 986       txn.commit();
 987       return true;
 988     }
 989   }
 990   // cout << "??? wanted to remove domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
 991   return true;
 992 }
 993
 994 bool LMDBBackend::addDomainKey(const DNSName& name, const KeyData& key, int64_t& id)
 995 {
 996   auto txn = d_tkdb->getRWTransaction();
 997   KeyDataDB kdb{name, key.content, key.flags, key.active};
 998   id = txn.put(kdb);
 999   txn.commit();
1000
1001   return true;
1002 }
1003
1004 bool LMDBBackend::activateDomainKey(const DNSName& name, unsigned int id)
1005 {
1006   auto txn = d_tkdb->getRWTransaction();
1007   KeyDataDB kdb;
1008   if(txn.get(id, kdb)) {
1009     if(kdb.domain == name) {
1010       txn.modify(id, [](KeyDataDB& kdbarg)
1011                  {
1012                    kdbarg.active = true;
1013                  });
1014       txn.commit();
1015       return true;
1016     }
1017   }
1018
1019   // cout << "??? wanted to activate domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
1020   return true;
1021 }
1022
1023 bool LMDBBackend::deactivateDomainKey(const DNSName& name, unsigned int id)
1024 {
1025   auto txn = d_tkdb->getRWTransaction();
1026   KeyDataDB kdb;
1027   if(txn.get(id, kdb)) {
1028     if(kdb.domain == name) {
1029       txn.modify(id, [](KeyDataDB& kdbarg)
1030                  {
1031                    kdbarg.active = false;
1032                  });
1033       txn.commit();
1034       return true;
1035     }
1036   }
1037   // cout << "??? wanted to activate domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
1038   return true;
1039 }
1040
1041 bool LMDBBackend::getBeforeAndAfterNamesAbsolute(uint32_t id, const DNSName& qname, DNSName& unhashed, DNSName& before, DNSName& after)
1042 {
1043   //  cout << __PRETTY_FUNCTION__<< ": "<<id <<", "<<qname << " " << unhashed<<endl;
1044
1045   DomainInfo di;
1046   if(!d_tdomains->getROTransaction().get(id, di)) {
1047     // domain does not exist, tough luck
1048     return false;
1049   }
1050   // cout <<"Zone: "<<di.zone<<endl;
1051
1052   compoundOrdername co;
1053   auto txn = getRecordsROTransaction(id);
1054
1055   auto cursor = txn->txn->getCursor(txn->db->dbi);
1056   MDBOutVal key, val;
1057
1058   DNSResourceRecord rr;
1059
1060   string matchkey = co(id, qname, QType::NSEC3);
1061   if(cursor.lower_bound(matchkey, key, val)) {
1062     // this is beyond the end of the database
1063     // cout << "Beyond end of database!" << endl;
1064     cursor.last(key, val);
1065
1066     for(;;) {
1067       if(co.getDomainID(key.get<StringView>()) != id) {
1068         //cout<<"Last record also not part of this zone!"<<endl;
1069         // this implies something is wrong in the database, nothing we can do
1070         return false;
1071       }
1072
1073       if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1074         serFromString(val.get<StringView>(), rr);
1075         if(!rr.ttl) // the kind of NSEC3 we need
1076           break;
1077       }
1078       if(cursor.prev(key, val)) {
1079         // hit beginning of database, again means something is wrong with it
1080         return false;
1081       }
1082     }
1083     before = co.getQName(key.get<StringView>());
1084     unhashed = DNSName(rr.content.c_str(), rr.content.size(), 0, false) + di.zone;
1085
1086     // now to find after .. at the beginning of the zone
1087     if(cursor.lower_bound(co(id), key, val)) {
1088       // cout<<"hit end of zone find when we shouldn't"<<endl;
1089       return false;
1090     }
1091     for(;;) {
1092       if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1093         serFromString(val.get<StringView>(), rr);
1094         if(!rr.ttl)
1095           break;
1096       }
1097
1098       if(cursor.next(key, val) || co.getDomainID(key.get<StringView>()) != id) {
1099         // cout<<"hit end of zone or database when we shouldn't"<<endl;
1100         return false;
1101       }
1102     }
1103     after = co.getQName(key.get<StringView>());
1104     // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1105     return true;
1106   }
1107
1108   // cout<<"Ended up at "<<co.getQName(key.get<StringView>()) <<endl;
1109
1110   before = co.getQName(key.get<StringView>());
1111   if(before == qname) {
1112     // cout << "Ended up on exact right node" << endl;
1113     before = co.getQName(key.get<StringView>());
1114     // unhashed should be correct now, maybe check?
1115     if(cursor.next(key, val)) {
1116       // xxx should find first hash now
1117
1118       if(cursor.lower_bound(co(id), key, val)) {
1119         // cout<<"hit end of zone find when we shouldn't for id "<<id<< __LINE__<<endl;
1120         return false;
1121       }
1122       for(;;) {
1123         if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1124           serFromString(val.get<StringView>(), rr);
1125           if(!rr.ttl)
1126             break;
1127         }
1128
1129         if(cursor.next(key, val) || co.getDomainID(key.get<StringView>()) != id) {
1130           // cout<<"hit end of zone or database when we shouldn't" << __LINE__<<endl;
1131           return false;
1132         }
1133       }
1134       after = co.getQName(key.get<StringView>());
1135       // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1136       return true;
1137     }
1138   }
1139   else {
1140     // cout <<"Going backwards to find 'before'"<<endl;
1141     int count=0;
1142     for(;;) {
1143       if(co.getQName(key.get<StringView>()).canonCompare(qname) && co.getQType(key.get<StringView>()) == QType::NSEC3) {
1144         // cout<<"Potentially stopping traverse at "<< co.getQName(key.get<StringView>()) <<", " << (co.getQName(key.get<StringView>()).canonCompare(qname))<<endl;
1145         // cout<<"qname = "<<qname<<endl;
1146         // cout<<"here  = "<<co.getQName(key.get<StringView>())<<endl;
1147         serFromString(val.get<StringView>(), rr);
1148         if(!rr.ttl)
1149           break;
1150       }
1151
1152       if(cursor.prev(key, val) || co.getDomainID(key.get<StringView>()) != id ) {
1153         // cout <<"XXX Hit *beginning* of zone or database"<<endl;
1154         // this can happen, must deal with it
1155         // should now find the last hash of the zone
1156
1157         if(cursor.lower_bound(co(id+1), key, val)) {
1158           // cout << "Could not find the next higher zone, going to the end of the database then"<<endl;
1159           cursor.last(key, val);
1160         }
1161         else
1162           cursor.prev(key, val);
1163
1164         for(;;) {
1165           if(co.getDomainID(key.get<StringView>()) != id) {
1166             //cout<<"Last record also not part of this zone!"<<endl;
1167             // this implies something is wrong in the database, nothing we can do
1168             return false;
1169           }
1170
1171           if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1172             serFromString(val.get<StringView>(), rr);
1173             if(!rr.ttl) // the kind of NSEC3 we need
1174               break;
1175           }
1176           if(cursor.prev(key, val)) {
1177             // hit beginning of database, again means something is wrong with it
1178             return false;
1179           }
1180         }
1181         before = co.getQName(key.get<StringView>());
1182         unhashed = DNSName(rr.content.c_str(), rr.content.size(), 0, false) + di.zone;
1183         // cout <<"Should still find 'after'!"<<endl;
1184         // for 'after', we need to find the first hash of this zone
1185
1186         if(cursor.lower_bound(co(id), key, val)) {
1187           // cout<<"hit end of zone find when we shouldn't"<<endl;
1188           // means database is wrong, nothing we can do
1189           return false;
1190         }
1191         for(;;) {
1192           if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1193             serFromString(val.get<StringView>(), rr);
1194             if(!rr.ttl)
1195               break;
1196           }
1197
1198           if(cursor.next(key, val)) {
1199             // means database is wrong, nothing we can do
1200             // cout<<"hit end of zone when we shouldn't 2"<<endl;
1201             return false;
1202           }
1203         }
1204         after = co.getQName(key.get<StringView>());
1205
1206
1207         // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1208         return true;
1209       }
1210       ++count;
1211     }
1212     before = co.getQName(key.get<StringView>());
1213     unhashed = DNSName(rr.content.c_str(), rr.content.size(), 0, false) + di.zone;
1214     // cout<<"Went backwards, found "<<before<<endl;
1215     // return us to starting point
1216     while(count--)
1217       cursor.next(key, val);
1218   }
1219   //  cout<<"Now going forward"<<endl;
1220   for(int count = 0 ;;++count) {
1221     if((count && cursor.next(key, val)) || co.getDomainID(key.get<StringView>()) != id ) {
1222       // cout <<"Hit end of database or zone, finding first hash then in zone "<<id<<endl;
1223       if(cursor.lower_bound(co(id), key, val)) {
1224         // cout<<"hit end of zone find when we shouldn't"<<endl;
1225         // means database is wrong, nothing we can do
1226         return false;
1227       }
1228       for(;;) {
1229         if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1230           serFromString(val.get<StringView>(), rr);
1231           if(!rr.ttl)
1232             break;
1233         }
1234
1235         if(cursor.next(key, val)) {
1236           // means database is wrong, nothing we can do
1237           // cout<<"hit end of zone when we shouldn't 2"<<endl;
1238           return false;
1239         }
1240         // cout << "Next.. "<<endl;
1241       }
1242       after = co.getQName(key.get<StringView>());
1243
1244       // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1245       return true;
1246     }
1247
1248     // cout<<"After "<<co.getQName(key.get<StringView>()) <<endl;
1249     if(co.getQType(key.get<StringView>()) == QType::NSEC3) {
1250       serFromString(val.get<StringView>(), rr);
1251       if(!rr.ttl) {
1252         break;
1253       }
1254     }
1255   }
1256   after = co.getQName(key.get<StringView>());
1257   // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
1258   return true;
1259 }
1260
1261 bool LMDBBackend::getBeforeAndAfterNames(uint32_t id, const DNSName& zonenameU, const DNSName& qname, DNSName& before, DNSName& after)
1262 {
1263   DNSName zonename = zonenameU.makeLowerCase();
1264   //  cout << __PRETTY_FUNCTION__<< ": "<<id <<", "<<zonename << ", '"<<qname<<"'"<<endl;
1265
1266   auto txn = getRecordsROTransaction(id);
1267   compoundOrdername co;
1268   DNSName qname2 = qname.makeRelative(zonename);
1269   string matchkey=co(id,qname2);
1270   auto cursor = txn->txn->getCursor(txn->db->dbi);
1271   MDBOutVal key, val;
1272   // cout<<"Lower_bound for "<<qname2<<endl;
1273   if(cursor.lower_bound(matchkey, key, val)) {
1274     // cout << "Hit end of database, bummer"<<endl;
1275     cursor.last(key, val);
1276     if(co.getDomainID(key.get<string_view>()) == id) {
1277       before = co.getQName(key.get<string_view>()) + zonename;
1278       after = zonename;
1279     }
1280     // else
1281       // cout << "We were at end of database, but this zone is not there?!"<<endl;
1282     return true;
1283   }
1284   // cout<<"Cursor is at "<<co.getQName(key.get<string_view>()) <<", in zone id "<<co.getDomainID(key.get<string_view>())<< endl;
1285
1286   if(co.getQType(key.get<string_view>()).getCode() && co.getDomainID(key.get<string_view>()) ==id && co.getQName(key.get<string_view>()) == qname2) { // don't match ENTs
1287     // cout << "Had an exact match!"<<endl;
1288     before = qname2 + zonename;
1289     int rc;
1290     for(;;) {
1291       rc=cursor.next(key, val);
1292       if(rc) break;
1293
1294       if(co.getDomainID(key.get<string_view>()) == id && key.get<StringView>().rfind(matchkey, 0)==0)
1295         continue;
1296       DNSResourceRecord rr;
1297       serFromString(val.get<StringView>(), rr);
1298       if(co.getQType(key.get<string_view>()).getCode() && (rr.auth || co.getQType(key.get<string_view>()).getCode() == QType::NS))
1299         break;
1300     }
1301     if(rc || co.getDomainID(key.get<string_view>()) != id) {
1302       // cout << "We hit the end of the zone or database. 'after' is apex" << endl;
1303       after=zonename;
1304       return false;
1305     }
1306     after = co.getQName(key.get<string_view>()) + zonename;
1307     return true;
1308   }
1309
1310
1311   if(co.getDomainID(key.get<string_view>()) != id) {
1312     // cout << "Ended up in next zone, 'after' is zonename" <<endl;
1313     after = zonename;
1314     // cout << "Now hunting for previous" << endl;
1315     int rc;
1316     for(;;) {
1317       rc=cursor.prev(key, val);
1318       if(rc) {
1319         // cout<<"Reversed into zone, but got not found from lmdb" <<endl;
1320         return false;
1321       }
1322
1323       if(co.getDomainID(key.get<string_view>()) != id) {
1324         // cout<<"Reversed into zone, but found wrong zone id " << co.getDomainID(key.get<string_view>()) << " != "<<id<<endl;
1325         // "this can't happen"
1326         return false;
1327       }
1328       DNSResourceRecord rr;
1329       serFromString(val.get<StringView>(), rr);
1330       if(co.getQType(key.get<string_view>()).getCode() && (rr.auth || co.getQType(key.get<string_view>()).getCode() == QType::NS))
1331         break;
1332     }
1333
1334     before = co.getQName(key.get<string_view>()) + zonename;
1335     // cout<<"Found: "<< before<<endl;
1336     return true;
1337   }
1338
1339   // cout <<"We ended up after "<<qname<<", on "<<co.getQName(key.get<string_view>())<<endl;
1340
1341   int skips = 0;
1342   for(; ;) {
1343     DNSResourceRecord rr;
1344     serFromString(val.get<StringView>(), rr);
1345     if(co.getQType(key.get<string_view>()).getCode() && (rr.auth || co.getQType(key.get<string_view>()).getCode() == QType::NS)) {
1346       after = co.getQName(key.get<string_view>()) + zonename;
1347       // cout <<"Found auth ("<<rr.auth<<") or an NS record "<<after<<", type: "<<co.getQType(key.get<string_view>()).getName()<<", ttl = "<<rr.ttl<<endl;
1348       // cout << makeHexDump(val.get<string>()) << endl;
1349       break;
1350     }
1351     // cout <<"  oops, " << co.getQName(key.get<string_view>()) << " was not auth "<<rr.auth<< " type=" << rr.qtype.getName()<<" or NS, so need to skip ahead a bit more" << endl;
1352     int rc = cursor.next(key, val);
1353     if(!rc)
1354       ++skips;
1355     if(rc || co.getDomainID(key.get<string_view>()) != id ) {
1356       // cout << "  oops, hit end of database or zone. This means after is apex" <<endl;
1357       after = zonename;
1358       break;
1359     }
1360   }
1361   // go back to where we were
1362   while(skips--)
1363     cursor.prev(key,val);
1364
1365   for(;;) {
1366     int rc = cursor.prev(key, val);
1367     if(rc || co.getDomainID(key.get<string_view>()) != id) {
1368       // XX I don't think this case can happen
1369       // cout << "We hit the beginning of the zone or database.. now what" << endl;
1370       return false;
1371     }
1372     before = co.getQName(key.get<string_view>()) + zonename;
1373     DNSResourceRecord rr;
1374     serFromString(val.get<string_view>(), rr);
1375     // cout<<"And before to "<<before<<", auth = "<<rr.auth<<endl;
1376     if(co.getQType(key.get<string_view>()).getCode() && (rr.auth || co.getQType(key.get<string_view>()) == QType::NS))
1377       break;
1378     // cout << "Oops, that was wrong, go back one more"<<endl;
1379   }
1380
1381   return true;
1382
1383 }
1384
1385 bool LMDBBackend::updateDNSSECOrderNameAndAuth(uint32_t domain_id, const DNSName& qname, const DNSName& ordername, bool auth, const uint16_t qtype)
1386 {
1387   //  cout << __PRETTY_FUNCTION__<< ": "<< domain_id <<", '"<<qname <<"', '"<<ordername<<"', "<<auth<< ", " << qtype << endl;
1388   shared_ptr<RecordsRWTransaction> txn;
1389   bool needCommit = false;
1390   if(d_rwtxn && d_transactiondomainid==domain_id) {
1391     txn = d_rwtxn;
1392     //    cout<<"Reusing open transaction"<<endl;
1393   }
1394   else {
1395     //    cout<<"Making a new RW txn for " << __PRETTY_FUNCTION__ <<endl;
1396     txn = getRecordsRWTransaction(domain_id);
1397     needCommit = true;
1398   }
1399
1400   DomainInfo di;
1401   if(!d_tdomains->getROTransaction().get(domain_id, di)) {
1402     //    cout<<"Could not find domain_id "<<domain_id <<endl;
1403     return false;
1404   }
1405
1406   DNSName rel = qname.makeRelative(di.zone);
1407
1408   compoundOrdername co;
1409   string matchkey = co(domain_id, rel);
1410
1411   auto cursor = txn->txn->getCursor(txn->db->dbi);
1412   MDBOutVal key, val;
1413   if(cursor.lower_bound(matchkey, key, val)) {
1414     // cout << "Could not find anything"<<endl;
1415     return false;
1416   }
1417
1418   bool hasOrderName = !ordername.empty();
1419   bool needNSEC3 = hasOrderName;
1420
1421   for(; key.get<StringView>().rfind(matchkey,0) == 0; ) {
1422     DNSResourceRecord rr;
1423     rr.qtype = co.getQType(key.get<StringView>());
1424
1425     if(rr.qtype != QType::NSEC3) {
1426       serFromString(val.get<StringView>(), rr);
1427       if(!needNSEC3 && qtype != QType::ANY) {
1428         needNSEC3 = (rr.disabled && QType(qtype) != rr.qtype);
1429       }
1430
1431       if((qtype == QType::ANY || QType(qtype) == rr.qtype) && (rr.disabled != hasOrderName || rr.auth != auth)) {
1432         rr.auth = auth;
1433         rr.disabled = hasOrderName;
1434         string repl = serToString(rr);
1435         cursor.put(key, repl);
1436       }
1437     }
1438
1439     if(cursor.next(key, val))
1440       break;
1441   }
1442
1443   bool del = false;
1444   DNSResourceRecord rr;
1445   matchkey = co(domain_id,rel,QType::NSEC3);
1446   if(!txn->txn->get(txn->db->dbi, matchkey, val)) {
1447     serFromString(val.get<string_view>(), rr);
1448
1449     if(needNSEC3) {
1450       if(hasOrderName && rr.content != ordername.toDNSStringLC()) {
1451           del = true;
1452       }
1453     } else {
1454       del = true;
1455     }
1456     if(del) {
1457       txn->txn->del(txn->db->dbi, co(domain_id, DNSName(rr.content.c_str(), rr.content.size(), 0, false), QType::NSEC3));
1458       txn->txn->del(txn->db->dbi, matchkey);
1459     }
1460   } else {
1461     del = true;
1462   }
1463
1464   if(hasOrderName && del) {
1465     matchkey = co(domain_id,rel,QType::NSEC3);
1466
1467     rr.ttl=0;
1468     rr.auth=0;
1469     rr.content=rel.toDNSStringLC();
1470
1471     string str = serToString(rr);
1472     txn->txn->put(txn->db->dbi, co(domain_id,ordername,QType::NSEC3), str);
1473     rr.ttl = 1;
1474     rr.content = ordername.toDNSStringLC();
1475     str = serToString(rr);
1476     txn->txn->put(txn->db->dbi, matchkey, str);  // 2
1477   }
1478
1479   if(needCommit)
1480     txn->txn->commit();
1481   return false;
1482 }
1483
1484 bool LMDBBackend::updateEmptyNonTerminals(uint32_t domain_id, set<DNSName>& insert, set<DNSName>& erase, bool remove)
1485 {
1486   // cout << __PRETTY_FUNCTION__<< ": "<< domain_id << ", insert.size() "<<insert.size()<<", "<<erase.size()<<", " <<remove<<endl;
1487
1488   bool needCommit = false;
1489   shared_ptr<RecordsRWTransaction> txn;
1490   if(d_rwtxn && d_transactiondomainid == domain_id) {
1491     txn = d_rwtxn;
1492     //    cout<<"Reusing open transaction"<<endl;
1493   }
1494   else {
1495     //    cout<<"Making a new RW txn for delete domain"<<endl;
1496     txn = getRecordsRWTransaction(domain_id);
1497     needCommit = true;
1498   }
1499
1500
1501   // if remove is set, all ENTs should be removed & nothing else should be done
1502   if(remove) {
1503     deleteDomainRecords(*txn, domain_id, 0);
1504   }
1505   else {
1506     DomainInfo di;
1507     auto rotxn = d_tdomains->getROTransaction();
1508     if(!rotxn.get(domain_id, di)) {
1509       // cout <<"No such domain with id "<<domain_id<<endl;
1510       return false;
1511     }
1512     compoundOrdername co;
1513     for(const auto& n : insert) {
1514       DNSResourceRecord rr;
1515       rr.qname = n.makeRelative(di.zone);
1516       rr.ttl = 0;
1517       rr.auth = true;
1518
1519       std::string ser = serToString(rr);
1520
1521       txn->txn->put(txn->db->dbi, co(domain_id, rr.qname, 0), ser);
1522
1523       DNSResourceRecord rr2;
1524       serFromString(ser, rr2);
1525
1526       // cout <<" +"<<n<<endl;
1527     }
1528     for(auto n : erase) {
1529       // cout <<" -"<<n<<endl;
1530       n.makeUsRelative(di.zone);
1531       txn->txn->del(txn->db->dbi, co(domain_id, n, 0));
1532     }
1533   }
1534   if(needCommit)
1535     txn->txn->commit();
1536   return false;
1537 }
1538
1539 /* TSIG */
1540 bool LMDBBackend::getTSIGKey(const DNSName& name, DNSName* algorithm, string* content)
1541 {
1542   auto txn = d_ttsig->getROTransaction();
1543
1544   TSIGKey tk;
1545   if(!txn.get<0>(name, tk))
1546     return false;
1547   if(algorithm)
1548     *algorithm = tk.algorithm;
1549   if(content)
1550     *content = tk.key;
1551   return true;
1552
1553 }
1554 // this deletes an old key if it has the same algorithm
1555 bool LMDBBackend::setTSIGKey(const DNSName& name, const DNSName& algorithm, const string& content)
1556 {
1557   auto txn = d_ttsig->getRWTransaction();
1558
1559   for(auto range = txn.equal_range<0>(name); range.first != range.second; ++range.first) {
1560     if(range.first->algorithm == algorithm)
1561       range.first.del();
1562   }
1563
1564   TSIGKey tk;
1565   tk.name = name;
1566   tk.algorithm = algorithm;
1567   tk.key=content;
1568
1569   txn.put(tk);
1570   txn.commit();
1571
1572   return true;
1573 }
1574 bool LMDBBackend::deleteTSIGKey(const DNSName& name)
1575 {
1576   auto txn = d_ttsig->getRWTransaction();
1577   TSIGKey tk;
1578
1579   for(auto range = txn.equal_range<0>(name); range.first != range.second; ++range.first) {
1580     range.first.del();
1581   }
1582   txn.commit();
1583   return true;
1584 }
1585 bool LMDBBackend::getTSIGKeys(std::vector< struct TSIGKey > &keys)
1586 {
1587   auto txn = d_ttsig->getROTransaction();
1588
1589   keys.clear();
1590   for(auto iter = txn.begin(); iter != txn.end(); ++iter) {
1591     keys.push_back(*iter);
1592   }
1593   return false;
1594 }
1595
1596
1597
1598
1599 class LMDBFactory : public BackendFactory
1600 {
1601 public:
1602   LMDBFactory() : BackendFactory("lmdb") {}
1603   void declareArguments(const string &suffix="")
1604   {
1605     declare(suffix,"filename","Filename for lmdb","./pdns.lmdb");
1606     declare(suffix,"sync-mode","Synchronisation mode: nosync, nometasync, mapasync, sync","mapasync");
1607     // there just is no room for more on 32 bit
1608     declare(suffix,"shards","Records database will be split into this number of shards", (sizeof(long) == 4) ? "2" : "64");
1609   }
1610   DNSBackend *make(const string &suffix="")
1611   {
1612     return new LMDBBackend(suffix);
1613   }
1614 };
1615
1616
1617
1618
1619 /* THIRD PART */
1620
1621 class LMDBLoader
1622 {
1623 public:
1624   LMDBLoader()
1625   {
1626     BackendMakers().report(new LMDBFactory);
1627     g_log << Logger::Info << "[lmdbbackend] This is the lmdb backend version " VERSION
1628 #ifndef REPRODUCIBLE
1629       << " (" __DATE__ " " __TIME__ ")"
1630 #endif
1631       << " reporting" << endl;
1632   }
1633 };
1634
1635 static LMDBLoader randomLoader;