modules/lmdbbackend/lmdbbackend.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22
  23 #include "ext/lmdb-safe/lmdb-safe.hh"
  24 #include <lmdb.h>
  25 #include <stdexcept>
  26 #include <utility>
  27 #ifdef HAVE_CONFIG_H
  28 #include "config.h"
  29 #endif
  30 #include "pdns/utility.hh"
  31 #include "pdns/dnsbackend.hh"
  32 #include "pdns/dns.hh"
  33 #include "pdns/dnspacket.hh"
  34 #include "pdns/base32.hh"
  35 #include "pdns/dnssecinfra.hh"
  36 #include "pdns/pdnsexception.hh"
  37 #include "pdns/logger.hh"
  38 #include "pdns/version.hh"
  39 #include "pdns/arguments.hh"
  40 #include "pdns/lock.hh"
  41 #include "pdns/uuid-utils.hh"
  42 #include <boost/archive/binary_oarchive.hpp>
  43 #include <boost/archive/binary_iarchive.hpp>
  44 #include <boost/serialization/vector.hpp>
  45 #include <boost/serialization/string.hpp>
  46 #include <boost/serialization/utility.hpp>
  47 #include <boost/uuid/uuid_serialize.hpp>
  48
  49 #include <boost/iostreams/device/back_inserter.hpp>
  50
  51 #ifdef HAVE_SYSTEMD
  52 #include <systemd/sd-daemon.h>
  53 #endif
  54
  55 #include <stdio.h>
  56 #include <unistd.h>
  57
  58 #include "lmdbbackend.hh"
  59
  60 #define SCHEMAVERSION 5
  61
  62 // List the class version here. Default is 0
  63 BOOST_CLASS_VERSION(LMDBBackend::KeyDataDB, 1)
  64 BOOST_CLASS_VERSION(DomainInfo, 1)
  65
  66 static bool s_first = true;
  67 static int s_shards = 0;
  68 static std::mutex s_lmdbStartupLock;
  69
  70 std::pair<uint32_t, uint32_t> LMDBBackend::getSchemaVersionAndShards(std::string& filename)
  71 {
  72   // cerr << "getting schema version for path " << filename << endl;
  73
  74   uint32_t schemaversion;
  75
  76   int rc;
  77   MDB_env* env = nullptr;
  78
  79   if ((rc = mdb_env_create(&env)) != 0) {
  80     throw std::runtime_error("mdb_env_create failed");
  81   }
  82
  83   if ((rc = mdb_env_set_mapsize(env, 0)) != 0) {
  84     throw std::runtime_error("mdb_env_set_mapsize failed");
  85   }
  86
  87   if ((rc = mdb_env_set_maxdbs(env, 20)) != 0) { // we need 17: 1 {"pdns"} + 4 {"domains", "keydata", "tsig", "metadata"} * 2 {v4, v5} * 2 {main, index in _0}
  88     mdb_env_close(env);
  89     throw std::runtime_error("mdb_env_set_maxdbs failed");
  90   }
  91
  92   if ((rc = mdb_env_open(env, filename.c_str(), MDB_NOSUBDIR | MDB_RDONLY, 0600)) != 0) {
  93     if (rc == ENOENT) {
  94       // we don't have a database yet! report schema 0, with 0 shards
  95       return {0u, 0u};
  96     }
  97     mdb_env_close(env);
  98     throw std::runtime_error("mdb_env_open failed");
  99   }
 100
 101   MDB_txn* txn = nullptr;
 102
 103   if ((rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)) != 0) {
 104     mdb_env_close(env);
 105     throw std::runtime_error("mdb_txn_begin failed");
 106   }
 107
 108   MDB_dbi dbi;
 109
 110   if ((rc = mdb_dbi_open(txn, "pdns", 0, &dbi)) != 0) {
 111     if (rc == MDB_NOTFOUND) {
 112       // this means nothing has been inited yet
 113       // we pretend this means 5
 114       mdb_txn_abort(txn);
 115       mdb_env_close(env);
 116       return {5u, 0u};
 117     }
 118     mdb_txn_abort(txn);
 119     mdb_env_close(env);
 120     throw std::runtime_error("mdb_dbi_open failed");
 121   }
 122
 123   MDB_val key, data;
 124
 125   key.mv_data = (char*)"schemaversion";
 126   key.mv_size = strlen((char*)key.mv_data);
 127
 128   if ((rc = mdb_get(txn, dbi, &key, &data)) != 0) {
 129     if (rc == MDB_NOTFOUND) {
 130       // this means nothing has been inited yet
 131       // we pretend this means 5
 132       mdb_txn_abort(txn);
 133       mdb_env_close(env);
 134       return {5u, 0u};
 135     }
 136
 137     throw std::runtime_error("mdb_get pdns.schemaversion failed");
 138   }
 139
 140   if (data.mv_size == 4) {
 141     // schemaversion is < 5 and is stored in 32 bits, in host order
 142
 143     memcpy(&schemaversion, data.mv_data, data.mv_size);
 144   }
 145   else if (data.mv_size >= LMDBLS::LS_MIN_HEADER_SIZE + sizeof(schemaversion)) {
 146     // schemaversion presumably is 5, stored in 32 bits, network order, after the LS header
 147
 148     // FIXME: get actual header size (including extension blocks) instead of just reading from the back
 149     // FIXME: add a test for reading schemaversion and shards (and actual data, later) when there are variably sized headers
 150     memcpy(&schemaversion, (char*)data.mv_data + data.mv_size - sizeof(schemaversion), sizeof(schemaversion));
 151     schemaversion = ntohl(schemaversion);
 152   }
 153   else {
 154     throw std::runtime_error("pdns.schemaversion had unexpected size");
 155   }
 156
 157   uint32_t shards;
 158
 159   key.mv_data = (char*)"shards";
 160   key.mv_size = strlen((char*)key.mv_data);
 161
 162   if ((rc = mdb_get(txn, dbi, &key, &data)) != 0) {
 163     if (rc == MDB_NOTFOUND) {
 164       cerr << "schemaversion was set, but shards was not. Dazed and confused, trying to exit." << endl;
 165       mdb_txn_abort(txn);
 166       mdb_env_close(env);
 167       exit(1);
 168     }
 169
 170     throw std::runtime_error("mdb_get pdns.shards failed");
 171   }
 172
 173   if (data.mv_size == 4) {
 174     // 'shards' is stored in 32 bits, in host order
 175
 176     memcpy(&shards, data.mv_data, data.mv_size);
 177   }
 178   else if (data.mv_size >= LMDBLS::LS_MIN_HEADER_SIZE + sizeof(shards)) {
 179     // FIXME: get actual header size (including extension blocks) instead of just reading from the back
 180     memcpy(&shards, (char*)data.mv_data + data.mv_size - sizeof(shards), sizeof(shards));
 181     shards = ntohl(shards);
 182   }
 183   else {
 184     throw std::runtime_error("pdns.shards had unexpected size");
 185   }
 186
 187   mdb_txn_abort(txn);
 188   mdb_env_close(env);
 189
 190   return {schemaversion, shards};
 191 }
 192
 193 namespace
 194 {
 195 // copy sdbi to tdbi, prepending an empty LS header (24 bytes of '\0') to all values
 196 void copyDBIAndAddLSHeader(MDB_txn* txn, MDB_dbi sdbi, MDB_dbi tdbi)
 197 {
 198   // FIXME: clear out target dbi first
 199
 200   std::string header(LMDBLS::LS_MIN_HEADER_SIZE, '\0');
 201   int rc;
 202
 203   MDB_cursor* cur;
 204
 205   if ((rc = mdb_cursor_open(txn, sdbi, &cur)) != 0) {
 206     throw std::runtime_error("mdb_cursur_open failed");
 207   }
 208
 209   MDB_val key, data;
 210
 211   rc = mdb_cursor_get(cur, &key, &data, MDB_FIRST);
 212
 213   while (rc == 0) {
 214     std::string skey(reinterpret_cast<const char*>(key.mv_data), key.mv_size);
 215     std::string sdata(reinterpret_cast<const char*>(data.mv_data), data.mv_size);
 216
 217     std::string stdata = header + sdata;
 218
 219     // cerr<<"got key="<<makeHexDump(skey)<<", data="<<makeHexDump(sdata)<<", sdata="<<makeHexDump(stdata)<<endl;
 220
 221     MDB_val tkey;
 222     MDB_val tdata;
 223
 224     tkey.mv_data = const_cast<char*>(skey.c_str());
 225     tkey.mv_size = skey.size();
 226     tdata.mv_data = const_cast<char*>(stdata.c_str());
 227     tdata.mv_size = stdata.size();
 228
 229     if ((rc = mdb_put(txn, tdbi, &tkey, &tdata, 0)) != 0) {
 230       throw std::runtime_error("mdb_put failed");
 231     }
 232
 233     rc = mdb_cursor_get(cur, &key, &data, MDB_NEXT);
 234   }
 235   if (rc != MDB_NOTFOUND) {
 236     cerr << "rc=" << rc << endl;
 237     throw std::runtime_error("error while iterating dbi");
 238   }
 239 }
 240
 241 // migrated a typed DBI:
 242 // 1. change keys (uint32_t) from host to network order
 243 // 2. prepend empty LS header to values
 244 void copyTypedDBI(MDB_txn* txn, MDB_dbi sdbi, MDB_dbi tdbi)
 245 {
 246   // FIXME: clear out target dbi first
 247
 248   std::string header(LMDBLS::LS_MIN_HEADER_SIZE, '\0');
 249   int rc;
 250
 251   MDB_cursor* cur;
 252
 253   if ((rc = mdb_cursor_open(txn, sdbi, &cur)) != 0) {
 254     throw std::runtime_error("mdb_cursur_open failed");
 255   }
 256
 257   MDB_val key, data;
 258
 259   rc = mdb_cursor_get(cur, &key, &data, MDB_FIRST);
 260
 261   while (rc == 0) {
 262     // std::string skey((char*) key.mv_data, key.mv_size);
 263     std::string sdata(reinterpret_cast<const char*>(data.mv_data), data.mv_size);
 264
 265     std::string stdata = header + sdata;
 266
 267     uint32_t id;
 268
 269     if (key.mv_size != sizeof(uint32_t)) {
 270       throw std::runtime_error("got non-uint32_t key in TypedDBI");
 271     }
 272
 273     memcpy(&id, key.mv_data, sizeof(uint32_t));
 274
 275     id = htonl(id);
 276
 277     // cerr<<"got key="<<makeHexDump(skey)<<", data="<<makeHexDump(sdata)<<", sdata="<<makeHexDump(stdata)<<endl;
 278
 279     MDB_val tkey;
 280     MDB_val tdata;
 281
 282     tkey.mv_data = reinterpret_cast<char*>(&id);
 283     tkey.mv_size = sizeof(uint32_t);
 284     tdata.mv_data = const_cast<char*>(stdata.c_str());
 285     tdata.mv_size = stdata.size();
 286
 287     if ((rc = mdb_put(txn, tdbi, &tkey, &tdata, 0)) != 0) {
 288       throw std::runtime_error("mdb_put failed");
 289     }
 290
 291     rc = mdb_cursor_get(cur, &key, &data, MDB_NEXT);
 292   }
 293   if (rc != MDB_NOTFOUND) {
 294     cerr << "rc=" << rc << endl;
 295     throw std::runtime_error("error while iterating dbi");
 296   }
 297 }
 298
 299 // migrating an index DBI:
 300 // newkey = oldkey.len(), oldkey, htonl(oldvalue)
 301 // newvalue = empty lsheader
 302 void copyIndexDBI(MDB_txn* txn, MDB_dbi sdbi, MDB_dbi tdbi)
 303 {
 304   // FIXME: clear out target dbi first
 305
 306   std::string header(LMDBLS::LS_MIN_HEADER_SIZE, '\0');
 307   int rc;
 308
 309   MDB_cursor* cur;
 310
 311   if ((rc = mdb_cursor_open(txn, sdbi, &cur)) != 0) {
 312     throw std::runtime_error("mdb_cursur_open failed");
 313   }
 314
 315   MDB_val key, data;
 316
 317   rc = mdb_cursor_get(cur, &key, &data, MDB_FIRST);
 318
 319   while (rc == 0) {
 320     std::string lenprefix(sizeof(uint16_t), '\0');
 321     std::string skey((char*)key.mv_data, key.mv_size);
 322
 323     uint32_t id;
 324
 325     if (data.mv_size != sizeof(uint32_t)) {
 326       throw std::runtime_error("got non-uint32_t ID value in IndexDBI");
 327     }
 328
 329     memcpy((void*)&id, data.mv_data, sizeof(uint32_t));
 330     id = htonl(id);
 331
 332     uint16_t len = htons(skey.size());
 333     memcpy((void*)lenprefix.data(), &len, sizeof(len));
 334     std::string stkey = lenprefix + skey + std::string((char*)&id, sizeof(uint32_t));
 335
 336     MDB_val tkey;
 337     MDB_val tdata;
 338
 339     tkey.mv_data = (char*)stkey.c_str();
 340     tkey.mv_size = stkey.size();
 341     tdata.mv_data = (char*)header.c_str();
 342     tdata.mv_size = header.size();
 343
 344     if ((rc = mdb_put(txn, tdbi, &tkey, &tdata, 0)) != 0) {
 345       throw std::runtime_error("mdb_put failed");
 346     }
 347
 348     rc = mdb_cursor_get(cur, &key, &data, MDB_NEXT);
 349   }
 350   if (rc != MDB_NOTFOUND) {
 351     throw std::runtime_error("error while iterating dbi");
 352   }
 353 }
 354
 355 }
 356
 357 bool LMDBBackend::upgradeToSchemav5(std::string& filename)
 358 {
 359   int rc;
 360
 361   auto currentSchemaVersionAndShards = getSchemaVersionAndShards(filename);
 362   uint32_t currentSchemaVersion = currentSchemaVersionAndShards.first;
 363   uint32_t shards = currentSchemaVersionAndShards.second;
 364
 365   if (currentSchemaVersion != 3 && currentSchemaVersion != 4) {
 366     throw std::runtime_error("upgrade to v5 requested but current schema is not v3 or v4, stopping");
 367   }
 368
 369   MDB_env* env = nullptr;
 370
 371   if ((rc = mdb_env_create(&env)) != 0) {
 372     throw std::runtime_error("mdb_env_create failed");
 373   }
 374
 375   if ((rc = mdb_env_set_maxdbs(env, 20)) != 0) {
 376     mdb_env_close(env);
 377     throw std::runtime_error("mdb_env_set_maxdbs failed");
 378   }
 379
 380   if ((rc = mdb_env_open(env, filename.c_str(), MDB_NOSUBDIR, 0600)) != 0) {
 381     mdb_env_close(env);
 382     throw std::runtime_error("mdb_env_open failed");
 383   }
 384
 385   MDB_txn* txn = nullptr;
 386
 387   if ((rc = mdb_txn_begin(env, NULL, 0, &txn)) != 0) {
 388     mdb_env_close(env);
 389     throw std::runtime_error("mdb_txn_begin failed");
 390   }
 391
 392 #ifdef HAVE_SYSTEMD
 393   /* A schema migration may take a long time. Extend the startup service timeout to 1 day,
 394    * but only if this is beyond the original maximum time of TimeoutStartSec=.
 395    */
 396   sd_notify(0, "EXTEND_TIMEOUT_USEC=86400000000");
 397 #endif
 398
 399   std::cerr << "migrating shards" << std::endl;
 400   for (uint32_t i = 0; i < shards; i++) {
 401     string shardfile = filename + "-" + std::to_string(i);
 402     if (access(shardfile.c_str(), F_OK) < 0) {
 403       if (errno == ENOENT) {
 404         // apparently this shard doesn't exist yet, moving on
 405         std::cerr << "shard " << shardfile << " not found, continuing" << std::endl;
 406         continue;
 407       }
 408     }
 409
 410     std::cerr << "migrating shard " << shardfile << std::endl;
 411     MDB_env* shenv = nullptr;
 412
 413     if ((rc = mdb_env_create(&shenv)) != 0) {
 414       throw std::runtime_error("mdb_env_create failed");
 415     }
 416
 417     if ((rc = mdb_env_set_maxdbs(shenv, 8)) != 0) {
 418       mdb_env_close(env);
 419       throw std::runtime_error("mdb_env_set_maxdbs failed");
 420     }
 421
 422     if ((rc = mdb_env_open(shenv, shardfile.c_str(), MDB_NOSUBDIR, 0600)) != 0) {
 423       mdb_env_close(env);
 424       throw std::runtime_error("mdb_env_open failed");
 425     }
 426
 427     MDB_txn* shtxn = nullptr;
 428
 429     if ((rc = mdb_txn_begin(shenv, NULL, 0, &shtxn)) != 0) {
 430       mdb_env_close(env);
 431       throw std::runtime_error("mdb_txn_begin failed");
 432     }
 433
 434     MDB_dbi shdbi;
 435
 436     if ((rc = mdb_dbi_open(shtxn, "records", 0, &shdbi)) != 0) {
 437       if (rc == MDB_NOTFOUND) {
 438         mdb_txn_abort(shtxn);
 439         mdb_env_close(shenv);
 440         continue;
 441       }
 442       mdb_txn_abort(shtxn);
 443       mdb_env_close(shenv);
 444       throw std::runtime_error("mdb_dbi_open shard records failed");
 445     }
 446
 447     MDB_dbi shdbi2;
 448
 449     if ((rc = mdb_dbi_open(shtxn, "records_v5", MDB_CREATE, &shdbi2)) != 0) {
 450       mdb_dbi_close(shenv, shdbi);
 451       mdb_txn_abort(shtxn);
 452       mdb_env_close(shenv);
 453       throw std::runtime_error("mdb_dbi_open shard records_v5 failed");
 454     }
 455
 456     try {
 457       copyDBIAndAddLSHeader(shtxn, shdbi, shdbi2);
 458     }
 459     catch (std::exception& e) {
 460       mdb_dbi_close(shenv, shdbi2);
 461       mdb_dbi_close(shenv, shdbi);
 462       mdb_txn_abort(shtxn);
 463       mdb_env_close(shenv);
 464       throw std::runtime_error("copyDBIAndAddLSHeader failed");
 465     }
 466
 467     cerr << "shard mbd_drop=" << mdb_drop(shtxn, shdbi, 1) << endl;
 468     mdb_txn_commit(shtxn);
 469     mdb_dbi_close(shenv, shdbi2);
 470     mdb_env_close(shenv);
 471   }
 472
 473   std::array<MDB_dbi, 4> fromtypeddbi;
 474   std::array<MDB_dbi, 4> totypeddbi;
 475
 476   int index = 0;
 477
 478   for (const std::string dbname : {"domains", "keydata", "tsig", "metadata"}) {
 479     std::cerr << "migrating " << dbname << std::endl;
 480     std::string tdbname = dbname + "_v5";
 481
 482     if ((rc = mdb_dbi_open(txn, dbname.c_str(), 0, &fromtypeddbi[index])) != 0) {
 483       mdb_txn_abort(txn);
 484       mdb_env_close(env);
 485       throw std::runtime_error("mdb_dbi_open typeddbi failed");
 486     }
 487
 488     if ((rc = mdb_dbi_open(txn, tdbname.c_str(), MDB_CREATE, &totypeddbi[index])) != 0) {
 489       mdb_dbi_close(env, fromtypeddbi[index]);
 490       mdb_txn_abort(txn);
 491       mdb_env_close(env);
 492       throw std::runtime_error("mdb_dbi_open typeddbi target failed");
 493     }
 494
 495     try {
 496       copyTypedDBI(txn, fromtypeddbi[index], totypeddbi[index]);
 497     }
 498     catch (std::exception& e) {
 499       mdb_dbi_close(env, totypeddbi[index]);
 500       mdb_dbi_close(env, fromtypeddbi[index]);
 501       mdb_txn_abort(txn);
 502       mdb_env_close(env);
 503       throw std::runtime_error("copyTypedDBI failed");
 504     }
 505
 506     // mdb_dbi_close(env, dbi2);
 507     // mdb_dbi_close(env, dbi);
 508     std::cerr << "migrated " << dbname << std::endl;
 509
 510     index++;
 511   }
 512
 513   std::array<MDB_dbi, 4> fromindexdbi;
 514   std::array<MDB_dbi, 4> toindexdbi;
 515
 516   index = 0;
 517
 518   for (const std::string dbname : {"domains", "keydata", "tsig", "metadata"}) {
 519     std::string fdbname = dbname + "_0";
 520     std::cerr << "migrating " << dbname << std::endl;
 521     std::string tdbname = dbname + "_v5_0";
 522
 523     if ((rc = mdb_dbi_open(txn, fdbname.c_str(), 0, &fromindexdbi[index])) != 0) {
 524       mdb_txn_abort(txn);
 525       mdb_env_close(env);
 526       throw std::runtime_error("mdb_dbi_open indexdbi failed");
 527     }
 528
 529     if ((rc = mdb_dbi_open(txn, tdbname.c_str(), MDB_CREATE, &toindexdbi[index])) != 0) {
 530       mdb_dbi_close(env, fromindexdbi[index]);
 531       mdb_txn_abort(txn);
 532       mdb_env_close(env);
 533       throw std::runtime_error("mdb_dbi_open indexdbi target failed");
 534     }
 535
 536     try {
 537       copyIndexDBI(txn, fromindexdbi[index], toindexdbi[index]);
 538     }
 539     catch (std::exception& e) {
 540       mdb_dbi_close(env, toindexdbi[index]);
 541       mdb_dbi_close(env, fromindexdbi[index]);
 542       mdb_txn_abort(txn);
 543       mdb_env_close(env);
 544       throw std::runtime_error("copyIndexDBI failed");
 545     }
 546
 547     // mdb_dbi_close(env, dbi2);
 548     // mdb_dbi_close(env, dbi);
 549     std::cerr << "migrated " << dbname << std::endl;
 550
 551     index++;
 552   }
 553
 554   MDB_dbi dbi;
 555
 556   // finally, migrate the pdns db
 557   if ((rc = mdb_dbi_open(txn, "pdns", 0, &dbi)) != 0) {
 558     mdb_txn_abort(txn);
 559     mdb_env_close(env);
 560     throw std::runtime_error("mdb_dbi_open pdns failed");
 561   }
 562
 563   MDB_val key, data;
 564
 565   std::string header(LMDBLS::LS_MIN_HEADER_SIZE, '\0');
 566
 567   for (const std::string keyname : {"schemaversion", "shards"}) {
 568     cerr << "migrating pdns." << keyname << endl;
 569
 570     key.mv_data = (char*)keyname.c_str();
 571     key.mv_size = keyname.size();
 572
 573     if ((rc = mdb_get(txn, dbi, &key, &data))) {
 574       throw std::runtime_error("mdb_get pdns.shards failed");
 575     }
 576
 577     uint32_t value;
 578
 579     if (data.mv_size != sizeof(uint32_t)) {
 580       throw std::runtime_error("got non-uint32_t key");
 581     }
 582
 583     memcpy((void*)&value, data.mv_data, sizeof(uint32_t));
 584
 585     value = htonl(value);
 586     if (keyname == "schemaversion") {
 587       value = htonl(5);
 588     }
 589
 590     std::string sdata((char*)data.mv_data, data.mv_size);
 591
 592     std::string stdata = header + std::string((char*)&value, sizeof(uint32_t));
 593     ;
 594
 595     MDB_val tdata;
 596
 597     tdata.mv_data = (char*)stdata.c_str();
 598     tdata.mv_size = stdata.size();
 599
 600     if ((rc = mdb_put(txn, dbi, &key, &tdata, 0)) != 0) {
 601       throw std::runtime_error("mdb_put failed");
 602     }
 603   }
 604
 605   for (const std::string keyname : {"uuid"}) {
 606     cerr << "migrating pdns." << keyname << endl;
 607
 608     key.mv_data = (char*)keyname.c_str();
 609     key.mv_size = keyname.size();
 610
 611     if ((rc = mdb_get(txn, dbi, &key, &data))) {
 612       throw std::runtime_error("mdb_get pdns.shards failed");
 613     }
 614
 615     std::string sdata((char*)data.mv_data, data.mv_size);
 616
 617     std::string stdata = header + sdata;
 618
 619     MDB_val tdata;
 620
 621     tdata.mv_data = (char*)stdata.c_str();
 622     tdata.mv_size = stdata.size();
 623
 624     if ((rc = mdb_put(txn, dbi, &key, &tdata, 0)) != 0) {
 625       throw std::runtime_error("mdb_put failed");
 626     }
 627   }
 628
 629   for (int i = 0; i < 4; i++) {
 630     mdb_drop(txn, fromtypeddbi[i], 1);
 631     mdb_drop(txn, fromindexdbi[i], 1);
 632   }
 633
 634   cerr << "txn commit=" << mdb_txn_commit(txn) << endl;
 635
 636   for (int i = 0; i < 4; i++) {
 637     mdb_dbi_close(env, totypeddbi[i]);
 638     mdb_dbi_close(env, toindexdbi[i]);
 639   }
 640   mdb_env_close(env);
 641
 642   // throw std::runtime_error("migration done");
 643   cerr << "migration done" << endl;
 644   // exit(1);
 645   return true;
 646 }
 647
 648 LMDBBackend::LMDBBackend(const std::string& suffix)
 649 {
 650   // overlapping domain ids in combination with relative names are a recipe for disaster
 651   if (!suffix.empty()) {
 652     throw std::runtime_error("LMDB backend does not support multiple instances");
 653   }
 654
 655   setArgPrefix("lmdb" + suffix);
 656
 657   string syncMode = toLower(getArg("sync-mode"));
 658
 659   d_random_ids = mustDo("random-ids");
 660
 661   if (syncMode == "nosync")
 662     d_asyncFlag = MDB_NOSYNC;
 663   else if (syncMode == "nometasync")
 664     d_asyncFlag = MDB_NOMETASYNC;
 665   else if (syncMode.empty() || syncMode == "sync")
 666     d_asyncFlag = 0;
 667   else
 668     throw std::runtime_error("Unknown sync mode " + syncMode + " requested for LMDB backend");
 669
 670   uint64_t mapSize = 0;
 671   try {
 672     mapSize = std::stoll(getArg("map-size"));
 673   }
 674   catch (const std::exception& e) {
 675     throw std::runtime_error(std::string("Unable to parse the 'map-size' LMDB value: ") + e.what());
 676   }
 677
 678   LMDBLS::s_flag_deleted = mustDo("flag-deleted");
 679   d_handle_dups = false;
 680
 681   if (mustDo("lightning-stream")) {
 682     d_random_ids = true;
 683     d_handle_dups = true;
 684     LMDBLS::s_flag_deleted = true;
 685
 686     if (atoi(getArg("shards").c_str()) != 1) {
 687       throw std::runtime_error(std::string("running with Lightning Stream support requires shards=1"));
 688     }
 689   }
 690
 691   bool opened = false;
 692
 693   if (s_first) {
 694     std::lock_guard<std::mutex> l(s_lmdbStartupLock);
 695     if (s_first) {
 696       auto filename = getArg("filename");
 697
 698       auto currentSchemaVersionAndShards = getSchemaVersionAndShards(filename);
 699       uint32_t currentSchemaVersion = currentSchemaVersionAndShards.first;
 700       // std::cerr<<"current schema version: "<<currentSchemaVersion<<", shards="<<currentSchemaVersionAndShards.second<<std::endl;
 701
 702       if (getArgAsNum("schema-version") != SCHEMAVERSION) {
 703         throw std::runtime_error("This version of the lmdbbackend only supports schema version 5. Configuration demands a lower version. Not starting up.");
 704       }
 705
 706       if (currentSchemaVersion > 0 && currentSchemaVersion < 3) {
 707         throw std::runtime_error("this version of the lmdbbackend can only upgrade from schema v3/v4 to v5. Upgrading from older schemas is not yet supported.");
 708       }
 709
 710       if (currentSchemaVersion == 0) {
 711         // no database is present yet, we can just create them
 712         currentSchemaVersion = 5;
 713       }
 714
 715       if (currentSchemaVersion == 3 || currentSchemaVersion == 4) {
 716         if (!upgradeToSchemav5(filename)) {
 717           throw std::runtime_error("Failed to perform LMDB schema version upgrade from v4 to v5");
 718         }
 719         currentSchemaVersion = 5;
 720       }
 721
 722       if (currentSchemaVersion != 5) {
 723         throw std::runtime_error("Somehow, we are not at schema version 5. Giving up");
 724       }
 725
 726       d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | d_asyncFlag, 0600, mapSize), "domains_v5");
 727       d_tmeta = std::make_shared<tmeta_t>(d_tdomains->getEnv(), "metadata_v5");
 728       d_tkdb = std::make_shared<tkdb_t>(d_tdomains->getEnv(), "keydata_v5");
 729       d_ttsig = std::make_shared<ttsig_t>(d_tdomains->getEnv(), "tsig_v5");
 730
 731       auto pdnsdbi = d_tdomains->getEnv()->openDB("pdns", MDB_CREATE);
 732
 733       opened = true;
 734
 735       auto txn = d_tdomains->getEnv()->getRWTransaction();
 736
 737       MDBOutVal shards;
 738       if (!txn->get(pdnsdbi, "shards", shards)) {
 739         s_shards = shards.get<uint32_t>();
 740
 741         if (mustDo("lightning-stream") && s_shards != 1) {
 742           throw std::runtime_error(std::string("running with Lightning Stream support enabled requires a database with exactly 1 shard"));
 743         }
 744
 745         if (s_shards != atoi(getArg("shards").c_str())) {
 746           g_log << Logger::Warning << "Note: configured number of lmdb shards (" << atoi(getArg("shards").c_str()) << ") is different from on-disk (" << s_shards << "). Using on-disk shard number" << endl;
 747         }
 748       }
 749       else {
 750         s_shards = atoi(getArg("shards").c_str());
 751         txn->put(pdnsdbi, "shards", s_shards);
 752       }
 753
 754       MDBOutVal gotuuid;
 755       if (txn->get(pdnsdbi, "uuid", gotuuid)) {
 756         const auto uuid = getUniqueID();
 757         const string uuids(uuid.begin(), uuid.end());
 758         txn->put(pdnsdbi, "uuid", uuids);
 759       }
 760
 761       MDBOutVal _schemaversion;
 762       if (txn->get(pdnsdbi, "schemaversion", _schemaversion)) {
 763         // our DB is entirely new, so we need to write the schemaversion
 764         txn->put(pdnsdbi, "schemaversion", currentSchemaVersion);
 765       }
 766       txn->commit();
 767
 768       s_first = false;
 769     }
 770   }
 771
 772   if (!opened) {
 773     d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | d_asyncFlag, 0600, mapSize), "domains_v5");
 774     d_tmeta = std::make_shared<tmeta_t>(d_tdomains->getEnv(), "metadata_v5");
 775     d_tkdb = std::make_shared<tkdb_t>(d_tdomains->getEnv(), "keydata_v5");
 776     d_ttsig = std::make_shared<ttsig_t>(d_tdomains->getEnv(), "tsig_v5");
 777   }
 778   d_trecords.resize(s_shards);
 779   d_dolog = ::arg().mustDo("query-logging");
 780 }
 781
 782 namespace boost
 783 {
 784 namespace serialization
 785 {
 786
 787   template <class Archive>
 788   void save(Archive& ar, const DNSName& g, const unsigned int /* version */)
 789   {
 790     if (g.empty()) {
 791       ar& std::string();
 792     }
 793     else {
 794       ar& g.toDNSStringLC();
 795     }
 796   }
 797
 798   template <class Archive>
 799   void load(Archive& ar, DNSName& g, const unsigned int /* version */)
 800   {
 801     string tmp;
 802     ar& tmp;
 803     if (tmp.empty()) {
 804       g = DNSName();
 805     }
 806     else {
 807       g = DNSName(tmp.c_str(), tmp.size(), 0, false);
 808     }
 809   }
 810
 811   template <class Archive>
 812   void save(Archive& ar, const QType& g, const unsigned int /* version */)
 813   {
 814     ar& g.getCode();
 815   }
 816
 817   template <class Archive>
 818   void load(Archive& ar, QType& g, const unsigned int /* version */)
 819   {
 820     uint16_t tmp;
 821     ar& tmp;
 822     g = QType(tmp);
 823   }
 824
 825   template <class Archive>
 826   void save(Archive& ar, const DomainInfo& g, const unsigned int /* version */)
 827   {
 828     ar& g.zone;
 829     ar& g.last_check;
 830     ar& g.account;
 831     ar& g.primaries;
 832     ar& g.id;
 833     ar& g.notified_serial;
 834     ar& g.kind;
 835     ar& g.options;
 836     ar& g.catalog;
 837   }
 838
 839   template <class Archive>
 840   void load(Archive& ar, DomainInfo& g, const unsigned int version)
 841   {
 842     ar& g.zone;
 843     ar& g.last_check;
 844     ar& g.account;
 845     ar& g.primaries;
 846     ar& g.id;
 847     ar& g.notified_serial;
 848     ar& g.kind;
 849     if (version >= 1) {
 850       ar& g.options;
 851       ar& g.catalog;
 852     }
 853     else {
 854       g.options.clear();
 855       g.catalog.clear();
 856     }
 857   }
 858
 859   template <class Archive>
 860   void serialize(Archive& ar, LMDBBackend::DomainMeta& g, const unsigned int /* version */)
 861   {
 862     ar& g.domain& g.key& g.value;
 863   }
 864
 865   template <class Archive>
 866   void save(Archive& ar, const LMDBBackend::KeyDataDB& g, const unsigned int /* version */)
 867   {
 868     ar& g.domain& g.content& g.flags& g.active& g.published;
 869   }
 870
 871   template <class Archive>
 872   void load(Archive& ar, LMDBBackend::KeyDataDB& g, const unsigned int version)
 873   {
 874     ar& g.domain& g.content& g.flags& g.active;
 875     if (version >= 1) {
 876       ar& g.published;
 877     }
 878     else {
 879       g.published = true;
 880     }
 881   }
 882
 883   template <class Archive>
 884   void serialize(Archive& ar, TSIGKey& g, const unsigned int /* version */)
 885   {
 886     ar& g.name;
 887     ar& g.algorithm; // this is the ordername
 888     ar& g.key;
 889   }
 890
 891 } // namespace serialization
 892 } // namespace boost
 893
 894 BOOST_SERIALIZATION_SPLIT_FREE(DNSName);
 895 BOOST_SERIALIZATION_SPLIT_FREE(QType);
 896 BOOST_SERIALIZATION_SPLIT_FREE(LMDBBackend::KeyDataDB);
 897 BOOST_SERIALIZATION_SPLIT_FREE(DomainInfo);
 898 BOOST_IS_BITWISE_SERIALIZABLE(ComboAddress);
 899
 900 template <>
 901 std::string serToString(const LMDBBackend::LMDBResourceRecord& lrr)
 902 {
 903   std::string ret;
 904   uint16_t len = lrr.content.length();
 905   ret.reserve(2 + len + 7);
 906
 907   ret.assign((const char*)&len, 2);
 908   ret += lrr.content;
 909   ret.append((const char*)&lrr.ttl, 4);
 910   ret.append(1, (char)lrr.auth);
 911   ret.append(1, (char)lrr.disabled);
 912   ret.append(1, (char)lrr.ordername);
 913   return ret;
 914 }
 915
 916 template <>
 917 std::string serToString(const vector<LMDBBackend::LMDBResourceRecord>& lrrs)
 918 {
 919   std::string ret;
 920   for (const auto& lrr : lrrs) {
 921     ret += serToString(lrr);
 922   }
 923   return ret;
 924 }
 925
 926 static inline size_t serOneRRFromString(const string_view& str, LMDBBackend::LMDBResourceRecord& lrr)
 927 {
 928   uint16_t len;
 929   memcpy(&len, &str[0], 2);
 930   lrr.content.assign(&str[2], len); // len bytes
 931   memcpy(&lrr.ttl, &str[2] + len, 4);
 932   lrr.auth = str[2 + len + 4];
 933   lrr.disabled = str[2 + len + 4 + 1];
 934   lrr.ordername = str[2 + len + 4 + 2];
 935   lrr.wildcardname.clear();
 936
 937   return 2 + len + 7;
 938 }
 939
 940 template <>
 941 void serFromString(const string_view& str, LMDBBackend::LMDBResourceRecord& lrr)
 942 {
 943   serOneRRFromString(str, lrr);
 944 }
 945
 946 template <>
 947 void serFromString(const string_view& str, vector<LMDBBackend::LMDBResourceRecord>& lrrs)
 948 {
 949   auto str_copy = str;
 950   while (str_copy.size() >= 9) { // minimum length for a record is 10
 951     LMDBBackend::LMDBResourceRecord lrr;
 952     auto rrLength = serOneRRFromString(str_copy, lrr);
 953     lrrs.emplace_back(lrr);
 954     str_copy.remove_prefix(rrLength);
 955   }
 956 }
 957
 958 static std::string serializeContent(uint16_t qtype, const DNSName& domain, const std::string& content)
 959 {
 960   auto drc = DNSRecordContent::make(qtype, QClass::IN, content);
 961   return drc->serialize(domain, false);
 962 }
 963
 964 static std::shared_ptr<DNSRecordContent> deserializeContentZR(uint16_t qtype, const DNSName& qname, const std::string& content)
 965 {
 966   if (qtype == QType::A && content.size() == 4) {
 967     return std::make_shared<ARecordContent>(*((uint32_t*)content.c_str()));
 968   }
 969   return DNSRecordContent::deserialize(qname, qtype, content);
 970 }
 971
 972 /* design. If you ask a question without a zone id, we lookup the best
 973    zone id for you, and answer from that. This is different than other backends, but I can't see why it would not work.
 974
 975    The index we use is "zoneid,canonical relative name". This index is also used
 976    for AXFR.
 977
 978    Note - domain_id, name and type are ONLY present on the index!
 979 */
 980
 981 #if BOOST_VERSION >= 106100
 982 #define StringView string_view
 983 #else
 984 #define StringView string
 985 #endif
 986
 987 void LMDBBackend::deleteDomainRecords(RecordsRWTransaction& txn, uint32_t domain_id, uint16_t qtype)
 988 {
 989   compoundOrdername co;
 990   string match = co(domain_id);
 991
 992   auto cursor = txn.txn->getCursor(txn.db->dbi);
 993   MDBOutVal key, val;
 994   //  cout<<"Match: "<<makeHexDump(match);
 995   if (!cursor.lower_bound(match, key, val)) {
 996     while (key.getNoStripHeader<StringView>().rfind(match, 0) == 0) {
 997       if (qtype == QType::ANY || co.getQType(key.getNoStripHeader<StringView>()) == qtype)
 998         cursor.del();
 999       if (cursor.next(key, val))
1000         break;
1001     }
1002   }
1003 }
1004
1005 /* Here's the complicated story. Other backends have just one transaction, which is either
1006    on or not.
1007
1008    You can't call feedRecord without a transaction started with startTransaction.
1009
1010    However, other functions can be called after startTransaction() or without startTransaction()
1011      (like updateDNSSECOrderNameAndAuth)
1012
1013
1014
1015 */
1016
1017 bool LMDBBackend::startTransaction(const DNSName& domain, int domain_id)
1018 {
1019   // cout <<"startTransaction("<<domain<<", "<<domain_id<<")"<<endl;
1020   int real_id = domain_id;
1021   if (real_id < 0) {
1022     auto rotxn = d_tdomains->getROTransaction();
1023     DomainInfo di;
1024     real_id = rotxn.get<0>(domain, di);
1025     // cout<<"real_id = "<<real_id << endl;
1026     if (!real_id)
1027       return false;
1028   }
1029   if (d_rwtxn) {
1030     throw DBException("Attempt to start a transaction while one was open already");
1031   }
1032   d_rwtxn = getRecordsRWTransaction(real_id);
1033
1034   d_transactiondomain = domain;
1035   d_transactiondomainid = real_id;
1036   if (domain_id >= 0) {
1037     deleteDomainRecords(*d_rwtxn, domain_id);
1038   }
1039
1040   return true;
1041 }
1042
1043 bool LMDBBackend::commitTransaction()
1044 {
1045   // cout<<"Commit transaction" <<endl;
1046   if (!d_rwtxn) {
1047     throw DBException("Attempt to commit a transaction while there isn't one open");
1048   }
1049
1050   d_rwtxn->txn->commit();
1051   d_rwtxn.reset();
1052   return true;
1053 }
1054
1055 bool LMDBBackend::abortTransaction()
1056 {
1057   // cout<<"Abort transaction"<<endl;
1058   if (!d_rwtxn) {
1059     throw DBException("Attempt to abort a transaction while there isn't one open");
1060   }
1061
1062   d_rwtxn->txn->abort();
1063   d_rwtxn.reset();
1064
1065   return true;
1066 }
1067
1068 // d_rwtxn must be set here
1069 bool LMDBBackend::feedRecord(const DNSResourceRecord& r, const DNSName& ordername, bool ordernameIsNSEC3)
1070 {
1071   LMDBResourceRecord lrr(r);
1072   lrr.qname.makeUsRelative(d_transactiondomain);
1073   lrr.content = serializeContent(lrr.qtype.getCode(), r.qname, lrr.content);
1074
1075   compoundOrdername co;
1076   string matchName = co(lrr.domain_id, lrr.qname, lrr.qtype.getCode());
1077
1078   string rrs;
1079   MDBOutVal _rrs;
1080   if (!d_rwtxn->txn->get(d_rwtxn->db->dbi, matchName, _rrs)) {
1081     rrs = _rrs.get<string>();
1082   }
1083
1084   rrs += serToString(lrr);
1085
1086   d_rwtxn->txn->put(d_rwtxn->db->dbi, matchName, rrs);
1087
1088   if (ordernameIsNSEC3 && !ordername.empty()) {
1089     MDBOutVal val;
1090     if (d_rwtxn->txn->get(d_rwtxn->db->dbi, co(lrr.domain_id, lrr.qname, QType::NSEC3), val)) {
1091       lrr.ttl = 0;
1092       lrr.content = lrr.qname.toDNSStringLC();
1093       lrr.auth = 0;
1094       string ser = serToString(lrr);
1095       d_rwtxn->txn->put(d_rwtxn->db->dbi, co(lrr.domain_id, ordername, QType::NSEC3), ser);
1096
1097       lrr.ttl = 1;
1098       lrr.content = ordername.toDNSString();
1099       ser = serToString(lrr);
1100       d_rwtxn->txn->put(d_rwtxn->db->dbi, co(lrr.domain_id, lrr.qname, QType::NSEC3), ser);
1101     }
1102   }
1103   return true;
1104 }
1105
1106 bool LMDBBackend::feedEnts(int domain_id, map<DNSName, bool>& nonterm)
1107 {
1108   LMDBResourceRecord lrr;
1109   lrr.ttl = 0;
1110   compoundOrdername co;
1111   for (const auto& nt : nonterm) {
1112     lrr.qname = nt.first.makeRelative(d_transactiondomain);
1113     lrr.auth = nt.second;
1114     lrr.ordername = true;
1115
1116     std::string ser = serToString(lrr);
1117     d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, lrr.qname, QType::ENT), ser);
1118   }
1119   return true;
1120 }
1121
1122 bool LMDBBackend::feedEnts3(int domain_id, const DNSName& domain, map<DNSName, bool>& nonterm, const NSEC3PARAMRecordContent& ns3prc, bool narrow)
1123 {
1124   string ser;
1125   DNSName ordername;
1126   LMDBResourceRecord lrr;
1127   compoundOrdername co;
1128   for (const auto& nt : nonterm) {
1129     lrr.qname = nt.first.makeRelative(domain);
1130     lrr.ttl = 0;
1131     lrr.auth = nt.second;
1132     lrr.ordername = nt.second;
1133     ser = serToString(lrr);
1134     d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, lrr.qname, QType::ENT), ser);
1135
1136     if (!narrow && lrr.auth) {
1137       lrr.content = lrr.qname.toDNSString();
1138       lrr.auth = false;
1139       lrr.ordername = false;
1140       ser = serToString(lrr);
1141
1142       ordername = DNSName(toBase32Hex(hashQNameWithSalt(ns3prc, nt.first)));
1143       d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, ordername, QType::NSEC3), ser);
1144
1145       lrr.ttl = 1;
1146       lrr.content = ordername.toDNSString();
1147       ser = serToString(lrr);
1148       d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, lrr.qname, QType::NSEC3), ser);
1149     }
1150   }
1151   return true;
1152 }
1153
1154 // might be called within a transaction, might also be called alone
1155 bool LMDBBackend::replaceRRSet(uint32_t domain_id, const DNSName& qname, const QType& qt, const vector<DNSResourceRecord>& rrset)
1156 {
1157   // zonk qname/qtype within domain_id (go through qname, check domain_id && qtype)
1158   shared_ptr<RecordsRWTransaction> txn;
1159   bool needCommit = false;
1160   if (d_rwtxn && d_transactiondomainid == domain_id) {
1161     txn = d_rwtxn;
1162     //    cout<<"Reusing open transaction"<<endl;
1163   }
1164   else {
1165     //    cout<<"Making a new RW txn for replace rrset"<<endl;
1166     txn = getRecordsRWTransaction(domain_id);
1167     needCommit = true;
1168   }
1169
1170   DomainInfo di;
1171   if (!d_tdomains->getROTransaction().get(domain_id, di)) {
1172     return false;
1173   }
1174
1175   compoundOrdername co;
1176   auto cursor = txn->txn->getCursor(txn->db->dbi);
1177   MDBOutVal key, val;
1178   string match = co(domain_id, qname.makeRelative(di.zone), qt.getCode());
1179   if (!cursor.find(match, key, val)) {
1180     cursor.del();
1181   }
1182
1183   if (!rrset.empty()) {
1184     vector<LMDBResourceRecord> adjustedRRSet;
1185     for (const auto& rr : rrset) {
1186       LMDBResourceRecord lrr(rr);
1187       lrr.content = serializeContent(lrr.qtype.getCode(), lrr.qname, lrr.content);
1188       lrr.qname.makeUsRelative(di.zone);
1189
1190       adjustedRRSet.emplace_back(lrr);
1191     }
1192     txn->txn->put(txn->db->dbi, match, serToString(adjustedRRSet));
1193   }
1194
1195   if (needCommit)
1196     txn->txn->commit();
1197
1198   return true;
1199 }
1200
1201 bool LMDBBackend::replaceComments([[maybe_unused]] const uint32_t domain_id, [[maybe_unused]] const DNSName& qname, [[maybe_unused]] const QType& qt, const vector<Comment>& comments)
1202 {
1203   // if the vector is empty, good, that's what we do here (LMDB does not store comments)
1204   // if it's not, report failure
1205   return comments.empty();
1206 }
1207
1208 // tempting to templatize these two functions but the pain is not worth it
1209 std::shared_ptr<LMDBBackend::RecordsRWTransaction> LMDBBackend::getRecordsRWTransaction(uint32_t id)
1210 {
1211   auto& shard = d_trecords[id % s_shards];
1212   if (!shard.env) {
1213     shard.env = getMDBEnv((getArg("filename") + "-" + std::to_string(id % s_shards)).c_str(),
1214                           MDB_NOSUBDIR | d_asyncFlag, 0600);
1215     shard.dbi = shard.env->openDB("records_v5", MDB_CREATE);
1216   }
1217   auto ret = std::make_shared<RecordsRWTransaction>(shard.env->getRWTransaction());
1218   ret->db = std::make_shared<RecordsDB>(shard);
1219
1220   return ret;
1221 }
1222
1223 std::shared_ptr<LMDBBackend::RecordsROTransaction> LMDBBackend::getRecordsROTransaction(uint32_t id, const std::shared_ptr<LMDBBackend::RecordsRWTransaction>& rwtxn)
1224 {
1225   auto& shard = d_trecords[id % s_shards];
1226   if (!shard.env) {
1227     if (rwtxn) {
1228       throw DBException("attempting to start nested transaction without open parent env");
1229     }
1230     shard.env = getMDBEnv((getArg("filename") + "-" + std::to_string(id % s_shards)).c_str(),
1231                           MDB_NOSUBDIR | d_asyncFlag, 0600);
1232     shard.dbi = shard.env->openDB("records_v5", MDB_CREATE);
1233   }
1234
1235   if (rwtxn) {
1236     auto ret = std::make_shared<RecordsROTransaction>(rwtxn->txn->getROTransaction());
1237     ret->db = std::make_shared<RecordsDB>(shard);
1238     return ret;
1239   }
1240   else {
1241     auto ret = std::make_shared<RecordsROTransaction>(shard.env->getROTransaction());
1242     ret->db = std::make_shared<RecordsDB>(shard);
1243     return ret;
1244   }
1245 }
1246
1247 #if 0
1248 // FIXME reinstate soon
1249 bool LMDBBackend::upgradeToSchemav3()
1250 {
1251   g_log << Logger::Warning << "Upgrading LMDB schema" << endl;
1252
1253   for (auto i = 0; i < s_shards; i++) {
1254     string filename = getArg("filename") + "-" + std::to_string(i);
1255     if (rename(filename.c_str(), (filename + "-old").c_str()) < 0) {
1256       if (errno == ENOENT) {
1257         // apparently this shard doesn't exist yet, moving on
1258         continue;
1259       }
1260       unixDie("Rename failed during LMDB upgrade");
1261     }
1262
1263     LMDBBackend::RecordsDB oldShard, newShard;
1264
1265     oldShard.env = getMDBEnv((filename + "-old").c_str(),
1266                              MDB_NOSUBDIR | d_asyncFlag, 0600);
1267     oldShard.dbi = oldShard.env->openDB("records", MDB_CREATE | MDB_DUPSORT);
1268     auto txn = oldShard.env->getROTransaction();
1269     auto cursor = txn->getROCursor(oldShard.dbi);
1270
1271     newShard.env = getMDBEnv((filename).c_str(),
1272                              MDB_NOSUBDIR | d_asyncFlag, 0600);
1273     newShard.dbi = newShard.env->openDB("records", MDB_CREATE);
1274     auto newTxn = newShard.env->getRWTransaction();
1275
1276     MDBOutVal key, val;
1277     if (cursor.first(key, val) != 0) {
1278       cursor.close();
1279       txn->abort();
1280       newTxn->abort();
1281       continue;
1282     }
1283     string_view currentKey;
1284     string value;
1285     for (;;) {
1286       auto newKey = key.getNoStripHeader<string_view>();
1287       if (currentKey.compare(newKey) != 0) {
1288         if (value.size() > 0) {
1289           newTxn->put(newShard.dbi, currentKey, value);
1290         }
1291         currentKey = newKey;
1292         value = "";
1293       }
1294       value += val.get<string>();
1295       if (cursor.next(key, val) != 0) {
1296         if (value.size() > 0) {
1297           newTxn->put(newShard.dbi, currentKey, value);
1298         }
1299         break;
1300       }
1301     }
1302
1303     cursor.close();
1304     txn->commit();
1305     newTxn->commit();
1306   }
1307
1308   return true;
1309 }
1310 #endif
1311
1312 bool LMDBBackend::deleteDomain(const DNSName& domain)
1313 {
1314   if (!d_rwtxn) {
1315     throw DBException(std::string(__PRETTY_FUNCTION__) + " called without a transaction");
1316   }
1317
1318   int transactionDomainId = d_transactiondomainid;
1319   DNSName transactionDomain = d_transactiondomain;
1320
1321   abortTransaction();
1322
1323   LMDBIDvec idvec;
1324
1325   if (!d_handle_dups) {
1326     // get domain id
1327     auto txn = d_tdomains->getROTransaction();
1328
1329     DomainInfo di;
1330     idvec.push_back(txn.get<0>(domain, di));
1331   }
1332   else {
1333     // this transaction used to be RO.
1334     // it is now RW to narrow a race window between PowerDNS and Lightning Stream
1335     // FIXME: turn the entire delete, including this ID scan, into one RW transaction
1336     // when doing that, first do a short RO check to see if we actually have anything to delete
1337     auto txn = d_tdomains->getRWTransaction();
1338
1339     txn.get_multi<0>(domain, idvec);
1340   }
1341
1342   for (auto id : idvec) {
1343
1344     startTransaction(domain, id);
1345
1346     { // Remove metadata
1347       auto txn = d_tmeta->getRWTransaction();
1348       LMDBIDvec ids;
1349
1350       txn.get_multi<0>(domain, ids);
1351
1352       for (auto& _id : ids) {
1353         txn.del(_id);
1354       }
1355
1356       txn.commit();
1357     }
1358
1359     { // Remove cryptokeys
1360       auto txn = d_tkdb->getRWTransaction();
1361       LMDBIDvec ids;
1362       txn.get_multi<0>(domain, ids);
1363
1364       for (auto _id : ids) {
1365         txn.del(_id);
1366       }
1367
1368       txn.commit();
1369     }
1370
1371     // Remove records
1372     commitTransaction();
1373
1374     // Remove zone
1375     auto txn = d_tdomains->getRWTransaction();
1376     txn.del(id);
1377     txn.commit();
1378   }
1379
1380   startTransaction(transactionDomain, transactionDomainId);
1381
1382   return true;
1383 }
1384
1385 bool LMDBBackend::list(const DNSName& target, int /* id */, bool include_disabled)
1386 {
1387   d_includedisabled = include_disabled;
1388
1389   DomainInfo di;
1390   {
1391     auto dtxn = d_tdomains->getROTransaction();
1392     if ((di.id = dtxn.get<0>(target, di))) {
1393       // cerr << "Found domain " << target << " on domain_id " << di.id << ", list requested " << id << endl;
1394     }
1395     else {
1396       // cerr << "Did not find " << target << endl;
1397       return false;
1398     }
1399   }
1400
1401   d_rotxn = getRecordsROTransaction(di.id, d_rwtxn);
1402   d_getcursor = std::make_shared<MDBROCursor>(d_rotxn->txn->getCursor(d_rotxn->db->dbi));
1403
1404   compoundOrdername co;
1405   d_matchkey = co(di.id);
1406
1407   MDBOutVal key, val;
1408   auto a = d_getcursor->lower_bound(d_matchkey, key, val);
1409   auto b0 = key.getNoStripHeader<StringView>();
1410   auto b = b0.rfind(d_matchkey, 0);
1411   if (a || b != 0) {
1412     d_getcursor.reset();
1413   }
1414
1415   d_lookupdomain = target;
1416
1417   // Make sure we start with fresh data
1418   d_currentrrset.clear();
1419   d_currentrrsetpos = 0;
1420
1421   return true;
1422 }
1423
1424 void LMDBBackend::lookup(const QType& type, const DNSName& qdomain, int zoneId, DNSPacket* /* p */)
1425 {
1426   if (d_dolog) {
1427     g_log << Logger::Warning << "Got lookup for " << qdomain << "|" << type.toString() << " in zone " << zoneId << endl;
1428     d_dtime.set();
1429   }
1430
1431   d_includedisabled = false;
1432
1433   DNSName hunt(qdomain);
1434   DomainInfo di;
1435   if (zoneId < 0) {
1436     auto rotxn = d_tdomains->getROTransaction();
1437
1438     do {
1439       zoneId = rotxn.get<0>(hunt, di);
1440     } while (!zoneId && type != QType::SOA && hunt.chopOff());
1441     if (zoneId <= 0) {
1442       //      cout << "Did not find zone for "<< qdomain<<endl;
1443       d_getcursor.reset();
1444       return;
1445     }
1446   }
1447   else {
1448     if (!d_tdomains->getROTransaction().get(zoneId, di)) {
1449       // cout<<"Could not find a zone with id "<<zoneId<<endl;
1450       d_getcursor.reset();
1451       return;
1452     }
1453     hunt = di.zone;
1454   }
1455
1456   DNSName relqname = qdomain.makeRelative(hunt);
1457   if (relqname.empty()) {
1458     return;
1459   }
1460   // cout<<"get will look for "<<relqname<< " in zone "<<hunt<<" with id "<<zoneId<<" and type "<<type.toString()<<endl;
1461   d_rotxn = getRecordsROTransaction(zoneId, d_rwtxn);
1462
1463   compoundOrdername co;
1464   d_getcursor = std::make_shared<MDBROCursor>(d_rotxn->txn->getCursor(d_rotxn->db->dbi));
1465   MDBOutVal key, val;
1466   if (type.getCode() == QType::ANY) {
1467     d_matchkey = co(zoneId, relqname);
1468   }
1469   else {
1470     d_matchkey = co(zoneId, relqname, type.getCode());
1471   }
1472
1473   if (d_getcursor->lower_bound(d_matchkey, key, val) || key.getNoStripHeader<StringView>().rfind(d_matchkey, 0) != 0) {
1474     d_getcursor.reset();
1475     if (d_dolog) {
1476       g_log << Logger::Warning << "Query " << ((long)(void*)this) << ": " << d_dtime.udiffNoReset() << " us to execute (found nothing)" << endl;
1477     }
1478     return;
1479   }
1480
1481   if (d_dolog) {
1482     g_log << Logger::Warning << "Query " << ((long)(void*)this) << ": " << d_dtime.udiffNoReset() << " us to execute" << endl;
1483   }
1484
1485   d_lookupdomain = hunt;
1486
1487   // Make sure we start with fresh data
1488   d_currentrrset.clear();
1489   d_currentrrsetpos = 0;
1490 }
1491
1492 bool LMDBBackend::get(DNSZoneRecord& zr)
1493 {
1494   for (;;) {
1495     // std::cerr<<"d_getcursor="<<d_getcursor<<std::endl;
1496     if (!d_getcursor) {
1497       d_rotxn.reset();
1498       return false;
1499     }
1500
1501     string_view key;
1502
1503     if (d_currentrrset.empty()) {
1504       d_getcursor->current(d_currentKey, d_currentVal);
1505
1506       key = d_currentKey.getNoStripHeader<string_view>();
1507       zr.dr.d_type = compoundOrdername::getQType(key).getCode();
1508
1509       if (zr.dr.d_type == QType::NSEC3) {
1510         // Hit a magic NSEC3 skipping
1511         if (d_getcursor->next(d_currentKey, d_currentVal) || d_currentKey.getNoStripHeader<StringView>().rfind(d_matchkey, 0) != 0) {
1512           // cerr<<"resetting d_getcursor 1"<<endl;
1513           d_getcursor.reset();
1514         }
1515         continue;
1516       }
1517
1518       serFromString(d_currentVal.get<string_view>(), d_currentrrset);
1519       d_currentrrsetpos = 0;
1520     }
1521     else {
1522       key = d_currentKey.getNoStripHeader<string_view>();
1523     }
1524     try {
1525       const auto& lrr = d_currentrrset.at(d_currentrrsetpos++);
1526
1527       zr.disabled = lrr.disabled;
1528       if (!zr.disabled || d_includedisabled) {
1529         zr.dr.d_name = compoundOrdername::getQName(key) + d_lookupdomain;
1530         zr.domain_id = compoundOrdername::getDomainID(key);
1531         zr.dr.d_type = compoundOrdername::getQType(key).getCode();
1532         zr.dr.d_ttl = lrr.ttl;
1533         zr.dr.setContent(deserializeContentZR(zr.dr.d_type, zr.dr.d_name, lrr.content));
1534         zr.auth = lrr.auth;
1535       }
1536
1537       if (d_currentrrsetpos >= d_currentrrset.size()) {
1538         d_currentrrset.clear(); // will invalidate lrr
1539         if (d_getcursor->next(d_currentKey, d_currentVal) || d_currentKey.getNoStripHeader<StringView>().rfind(d_matchkey, 0) != 0) {
1540           // cerr<<"resetting d_getcursor 2"<<endl;
1541           d_getcursor.reset();
1542         }
1543       }
1544
1545       if (zr.disabled && !d_includedisabled) {
1546         continue;
1547       }
1548     }
1549     catch (const std::exception& e) {
1550       throw PDNSException(e.what());
1551     }
1552
1553     break;
1554   }
1555
1556   return true;
1557 }
1558
1559 bool LMDBBackend::get(DNSResourceRecord& rr)
1560 {
1561   DNSZoneRecord zr;
1562   if (!get(zr)) {
1563     return false;
1564   }
1565
1566   rr.qname = zr.dr.d_name;
1567   rr.ttl = zr.dr.d_ttl;
1568   rr.qtype = zr.dr.d_type;
1569   rr.content = zr.dr.getContent()->getZoneRepresentation(true);
1570   rr.domain_id = zr.domain_id;
1571   rr.auth = zr.auth;
1572   rr.disabled = zr.disabled;
1573
1574   return true;
1575 }
1576
1577 bool LMDBBackend::getSerial(DomainInfo& di)
1578 {
1579   auto txn = getRecordsROTransaction(di.id);
1580   compoundOrdername co;
1581   MDBOutVal val;
1582   if (!txn->txn->get(txn->db->dbi, co(di.id, g_rootdnsname, QType::SOA), val)) {
1583     LMDBResourceRecord lrr;
1584     serFromString(val.get<string_view>(), lrr);
1585     if (lrr.content.size() >= 5 * sizeof(uint32_t)) {
1586       uint32_t serial;
1587       // a SOA has five 32 bit fields, the first of which is the serial
1588       // there are two variable length names before the serial, so we calculate from the back
1589       memcpy(&serial, &lrr.content[lrr.content.size() - (5 * sizeof(uint32_t))], sizeof(serial));
1590       di.serial = ntohl(serial);
1591     }
1592     return !lrr.disabled;
1593   }
1594   return false;
1595 }
1596
1597 bool LMDBBackend::getDomainInfo(const DNSName& domain, DomainInfo& di, bool getserial)
1598 {
1599   {
1600     auto txn = d_tdomains->getROTransaction();
1601     // auto range = txn.prefix_range<0>(domain);
1602
1603     // bool found = false;
1604
1605     // for (auto& iter = range.first ; iter != range.second; ++iter) {
1606     //   found = true;
1607     //   di.id = iter.getID();
1608     //   di.backend = this;
1609     // }
1610
1611     // if (!found) {
1612     //   return false;
1613     // }
1614     if (!(di.id = txn.get<0>(domain, di))) {
1615       return false;
1616     }
1617
1618     di.backend = this;
1619   }
1620
1621   if (getserial) {
1622     getSerial(di);
1623   }
1624
1625   return true;
1626 }
1627
1628 int LMDBBackend::genChangeDomain(const DNSName& domain, const std::function<void(DomainInfo&)>& func)
1629 {
1630   auto txn = d_tdomains->getRWTransaction();
1631
1632   DomainInfo di;
1633
1634   auto id = txn.get<0>(domain, di);
1635   func(di);
1636   txn.put(di, id);
1637
1638   txn.commit();
1639   return true;
1640 }
1641
1642 int LMDBBackend::genChangeDomain(uint32_t id, const std::function<void(DomainInfo&)>& func)
1643 {
1644   DomainInfo di;
1645
1646   auto txn = d_tdomains->getRWTransaction();
1647
1648   if (!txn.get(id, di))
1649     return false;
1650
1651   func(di);
1652
1653   txn.put(di, id);
1654
1655   txn.commit();
1656   return true;
1657 }
1658
1659 bool LMDBBackend::setKind(const DNSName& domain, const DomainInfo::DomainKind kind)
1660 {
1661   return genChangeDomain(domain, [kind](DomainInfo& di) {
1662     di.kind = kind;
1663   });
1664 }
1665
1666 bool LMDBBackend::setAccount(const DNSName& domain, const std::string& account)
1667 {
1668   return genChangeDomain(domain, [account](DomainInfo& di) {
1669     di.account = account;
1670   });
1671 }
1672
1673 bool LMDBBackend::setPrimaries(const DNSName& domain, const vector<ComboAddress>& primaries)
1674 {
1675   return genChangeDomain(domain, [&primaries](DomainInfo& di) {
1676     di.primaries = primaries;
1677   });
1678 }
1679
1680 bool LMDBBackend::createDomain(const DNSName& domain, const DomainInfo::DomainKind kind, const vector<ComboAddress>& primaries, const string& account)
1681 {
1682   DomainInfo di;
1683
1684   {
1685     auto txn = d_tdomains->getRWTransaction();
1686     if (txn.get<0>(domain, di)) {
1687       throw DBException("Domain '" + domain.toLogString() + "' exists already");
1688     }
1689
1690     di.zone = domain;
1691     di.kind = kind;
1692     di.primaries = primaries;
1693     di.account = account;
1694
1695     txn.put(di, 0, d_random_ids);
1696     txn.commit();
1697   }
1698
1699   return true;
1700 }
1701
1702 void LMDBBackend::getAllDomainsFiltered(vector<DomainInfo>* domains, const std::function<bool(DomainInfo&)>& allow)
1703 {
1704   auto txn = d_tdomains->getROTransaction();
1705   if (d_handle_dups) {
1706     map<DNSName, DomainInfo> zonemap;
1707     set<DNSName> dups;
1708
1709     for (auto iter = txn.begin(); iter != txn.end(); ++iter) {
1710       DomainInfo di = *iter;
1711       di.id = iter.getID();
1712       di.backend = this;
1713
1714       if (!zonemap.emplace(di.zone, di).second) {
1715         dups.insert(di.zone);
1716       }
1717     }
1718
1719     for (const auto& zone : dups) {
1720       DomainInfo di;
1721
1722       // this get grabs the oldest item if there are duplicates
1723       di.id = txn.get<0>(zone, di);
1724
1725       if (di.id == 0) {
1726         // .get actually found nothing for us
1727         continue;
1728       }
1729
1730       di.backend = this;
1731       zonemap[di.zone] = di;
1732     }
1733
1734     for (auto& [k, v] : zonemap) {
1735       if (allow(v)) {
1736         domains->push_back(std::move(v));
1737       }
1738     }
1739   }
1740   else {
1741     for (auto iter = txn.begin(); iter != txn.end(); ++iter) {
1742       DomainInfo di = *iter;
1743       di.id = iter.getID();
1744       di.backend = this;
1745
1746       if (allow(di)) {
1747         domains->push_back(di);
1748       }
1749     }
1750   }
1751 }
1752
1753 void LMDBBackend::getAllDomains(vector<DomainInfo>* domains, bool /* doSerial */, bool include_disabled)
1754 {
1755   domains->clear();
1756
1757   getAllDomainsFiltered(domains, [this, include_disabled](DomainInfo& di) {
1758     if (!getSerial(di) && !include_disabled) {
1759       return false;
1760     }
1761
1762     return true;
1763   });
1764 }
1765
1766 void LMDBBackend::getUnfreshSecondaryInfos(vector<DomainInfo>* domains)
1767 {
1768   uint32_t serial;
1769   time_t now = time(0);
1770   LMDBResourceRecord lrr;
1771   soatimes st;
1772
1773   getAllDomainsFiltered(domains, [this, &lrr, &st, &now, &serial](DomainInfo& di) {
1774     if (!di.isSecondaryType()) {
1775       return false;
1776     }
1777
1778     auto txn2 = getRecordsROTransaction(di.id);
1779     compoundOrdername co;
1780     MDBOutVal val;
1781     if (!txn2->txn->get(txn2->db->dbi, co(di.id, g_rootdnsname, QType::SOA), val)) {
1782       serFromString(val.get<string_view>(), lrr);
1783       memcpy(&st, &lrr.content[lrr.content.size() - sizeof(soatimes)], sizeof(soatimes));
1784       if ((time_t)(di.last_check + ntohl(st.refresh)) > now) { // still fresh
1785         return false;
1786       }
1787       serial = ntohl(st.serial);
1788     }
1789     else {
1790       serial = 0;
1791     }
1792
1793     return true;
1794   });
1795 }
1796
1797 void LMDBBackend::setStale(uint32_t domain_id)
1798 {
1799   genChangeDomain(domain_id, [](DomainInfo& di) {
1800     di.last_check = 0;
1801   });
1802 }
1803
1804 void LMDBBackend::setFresh(uint32_t domain_id)
1805 {
1806   genChangeDomain(domain_id, [](DomainInfo& di) {
1807     di.last_check = time(nullptr);
1808   });
1809 }
1810
1811 void LMDBBackend::getUpdatedPrimaries(vector<DomainInfo>& updatedDomains, std::unordered_set<DNSName>& catalogs, CatalogHashMap& catalogHashes)
1812 {
1813   CatalogInfo ci;
1814
1815   getAllDomainsFiltered(&(updatedDomains), [this, &catalogs, &catalogHashes, &ci](DomainInfo& di) {
1816     if (!di.isPrimaryType()) {
1817       return false;
1818     }
1819
1820     if (di.kind == DomainInfo::Producer) {
1821       catalogs.insert(di.zone);
1822       catalogHashes[di.zone].process("\0");
1823       return false; // Producer fresness check is performed elsewhere
1824     }
1825
1826     if (!di.catalog.empty()) {
1827       ci.fromJson(di.options, CatalogInfo::CatalogType::Producer);
1828       ci.updateHash(catalogHashes, di);
1829     }
1830
1831     if (getSerial(di) && di.serial != di.notified_serial) {
1832       di.backend = this;
1833       return true;
1834     }
1835
1836     return false;
1837   });
1838 }
1839
1840 void LMDBBackend::setNotified(uint32_t domain_id, uint32_t serial)
1841 {
1842   genChangeDomain(domain_id, [serial](DomainInfo& di) {
1843     di.notified_serial = serial;
1844   });
1845 }
1846
1847 class getCatalogMembersReturnFalseException : std::runtime_error
1848 {
1849 public:
1850   getCatalogMembersReturnFalseException() :
1851     std::runtime_error("getCatalogMembers should return false") {}
1852 };
1853
1854 bool LMDBBackend::getCatalogMembers(const DNSName& catalog, vector<CatalogInfo>& members, CatalogInfo::CatalogType type)
1855 {
1856   vector<DomainInfo> scratch;
1857
1858   try {
1859     getAllDomainsFiltered(&scratch, [&catalog, &members, &type](DomainInfo& di) {
1860       if ((type == CatalogInfo::CatalogType::Producer && di.kind != DomainInfo::Primary) || (type == CatalogInfo::CatalogType::Consumer && di.kind != DomainInfo::Secondary) || di.catalog != catalog) {
1861         return false;
1862       }
1863
1864       CatalogInfo ci;
1865       ci.d_id = di.id;
1866       ci.d_zone = di.zone;
1867       ci.d_primaries = di.primaries;
1868       try {
1869         ci.fromJson(di.options, type);
1870       }
1871       catch (const std::runtime_error& e) {
1872         g_log << Logger::Warning << __PRETTY_FUNCTION__ << " options '" << di.options << "' for zone '" << di.zone << "' is no valid JSON: " << e.what() << endl;
1873         members.clear();
1874         throw getCatalogMembersReturnFalseException();
1875       }
1876       members.emplace_back(ci);
1877
1878       return false;
1879     });
1880   }
1881   catch (const getCatalogMembersReturnFalseException& e) {
1882     return false;
1883   }
1884   return true;
1885 }
1886
1887 bool LMDBBackend::setOptions(const DNSName& domain, const std::string& options)
1888 {
1889   return genChangeDomain(domain, [options](DomainInfo& di) {
1890     di.options = options;
1891   });
1892 }
1893
1894 bool LMDBBackend::setCatalog(const DNSName& domain, const DNSName& catalog)
1895 {
1896   return genChangeDomain(domain, [catalog](DomainInfo& di) {
1897     di.catalog = catalog;
1898   });
1899 }
1900
1901 bool LMDBBackend::getAllDomainMetadata(const DNSName& name, std::map<std::string, std::vector<std::string>>& meta)
1902 {
1903   meta.clear();
1904   auto txn = d_tmeta->getROTransaction();
1905   LMDBIDvec ids;
1906   txn.get_multi<0>(name, ids);
1907
1908   DomainMeta dm;
1909   // cerr<<"getAllDomainMetadata start"<<endl;
1910   for (auto id : ids) {
1911     if (txn.get(id, dm)) {
1912       meta[dm.key].push_back(dm.value);
1913     }
1914   }
1915   return true;
1916 }
1917
1918 bool LMDBBackend::setDomainMetadata(const DNSName& name, const std::string& kind, const std::vector<std::string>& meta)
1919 {
1920   auto txn = d_tmeta->getRWTransaction();
1921
1922   LMDBIDvec ids;
1923   txn.get_multi<0>(name, ids);
1924
1925   DomainMeta dmeta;
1926   for (auto id : ids) {
1927     if (txn.get(id, dmeta)) {
1928       if (dmeta.key == kind) {
1929         // cerr<<"delete"<<endl;
1930         txn.del(id);
1931       }
1932     }
1933   }
1934
1935   for (const auto& m : meta) {
1936     DomainMeta dm{name, kind, m};
1937     txn.put(dm, 0, d_random_ids);
1938   }
1939   txn.commit();
1940   return true;
1941 }
1942
1943 bool LMDBBackend::getDomainKeys(const DNSName& name, std::vector<KeyData>& keys)
1944 {
1945   auto txn = d_tkdb->getROTransaction();
1946   LMDBIDvec ids;
1947   txn.get_multi<0>(name, ids);
1948
1949   KeyDataDB key;
1950
1951   for (auto id : ids) {
1952     if (txn.get(id, key)) {
1953       KeyData kd{key.content, id, key.flags, key.active, key.published};
1954       keys.push_back(kd);
1955     }
1956   }
1957
1958   return true;
1959 }
1960
1961 bool LMDBBackend::removeDomainKey(const DNSName& name, unsigned int id)
1962 {
1963   auto txn = d_tkdb->getRWTransaction();
1964   KeyDataDB kdb;
1965   if (txn.get(id, kdb)) {
1966     if (kdb.domain == name) {
1967       txn.del(id);
1968       txn.commit();
1969       return true;
1970     }
1971   }
1972   // cout << "??? wanted to remove domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
1973   return true;
1974 }
1975
1976 bool LMDBBackend::addDomainKey(const DNSName& name, const KeyData& key, int64_t& id)
1977 {
1978   auto txn = d_tkdb->getRWTransaction();
1979   KeyDataDB kdb{name, key.content, key.flags, key.active, key.published};
1980   id = txn.put(kdb, 0, d_random_ids);
1981   txn.commit();
1982
1983   return true;
1984 }
1985
1986 bool LMDBBackend::activateDomainKey(const DNSName& name, unsigned int id)
1987 {
1988   auto txn = d_tkdb->getRWTransaction();
1989   KeyDataDB kdb;
1990   if (txn.get(id, kdb)) {
1991     if (kdb.domain == name) {
1992       txn.modify(id, [](KeyDataDB& kdbarg) {
1993         kdbarg.active = true;
1994       });
1995       txn.commit();
1996       return true;
1997     }
1998   }
1999
2000   // cout << "??? wanted to activate domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
2001   return true;
2002 }
2003
2004 bool LMDBBackend::deactivateDomainKey(const DNSName& name, unsigned int id)
2005 {
2006   auto txn = d_tkdb->getRWTransaction();
2007   KeyDataDB kdb;
2008   if (txn.get(id, kdb)) {
2009     if (kdb.domain == name) {
2010       txn.modify(id, [](KeyDataDB& kdbarg) {
2011         kdbarg.active = false;
2012       });
2013       txn.commit();
2014       return true;
2015     }
2016   }
2017   // cout << "??? wanted to deactivate domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
2018   return true;
2019 }
2020
2021 bool LMDBBackend::publishDomainKey(const DNSName& name, unsigned int id)
2022 {
2023   auto txn = d_tkdb->getRWTransaction();
2024   KeyDataDB kdb;
2025   if (txn.get(id, kdb)) {
2026     if (kdb.domain == name) {
2027       txn.modify(id, [](KeyDataDB& kdbarg) {
2028         kdbarg.published = true;
2029       });
2030       txn.commit();
2031       return true;
2032     }
2033   }
2034
2035   // cout << "??? wanted to hide domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
2036   return true;
2037 }
2038
2039 bool LMDBBackend::unpublishDomainKey(const DNSName& name, unsigned int id)
2040 {
2041   auto txn = d_tkdb->getRWTransaction();
2042   KeyDataDB kdb;
2043   if (txn.get(id, kdb)) {
2044     if (kdb.domain == name) {
2045       txn.modify(id, [](KeyDataDB& kdbarg) {
2046         kdbarg.published = false;
2047       });
2048       txn.commit();
2049       return true;
2050     }
2051   }
2052   // cout << "??? wanted to unhide domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
2053   return true;
2054 }
2055
2056 bool LMDBBackend::getBeforeAndAfterNamesAbsolute(uint32_t id, const DNSName& qname, DNSName& unhashed, DNSName& before, DNSName& after)
2057 {
2058   //  cout << __PRETTY_FUNCTION__<< ": "<<id <<", "<<qname << " " << unhashed<<endl;
2059
2060   DomainInfo di;
2061   if (!d_tdomains->getROTransaction().get(id, di)) {
2062     // domain does not exist, tough luck
2063     return false;
2064   }
2065   // cout <<"Zone: "<<di.zone<<endl;
2066
2067   compoundOrdername co;
2068   auto txn = getRecordsROTransaction(id);
2069
2070   auto cursor = txn->txn->getCursor(txn->db->dbi);
2071   MDBOutVal key, val;
2072
2073   LMDBResourceRecord lrr;
2074
2075   string matchkey = co(id, qname, QType::NSEC3);
2076   if (cursor.lower_bound(matchkey, key, val)) {
2077     // this is beyond the end of the database
2078     // cout << "Beyond end of database!" << endl;
2079     cursor.last(key, val);
2080
2081     for (;;) {
2082       if (co.getDomainID(key.getNoStripHeader<StringView>()) != id) {
2083         //cout<<"Last record also not part of this zone!"<<endl;
2084         // this implies something is wrong in the database, nothing we can do
2085         return false;
2086       }
2087
2088       if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2089         serFromString(val.get<StringView>(), lrr);
2090         if (!lrr.ttl) // the kind of NSEC3 we need
2091           break;
2092       }
2093       if (cursor.prev(key, val)) {
2094         // hit beginning of database, again means something is wrong with it
2095         return false;
2096       }
2097     }
2098     before = co.getQName(key.getNoStripHeader<StringView>());
2099     unhashed = DNSName(lrr.content.c_str(), lrr.content.size(), 0, false) + di.zone;
2100
2101     // now to find after .. at the beginning of the zone
2102     if (cursor.lower_bound(co(id), key, val)) {
2103       // cout<<"hit end of zone find when we shouldn't"<<endl;
2104       return false;
2105     }
2106     for (;;) {
2107       if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2108         serFromString(val.get<StringView>(), lrr);
2109         if (!lrr.ttl)
2110           break;
2111       }
2112
2113       if (cursor.next(key, val) || co.getDomainID(key.getNoStripHeader<StringView>()) != id) {
2114         // cout<<"hit end of zone or database when we shouldn't"<<endl;
2115         return false;
2116       }
2117     }
2118     after = co.getQName(key.getNoStripHeader<StringView>());
2119     // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2120     return true;
2121   }
2122
2123   // cout<<"Ended up at "<<co.getQName(key.get<StringView>()) <<endl;
2124
2125   before = co.getQName(key.getNoStripHeader<StringView>());
2126   if (before == qname) {
2127     // cout << "Ended up on exact right node" << endl;
2128     before = co.getQName(key.getNoStripHeader<StringView>());
2129     // unhashed should be correct now, maybe check?
2130     if (cursor.next(key, val)) {
2131       // xxx should find first hash now
2132
2133       if (cursor.lower_bound(co(id), key, val)) {
2134         // cout<<"hit end of zone find when we shouldn't for id "<<id<< __LINE__<<endl;
2135         return false;
2136       }
2137       for (;;) {
2138         if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2139           serFromString(val.get<StringView>(), lrr);
2140           if (!lrr.ttl)
2141             break;
2142         }
2143
2144         if (cursor.next(key, val) || co.getDomainID(key.getNoStripHeader<StringView>()) != id) {
2145           // cout<<"hit end of zone or database when we shouldn't" << __LINE__<<endl;
2146           return false;
2147         }
2148       }
2149       after = co.getQName(key.getNoStripHeader<StringView>());
2150       // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2151       return true;
2152     }
2153   }
2154   else {
2155     // cout <<"Going backwards to find 'before'"<<endl;
2156     int count = 0;
2157     for (;;) {
2158       if (co.getQName(key.getNoStripHeader<StringView>()).canonCompare(qname) && co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2159         // cout<<"Potentially stopping traverse at "<< co.getQName(key.get<StringView>()) <<", " << (co.getQName(key.get<StringView>()).canonCompare(qname))<<endl;
2160         // cout<<"qname = "<<qname<<endl;
2161         // cout<<"here  = "<<co.getQName(key.get<StringView>())<<endl;
2162         serFromString(val.get<StringView>(), lrr);
2163         if (!lrr.ttl)
2164           break;
2165       }
2166
2167       if (cursor.prev(key, val) || co.getDomainID(key.getNoStripHeader<StringView>()) != id) {
2168         // cout <<"XXX Hit *beginning* of zone or database"<<endl;
2169         // this can happen, must deal with it
2170         // should now find the last hash of the zone
2171
2172         if (cursor.lower_bound(co(id + 1), key, val)) {
2173           // cout << "Could not find the next higher zone, going to the end of the database then"<<endl;
2174           cursor.last(key, val);
2175         }
2176         else
2177           cursor.prev(key, val);
2178
2179         for (;;) {
2180           if (co.getDomainID(key.getNoStripHeader<StringView>()) != id) {
2181             //cout<<"Last record also not part of this zone!"<<endl;
2182             // this implies something is wrong in the database, nothing we can do
2183             return false;
2184           }
2185
2186           if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2187             serFromString(val.get<StringView>(), lrr);
2188             if (!lrr.ttl) // the kind of NSEC3 we need
2189               break;
2190           }
2191           if (cursor.prev(key, val)) {
2192             // hit beginning of database, again means something is wrong with it
2193             return false;
2194           }
2195         }
2196         before = co.getQName(key.getNoStripHeader<StringView>());
2197         unhashed = DNSName(lrr.content.c_str(), lrr.content.size(), 0, false) + di.zone;
2198         // cout <<"Should still find 'after'!"<<endl;
2199         // for 'after', we need to find the first hash of this zone
2200
2201         if (cursor.lower_bound(co(id), key, val)) {
2202           // cout<<"hit end of zone find when we shouldn't"<<endl;
2203           // means database is wrong, nothing we can do
2204           return false;
2205         }
2206         for (;;) {
2207           if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2208             serFromString(val.get<StringView>(), lrr);
2209             if (!lrr.ttl)
2210               break;
2211           }
2212
2213           if (cursor.next(key, val)) {
2214             // means database is wrong, nothing we can do
2215             // cout<<"hit end of zone when we shouldn't 2"<<endl;
2216             return false;
2217           }
2218         }
2219         after = co.getQName(key.getNoStripHeader<StringView>());
2220
2221         // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2222         return true;
2223       }
2224       ++count;
2225     }
2226     before = co.getQName(key.getNoStripHeader<StringView>());
2227     unhashed = DNSName(lrr.content.c_str(), lrr.content.size(), 0, false) + di.zone;
2228     // cout<<"Went backwards, found "<<before<<endl;
2229     // return us to starting point
2230     while (count--)
2231       cursor.next(key, val);
2232   }
2233   //  cout<<"Now going forward"<<endl;
2234   for (int count = 0;; ++count) {
2235     if ((count && cursor.next(key, val)) || co.getDomainID(key.getNoStripHeader<StringView>()) != id) {
2236       // cout <<"Hit end of database or zone, finding first hash then in zone "<<id<<endl;
2237       if (cursor.lower_bound(co(id), key, val)) {
2238         // cout<<"hit end of zone find when we shouldn't"<<endl;
2239         // means database is wrong, nothing we can do
2240         return false;
2241       }
2242       for (;;) {
2243         if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2244           serFromString(val.get<StringView>(), lrr);
2245           if (!lrr.ttl)
2246             break;
2247         }
2248
2249         if (cursor.next(key, val)) {
2250           // means database is wrong, nothing we can do
2251           // cout<<"hit end of zone when we shouldn't 2"<<endl;
2252           return false;
2253         }
2254         // cout << "Next.. "<<endl;
2255       }
2256       after = co.getQName(key.getNoStripHeader<StringView>());
2257
2258       // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2259       return true;
2260     }
2261
2262     // cout<<"After "<<co.getQName(key.get<StringView>()) <<endl;
2263     if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2264       serFromString(val.get<StringView>(), lrr);
2265       if (!lrr.ttl) {
2266         break;
2267       }
2268     }
2269   }
2270   after = co.getQName(key.getNoStripHeader<StringView>());
2271   // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2272   return true;
2273 }
2274
2275 bool LMDBBackend::getBeforeAndAfterNames(uint32_t id, const DNSName& zonenameU, const DNSName& qname, DNSName& before, DNSName& after)
2276 {
2277   DNSName zonename = zonenameU.makeLowerCase();
2278   //  cout << __PRETTY_FUNCTION__<< ": "<<id <<", "<<zonename << ", '"<<qname<<"'"<<endl;
2279
2280   auto txn = getRecordsROTransaction(id);
2281   compoundOrdername co;
2282   DNSName qname2 = qname.makeRelative(zonename);
2283   string matchkey = co(id, qname2);
2284   auto cursor = txn->txn->getCursor(txn->db->dbi);
2285   MDBOutVal key, val;
2286   // cout<<"Lower_bound for "<<qname2<<endl;
2287   if (cursor.lower_bound(matchkey, key, val)) {
2288     // cout << "Hit end of database, bummer"<<endl;
2289     cursor.last(key, val);
2290     if (co.getDomainID(key.getNoStripHeader<string_view>()) == id) {
2291       before = co.getQName(key.getNoStripHeader<string_view>()) + zonename;
2292       after = zonename;
2293     }
2294     // else
2295     // cout << "We were at end of database, but this zone is not there?!"<<endl;
2296     return true;
2297   }
2298   // cout<<"Cursor is at "<<co.getQName(key.get<string_view>()) <<", in zone id "<<co.getDomainID(key.get<string_view>())<< endl;
2299
2300   if (co.getQType(key.getNoStripHeader<string_view>()).getCode() && co.getDomainID(key.getNoStripHeader<string_view>()) == id && co.getQName(key.getNoStripHeader<string_view>()) == qname2) { // don't match ENTs
2301     // cout << "Had an exact match!"<<endl;
2302     before = qname2 + zonename;
2303     int rc;
2304     for (;;) {
2305       rc = cursor.next(key, val);
2306       if (rc)
2307         break;
2308
2309       if (co.getDomainID(key.getNoStripHeader<string_view>()) == id && key.getNoStripHeader<StringView>().rfind(matchkey, 0) == 0)
2310         continue;
2311       LMDBResourceRecord lrr;
2312       serFromString(val.get<StringView>(), lrr);
2313       if (co.getQType(key.getNoStripHeader<string_view>()).getCode() && (lrr.auth || co.getQType(key.getNoStripHeader<string_view>()).getCode() == QType::NS))
2314         break;
2315     }
2316     if (rc || co.getDomainID(key.getNoStripHeader<string_view>()) != id) {
2317       // cout << "We hit the end of the zone or database. 'after' is apex" << endl;
2318       after = zonename;
2319       return false;
2320     }
2321     after = co.getQName(key.getNoStripHeader<string_view>()) + zonename;
2322     return true;
2323   }
2324
2325   if (co.getDomainID(key.getNoStripHeader<string_view>()) != id) {
2326     // cout << "Ended up in next zone, 'after' is zonename" <<endl;
2327     after = zonename;
2328     // cout << "Now hunting for previous" << endl;
2329     int rc;
2330     for (;;) {
2331       rc = cursor.prev(key, val);
2332       if (rc) {
2333         // cout<<"Reversed into zone, but got not found from lmdb" <<endl;
2334         return false;
2335       }
2336
2337       if (co.getDomainID(key.getNoStripHeader<string_view>()) != id) {
2338         // cout<<"Reversed into zone, but found wrong zone id " << co.getDomainID(key.getNoStripHeader<string_view>()) << " != "<<id<<endl;
2339         // "this can't happen"
2340         return false;
2341       }
2342       LMDBResourceRecord lrr;
2343       serFromString(val.get<StringView>(), lrr);
2344       if (co.getQType(key.getNoStripHeader<string_view>()).getCode() && (lrr.auth || co.getQType(key.getNoStripHeader<string_view>()).getCode() == QType::NS))
2345         break;
2346     }
2347
2348     before = co.getQName(key.getNoStripHeader<string_view>()) + zonename;
2349     // cout<<"Found: "<< before<<endl;
2350     return true;
2351   }
2352
2353   // cout <<"We ended up after "<<qname<<", on "<<co.getQName(key.getNoStripHeader<string_view>())<<endl;
2354
2355   int skips = 0;
2356   for (;;) {
2357     LMDBResourceRecord lrr;
2358     serFromString(val.get<StringView>(), lrr);
2359     if (co.getQType(key.getNoStripHeader<string_view>()).getCode() && (lrr.auth || co.getQType(key.getNoStripHeader<string_view>()).getCode() == QType::NS)) {
2360       after = co.getQName(key.getNoStripHeader<string_view>()) + zonename;
2361       // cout <<"Found auth ("<<lrr.auth<<") or an NS record "<<after<<", type: "<<co.getQType(key.getNoStripHeader<string_view>()).toString()<<", ttl = "<<lrr.ttl<<endl;
2362       // cout << makeHexDump(val.get<string>()) << endl;
2363       break;
2364     }
2365     // cout <<"  oops, " << co.getQName(key.getNoStripHeader<string_view>()) << " was not auth "<<lrr.auth<< " type=" << lrr.qtype.toString()<<" or NS, so need to skip ahead a bit more" << endl;
2366     int rc = cursor.next(key, val);
2367     if (!rc)
2368       ++skips;
2369     if (rc || co.getDomainID(key.getNoStripHeader<string_view>()) != id) {
2370       // cout << "  oops, hit end of database or zone. This means after is apex" <<endl;
2371       after = zonename;
2372       break;
2373     }
2374   }
2375   // go back to where we were
2376   while (skips--)
2377     cursor.prev(key, val);
2378
2379   for (;;) {
2380     int rc = cursor.prev(key, val);
2381     if (rc || co.getDomainID(key.getNoStripHeader<string_view>()) != id) {
2382       // XX I don't think this case can happen
2383       // cout << "We hit the beginning of the zone or database.. now what" << endl;
2384       return false;
2385     }
2386     before = co.getQName(key.getNoStripHeader<string_view>()) + zonename;
2387     LMDBResourceRecord lrr;
2388     serFromString(val.get<string_view>(), lrr);
2389     // cout<<"And before to "<<before<<", auth = "<<rr.auth<<endl;
2390     if (co.getQType(key.getNoStripHeader<string_view>()).getCode() && (lrr.auth || co.getQType(key.getNoStripHeader<string_view>()) == QType::NS))
2391       break;
2392     // cout << "Oops, that was wrong, go back one more"<<endl;
2393   }
2394
2395   return true;
2396 }
2397
2398 bool LMDBBackend::updateDNSSECOrderNameAndAuth(uint32_t domain_id, const DNSName& qname, const DNSName& ordername, bool auth, const uint16_t qtype)
2399 {
2400   //  cout << __PRETTY_FUNCTION__<< ": "<< domain_id <<", '"<<qname <<"', '"<<ordername<<"', "<<auth<< ", " << qtype << endl;
2401   shared_ptr<RecordsRWTransaction> txn;
2402   bool needCommit = false;
2403   if (d_rwtxn && d_transactiondomainid == domain_id) {
2404     txn = d_rwtxn;
2405     //    cout<<"Reusing open transaction"<<endl;
2406   }
2407   else {
2408     //    cout<<"Making a new RW txn for " << __PRETTY_FUNCTION__ <<endl;
2409     txn = getRecordsRWTransaction(domain_id);
2410     needCommit = true;
2411   }
2412
2413   DomainInfo di;
2414   if (!d_tdomains->getROTransaction().get(domain_id, di)) {
2415     //    cout<<"Could not find domain_id "<<domain_id <<endl;
2416     return false;
2417   }
2418
2419   DNSName rel = qname.makeRelative(di.zone);
2420
2421   compoundOrdername co;
2422   string matchkey = co(domain_id, rel);
2423
2424   auto cursor = txn->txn->getCursor(txn->db->dbi);
2425   MDBOutVal key, val;
2426   if (cursor.lower_bound(matchkey, key, val)) {
2427     // cout << "Could not find anything"<<endl;
2428     return false;
2429   }
2430
2431   bool hasOrderName = !ordername.empty();
2432   bool needNSEC3 = hasOrderName;
2433
2434   for (; key.getNoStripHeader<StringView>().rfind(matchkey, 0) == 0;) {
2435     vector<LMDBResourceRecord> lrrs;
2436
2437     if (co.getQType(key.getNoStripHeader<StringView>()) != QType::NSEC3) {
2438       serFromString(val.get<StringView>(), lrrs);
2439       bool changed = false;
2440       vector<LMDBResourceRecord> newRRs;
2441       for (auto& lrr : lrrs) {
2442         lrr.qtype = co.getQType(key.getNoStripHeader<StringView>());
2443         if (!needNSEC3 && qtype != QType::ANY) {
2444           needNSEC3 = (lrr.ordername && QType(qtype) != lrr.qtype);
2445         }
2446
2447         if ((qtype == QType::ANY || QType(qtype) == lrr.qtype) && (lrr.ordername != hasOrderName || lrr.auth != auth)) {
2448           lrr.auth = auth;
2449           lrr.ordername = hasOrderName;
2450           changed = true;
2451         }
2452         newRRs.push_back(std::move(lrr));
2453       }
2454       if (changed) {
2455         cursor.put(key, serToString(newRRs));
2456       }
2457     }
2458
2459     if (cursor.next(key, val))
2460       break;
2461   }
2462
2463   bool del = false;
2464   LMDBResourceRecord lrr;
2465   matchkey = co(domain_id, rel, QType::NSEC3);
2466   // cerr<<"here qname="<<qname<<" ordername="<<ordername<<" qtype="<<qtype<<" matchkey="<<makeHexDump(matchkey)<<endl;
2467   int txngetrc;
2468   if (!(txngetrc = txn->txn->get(txn->db->dbi, matchkey, val))) {
2469     serFromString(val.get<string_view>(), lrr);
2470
2471     if (needNSEC3) {
2472       if (hasOrderName && lrr.content != ordername.toDNSStringLC()) {
2473         del = true;
2474       }
2475     }
2476     else {
2477       del = true;
2478     }
2479     if (del) {
2480       txn->txn->del(txn->db->dbi, co(domain_id, DNSName(lrr.content.c_str(), lrr.content.size(), 0, false), QType::NSEC3));
2481       txn->txn->del(txn->db->dbi, matchkey);
2482     }
2483   }
2484   else {
2485     del = true;
2486   }
2487
2488   if (hasOrderName && del) {
2489     matchkey = co(domain_id, rel, QType::NSEC3);
2490
2491     lrr.ttl = 0;
2492     lrr.auth = 0;
2493     lrr.content = rel.toDNSStringLC();
2494
2495     string str = serToString(lrr);
2496     txn->txn->put(txn->db->dbi, co(domain_id, ordername, QType::NSEC3), str);
2497     lrr.ttl = 1;
2498     lrr.content = ordername.toDNSStringLC();
2499     str = serToString(lrr);
2500     txn->txn->put(txn->db->dbi, matchkey, str); // 2
2501   }
2502
2503   if (needCommit)
2504     txn->txn->commit();
2505   return false;
2506 }
2507
2508 bool LMDBBackend::updateEmptyNonTerminals(uint32_t domain_id, set<DNSName>& insert, set<DNSName>& erase, bool remove)
2509 {
2510   // cout << __PRETTY_FUNCTION__<< ": "<< domain_id << ", insert.size() "<<insert.size()<<", "<<erase.size()<<", " <<remove<<endl;
2511
2512   bool needCommit = false;
2513   shared_ptr<RecordsRWTransaction> txn;
2514   if (d_rwtxn && d_transactiondomainid == domain_id) {
2515     txn = d_rwtxn;
2516     //    cout<<"Reusing open transaction"<<endl;
2517   }
2518   else {
2519     //    cout<<"Making a new RW txn for delete domain"<<endl;
2520     txn = getRecordsRWTransaction(domain_id);
2521     needCommit = true;
2522   }
2523
2524   // if remove is set, all ENTs should be removed & nothing else should be done
2525   if (remove) {
2526     deleteDomainRecords(*txn, domain_id, 0);
2527   }
2528   else {
2529     DomainInfo di;
2530     auto rotxn = d_tdomains->getROTransaction();
2531     if (!rotxn.get(domain_id, di)) {
2532       // cout <<"No such domain with id "<<domain_id<<endl;
2533       return false;
2534     }
2535     compoundOrdername co;
2536     for (const auto& n : insert) {
2537       LMDBResourceRecord lrr;
2538       lrr.qname = n.makeRelative(di.zone);
2539       lrr.ttl = 0;
2540       lrr.auth = true;
2541
2542       std::string ser = serToString(lrr);
2543
2544       txn->txn->put(txn->db->dbi, co(domain_id, lrr.qname, 0), ser);
2545
2546       // cout <<" +"<<n<<endl;
2547     }
2548     for (auto n : erase) {
2549       // cout <<" -"<<n<<endl;
2550       n.makeUsRelative(di.zone);
2551       txn->txn->del(txn->db->dbi, co(domain_id, n, 0));
2552     }
2553   }
2554   if (needCommit)
2555     txn->txn->commit();
2556   return false;
2557 }
2558
2559 /* TSIG */
2560 bool LMDBBackend::getTSIGKey(const DNSName& name, DNSName& algorithm, string& content)
2561 {
2562   auto txn = d_ttsig->getROTransaction();
2563   LMDBIDvec ids;
2564   txn.get_multi<0>(name, ids);
2565
2566   TSIGKey key;
2567   for (auto id : ids) {
2568     if (txn.get(id, key)) {
2569       if (algorithm.empty() || algorithm == DNSName(key.algorithm)) {
2570         algorithm = DNSName(key.algorithm);
2571         content = key.key;
2572       }
2573     }
2574   }
2575
2576   return true;
2577 }
2578
2579 // this deletes an old key if it has the same algorithm
2580 bool LMDBBackend::setTSIGKey(const DNSName& name, const DNSName& algorithm, const string& content)
2581 {
2582   auto txn = d_ttsig->getRWTransaction();
2583
2584   LMDBIDvec ids;
2585   txn.get_multi<0>(name, ids);
2586
2587   TSIGKey key;
2588   for (auto id : ids) {
2589     if (txn.get(id, key)) {
2590       if (key.algorithm == algorithm) {
2591         txn.del(id);
2592       }
2593     }
2594   }
2595
2596   TSIGKey tk;
2597   tk.name = name;
2598   tk.algorithm = algorithm;
2599   tk.key = content;
2600
2601   txn.put(tk, 0, d_random_ids);
2602   txn.commit();
2603
2604   return true;
2605 }
2606 bool LMDBBackend::deleteTSIGKey(const DNSName& name)
2607 {
2608   auto txn = d_ttsig->getRWTransaction();
2609
2610   LMDBIDvec ids;
2611   txn.get_multi<0>(name, ids);
2612
2613   TSIGKey key;
2614
2615   for (auto id : ids) {
2616     if (txn.get(id, key)) {
2617       txn.del(id);
2618     }
2619   }
2620   txn.commit();
2621   return true;
2622 }
2623 bool LMDBBackend::getTSIGKeys(std::vector<struct TSIGKey>& keys)
2624 {
2625   auto txn = d_ttsig->getROTransaction();
2626
2627   keys.clear();
2628   for (auto iter = txn.begin(); iter != txn.end(); ++iter) {
2629     keys.push_back(*iter);
2630   }
2631   return true;
2632 }
2633
2634 string LMDBBackend::directBackendCmd(const string& query)
2635 {
2636   ostringstream ret, usage;
2637
2638   usage << "info                               show some information about the database" << endl;
2639   usage << "index check domains                check zone<>ID indexes" << endl;
2640   usage << "index refresh domains <ID>         refresh index for zone with this ID" << endl;
2641   usage << "index refresh-all domains          refresh index for all zones with disconnected indexes" << endl;
2642   vector<string> argv;
2643   stringtok(argv, query);
2644
2645   if (argv.empty()) {
2646     return usage.str();
2647   }
2648
2649   string& cmd = argv[0];
2650
2651   if (cmd == "help") {
2652     return usage.str();
2653   }
2654
2655   if (cmd == "info") {
2656     ret << "shards: " << s_shards << endl;
2657     ret << "schemaversion: " << SCHEMAVERSION << endl;
2658
2659     return ret.str();
2660   }
2661
2662   if (cmd == "index") {
2663     if (argv.size() < 2) {
2664       return "need an index subcommand\n";
2665     }
2666
2667     string& subcmd = argv[1];
2668
2669     if (subcmd == "check" || subcmd == "refresh-all") {
2670       bool refresh = false;
2671
2672       if (subcmd == "refresh-all") {
2673         refresh = true;
2674       }
2675
2676       if (argv.size() < 3) {
2677         return "need an index name\n";
2678       }
2679
2680       if (argv[2] != "domains") {
2681         return "can only check the domains index\n";
2682       }
2683
2684       vector<uint32_t> refreshQueue;
2685
2686       {
2687         auto txn = d_tdomains->getROTransaction();
2688
2689         for (auto iter = txn.begin(); iter != txn.end(); ++iter) {
2690           DomainInfo di = *iter;
2691
2692           auto id = iter.getID();
2693
2694           LMDBIDvec ids;
2695           txn.get_multi<0>(di.zone, ids);
2696
2697           if (ids.size() != 1) {
2698             ret << "ID->zone index has " << id << "->" << di.zone << ", ";
2699
2700             if (ids.empty()) {
2701               ret << "zone->ID index has no entry for " << di.zone << endl;
2702               if (refresh) {
2703                 refreshQueue.push_back(id);
2704               }
2705               else {
2706                 ret << "  suggested remedy: index refresh domains " << id << endl;
2707               }
2708             }
2709             else {
2710               // ids.size() > 1
2711               ret << "zone->ID index has multiple entries for " << di.zone << ": ";
2712               for (auto id_ : ids) {
2713                 ret << id_ << " ";
2714               }
2715               ret << endl;
2716             }
2717           }
2718         }
2719       }
2720
2721       if (refresh) {
2722         for (const auto& id : refreshQueue) {
2723           if (genChangeDomain(id, [](DomainInfo& /* di */) {})) {
2724             ret << "refreshed " << id << endl;
2725           }
2726           else {
2727             ret << "failed to refresh " << id << endl;
2728           }
2729         }
2730       }
2731       return ret.str();
2732     }
2733     if (subcmd == "refresh") {
2734       // index refresh domains 12345
2735       if (argv.size() < 4) {
2736         return "usage: index refresh domains <ID>\n";
2737       }
2738
2739       if (argv[2] != "domains") {
2740         return "can only refresh in the domains index\n";
2741       }
2742
2743       uint32_t id = 0;
2744
2745       try {
2746         id = pdns::checked_stoi<uint32_t>(argv[3]);
2747       }
2748       catch (const std::out_of_range& e) {
2749         return "ID out of range\n";
2750       }
2751
2752       if (genChangeDomain(id, [](DomainInfo& /* di */) {})) {
2753         ret << "refreshed" << endl;
2754       }
2755       else {
2756         ret << "failed" << endl;
2757       }
2758       return ret.str();
2759     }
2760   }
2761
2762   return "unknown lmdbbackend command\n";
2763 }
2764
2765 class LMDBFactory : public BackendFactory
2766 {
2767 public:
2768   LMDBFactory() :
2769     BackendFactory("lmdb") {}
2770   void declareArguments(const string& suffix = "") override
2771   {
2772     declare(suffix, "filename", "Filename for lmdb", "./pdns.lmdb");
2773     declare(suffix, "sync-mode", "Synchronisation mode: nosync, nometasync, sync", "sync");
2774     // there just is no room for more on 32 bit
2775     declare(suffix, "shards", "Records database will be split into this number of shards", (sizeof(void*) == 4) ? "2" : "64");
2776     declare(suffix, "schema-version", "Maximum allowed schema version to run on this DB. If a lower version is found, auto update is performed", std::to_string(SCHEMAVERSION));
2777     declare(suffix, "random-ids", "Numeric IDs inside the database are generated randomly instead of sequentially", "no");
2778     declare(suffix, "map-size", "LMDB map size in megabytes", (sizeof(void*) == 4) ? "100" : "16000");
2779     declare(suffix, "flag-deleted", "Flag entries on deletion instead of deleting them", "no");
2780     declare(suffix, "lightning-stream", "Run in Lightning Stream compatible mode", "no");
2781   }
2782   DNSBackend* make(const string& suffix = "") override
2783   {
2784     return new LMDBBackend(suffix);
2785   }
2786 };
2787
2788 /* THIRD PART */
2789
2790 class LMDBLoader
2791 {
2792 public:
2793   LMDBLoader()
2794   {
2795     BackendMakers().report(new LMDBFactory);
2796     g_log << Logger::Info << "[lmdbbackend] This is the lmdb backend version " VERSION
2797 #ifndef REPRODUCIBLE
2798           << " (" __DATE__ " " __TIME__ ")"
2799 #endif
2800           << " reporting" << endl;
2801   }
2802 };
2803
2804 static LMDBLoader randomLoader;