modules/lmdbbackend/lmdbbackend.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22
  23 #include "ext/lmdb-safe/lmdb-safe.hh"
  24 #include <lmdb.h>
  25 #include <stdexcept>
  26 #include <utility>
  27 #ifdef HAVE_CONFIG_H
  28 #include "config.h"
  29 #endif
  30 #include "pdns/utility.hh"
  31 #include "pdns/dnsbackend.hh"
  32 #include "pdns/dns.hh"
  33 #include "pdns/dnspacket.hh"
  34 #include "pdns/base32.hh"
  35 #include "pdns/dnssecinfra.hh"
  36 #include "pdns/pdnsexception.hh"
  37 #include "pdns/logger.hh"
  38 #include "pdns/version.hh"
  39 #include "pdns/arguments.hh"
  40 #include "pdns/lock.hh"
  41 #include "pdns/uuid-utils.hh"
  42 #include <boost/archive/binary_oarchive.hpp>
  43 #include <boost/archive/binary_iarchive.hpp>
  44 #include <boost/serialization/vector.hpp>
  45 #include <boost/serialization/string.hpp>
  46 #include <boost/serialization/utility.hpp>
  47 #include <boost/uuid/uuid_serialize.hpp>
  48
  49 #include <boost/iostreams/device/back_inserter.hpp>
  50
  51 #include <stdio.h>
  52 #include <unistd.h>
  53
  54 #include "lmdbbackend.hh"
  55
  56 #define SCHEMAVERSION 5
  57
  58 // List the class version here. Default is 0
  59 BOOST_CLASS_VERSION(LMDBBackend::KeyDataDB, 1)
  60 BOOST_CLASS_VERSION(DomainInfo, 1)
  61
  62 static bool s_first = true;
  63 static int s_shards = 0;
  64 static std::mutex s_lmdbStartupLock;
  65
  66 std::pair<uint32_t, uint32_t> LMDBBackend::getSchemaVersionAndShards(std::string& filename)
  67 {
  68   // cerr << "getting schema version for path " << filename << endl;
  69
  70   uint32_t schemaversion;
  71
  72   int rc;
  73   MDB_env* env = nullptr;
  74
  75   if ((rc = mdb_env_create(&env)) != 0) {
  76     throw std::runtime_error("mdb_env_create failed");
  77   }
  78
  79   if ((rc = mdb_env_set_mapsize(env, 0)) != 0) {
  80     throw std::runtime_error("mdb_env_set_mapsize failed");
  81   }
  82
  83   if ((rc = mdb_env_set_maxdbs(env, 20)) != 0) { // we need 17: 1 {"pdns"} + 4 {"domains", "keydata", "tsig", "metadata"} * 2 {v4, v5} * 2 {main, index in _0}
  84     mdb_env_close(env);
  85     throw std::runtime_error("mdb_env_set_maxdbs failed");
  86   }
  87
  88   if ((rc = mdb_env_open(env, filename.c_str(), MDB_NOSUBDIR | MDB_RDONLY, 0600)) != 0) {
  89     if (rc == ENOENT) {
  90       // we don't have a database yet! report schema 0, with 0 shards
  91       return {0u, 0u};
  92     }
  93     mdb_env_close(env);
  94     throw std::runtime_error("mdb_env_open failed");
  95   }
  96
  97   MDB_txn* txn = nullptr;
  98
  99   if ((rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)) != 0) {
 100     mdb_env_close(env);
 101     throw std::runtime_error("mdb_txn_begin failed");
 102   }
 103
 104   MDB_dbi dbi;
 105
 106   if ((rc = mdb_dbi_open(txn, "pdns", 0, &dbi)) != 0) {
 107     if (rc == MDB_NOTFOUND) {
 108       // this means nothing has been inited yet
 109       // we pretend this means 5
 110       mdb_txn_abort(txn);
 111       mdb_env_close(env);
 112       return {5u, 0u};
 113     }
 114     mdb_txn_abort(txn);
 115     mdb_env_close(env);
 116     throw std::runtime_error("mdb_dbi_open failed");
 117   }
 118
 119   MDB_val key, data;
 120
 121   key.mv_data = (char*)"schemaversion";
 122   key.mv_size = strlen((char*)key.mv_data);
 123
 124   if ((rc = mdb_get(txn, dbi, &key, &data)) != 0) {
 125     if (rc == MDB_NOTFOUND) {
 126       // this means nothing has been inited yet
 127       // we pretend this means 5
 128       mdb_txn_abort(txn);
 129       mdb_env_close(env);
 130       return {5u, 0u};
 131     }
 132
 133     throw std::runtime_error("mdb_get pdns.schemaversion failed");
 134   }
 135
 136   if (data.mv_size == 4) {
 137     // schemaversion is < 5 and is stored in 32 bits, in host order
 138
 139     memcpy(&schemaversion, data.mv_data, data.mv_size);
 140   }
 141   else if (data.mv_size >= LMDBLS::LS_MIN_HEADER_SIZE + sizeof(schemaversion)) {
 142     // schemaversion presumably is 5, stored in 32 bits, network order, after the LS header
 143
 144     // FIXME: get actual header size (including extension blocks) instead of just reading from the back
 145     // FIXME: add a test for reading schemaversion and shards (and actual data, later) when there are variably sized headers
 146     memcpy(&schemaversion, (char*)data.mv_data + data.mv_size - sizeof(schemaversion), sizeof(schemaversion));
 147     schemaversion = ntohl(schemaversion);
 148   }
 149   else {
 150     throw std::runtime_error("pdns.schemaversion had unexpected size");
 151   }
 152
 153   uint32_t shards;
 154
 155   key.mv_data = (char*)"shards";
 156   key.mv_size = strlen((char*)key.mv_data);
 157
 158   if ((rc = mdb_get(txn, dbi, &key, &data)) != 0) {
 159     if (rc == MDB_NOTFOUND) {
 160       cerr << "schemaversion was set, but shards was not. Dazed and confused, trying to exit." << endl;
 161       mdb_txn_abort(txn);
 162       mdb_env_close(env);
 163       exit(1);
 164     }
 165
 166     throw std::runtime_error("mdb_get pdns.shards failed");
 167   }
 168
 169   if (data.mv_size == 4) {
 170     // 'shards' is stored in 32 bits, in host order
 171
 172     memcpy(&shards, data.mv_data, data.mv_size);
 173   }
 174   else if (data.mv_size >= LMDBLS::LS_MIN_HEADER_SIZE + sizeof(shards)) {
 175     // FIXME: get actual header size (including extension blocks) instead of just reading from the back
 176     memcpy(&shards, (char*)data.mv_data + data.mv_size - sizeof(shards), sizeof(shards));
 177     shards = ntohl(shards);
 178   }
 179   else {
 180     throw std::runtime_error("pdns.shards had unexpected size");
 181   }
 182
 183   mdb_txn_abort(txn);
 184   mdb_env_close(env);
 185
 186   return {schemaversion, shards};
 187 }
 188
 189 namespace
 190 {
 191 // copy sdbi to tdbi, prepending an empty LS header (24 bytes of '\0') to all values
 192 void copyDBIAndAddLSHeader(MDB_txn* txn, MDB_dbi sdbi, MDB_dbi tdbi)
 193 {
 194   // FIXME: clear out target dbi first
 195
 196   std::string header(LMDBLS::LS_MIN_HEADER_SIZE, '\0');
 197   int rc;
 198
 199   MDB_cursor* cur;
 200
 201   if ((rc = mdb_cursor_open(txn, sdbi, &cur)) != 0) {
 202     throw std::runtime_error("mdb_cursur_open failed");
 203   }
 204
 205   MDB_val key, data;
 206
 207   rc = mdb_cursor_get(cur, &key, &data, MDB_FIRST);
 208
 209   while (rc == 0) {
 210     std::string skey(reinterpret_cast<const char*>(key.mv_data), key.mv_size);
 211     std::string sdata(reinterpret_cast<const char*>(data.mv_data), data.mv_size);
 212
 213     std::string stdata = header + sdata;
 214
 215     // cerr<<"got key="<<makeHexDump(skey)<<", data="<<makeHexDump(sdata)<<", sdata="<<makeHexDump(stdata)<<endl;
 216
 217     MDB_val tkey;
 218     MDB_val tdata;
 219
 220     tkey.mv_data = const_cast<char*>(skey.c_str());
 221     tkey.mv_size = skey.size();
 222     tdata.mv_data = const_cast<char*>(stdata.c_str());
 223     tdata.mv_size = stdata.size();
 224
 225     if ((rc = mdb_put(txn, tdbi, &tkey, &tdata, 0)) != 0) {
 226       throw std::runtime_error("mdb_put failed");
 227     }
 228
 229     rc = mdb_cursor_get(cur, &key, &data, MDB_NEXT);
 230   }
 231   if (rc != MDB_NOTFOUND) {
 232     cerr << "rc=" << rc << endl;
 233     throw std::runtime_error("error while iterating dbi");
 234   }
 235 }
 236
 237 // migrated a typed DBI:
 238 // 1. change keys (uint32_t) from host to network order
 239 // 2. prepend empty LS header to values
 240 void copyTypedDBI(MDB_txn* txn, MDB_dbi sdbi, MDB_dbi tdbi)
 241 {
 242   // FIXME: clear out target dbi first
 243
 244   std::string header(LMDBLS::LS_MIN_HEADER_SIZE, '\0');
 245   int rc;
 246
 247   MDB_cursor* cur;
 248
 249   if ((rc = mdb_cursor_open(txn, sdbi, &cur)) != 0) {
 250     throw std::runtime_error("mdb_cursur_open failed");
 251   }
 252
 253   MDB_val key, data;
 254
 255   rc = mdb_cursor_get(cur, &key, &data, MDB_FIRST);
 256
 257   while (rc == 0) {
 258     // std::string skey((char*) key.mv_data, key.mv_size);
 259     std::string sdata(reinterpret_cast<const char*>(data.mv_data), data.mv_size);
 260
 261     std::string stdata = header + sdata;
 262
 263     uint32_t id;
 264
 265     if (key.mv_size != sizeof(uint32_t)) {
 266       throw std::runtime_error("got non-uint32_t key in TypedDBI");
 267     }
 268
 269     memcpy(&id, key.mv_data, sizeof(uint32_t));
 270
 271     id = htonl(id);
 272
 273     // cerr<<"got key="<<makeHexDump(skey)<<", data="<<makeHexDump(sdata)<<", sdata="<<makeHexDump(stdata)<<endl;
 274
 275     MDB_val tkey;
 276     MDB_val tdata;
 277
 278     tkey.mv_data = reinterpret_cast<char*>(&id);
 279     tkey.mv_size = sizeof(uint32_t);
 280     tdata.mv_data = const_cast<char*>(stdata.c_str());
 281     tdata.mv_size = stdata.size();
 282
 283     if ((rc = mdb_put(txn, tdbi, &tkey, &tdata, 0)) != 0) {
 284       throw std::runtime_error("mdb_put failed");
 285     }
 286
 287     rc = mdb_cursor_get(cur, &key, &data, MDB_NEXT);
 288   }
 289   if (rc != MDB_NOTFOUND) {
 290     cerr << "rc=" << rc << endl;
 291     throw std::runtime_error("error while iterating dbi");
 292   }
 293 }
 294
 295 // migrating an index DBI:
 296 // newkey = oldkey.len(), oldkey, htonl(oldvalue)
 297 // newvalue = empty lsheader
 298 void copyIndexDBI(MDB_txn* txn, MDB_dbi sdbi, MDB_dbi tdbi)
 299 {
 300   // FIXME: clear out target dbi first
 301
 302   std::string header(LMDBLS::LS_MIN_HEADER_SIZE, '\0');
 303   int rc;
 304
 305   MDB_cursor* cur;
 306
 307   if ((rc = mdb_cursor_open(txn, sdbi, &cur)) != 0) {
 308     throw std::runtime_error("mdb_cursur_open failed");
 309   }
 310
 311   MDB_val key, data;
 312
 313   rc = mdb_cursor_get(cur, &key, &data, MDB_FIRST);
 314
 315   while (rc == 0) {
 316     std::string lenprefix(sizeof(uint16_t), '\0');
 317     std::string skey((char*)key.mv_data, key.mv_size);
 318
 319     uint32_t id;
 320
 321     if (data.mv_size != sizeof(uint32_t)) {
 322       throw std::runtime_error("got non-uint32_t ID value in IndexDBI");
 323     }
 324
 325     memcpy((void*)&id, data.mv_data, sizeof(uint32_t));
 326     id = htonl(id);
 327
 328     uint16_t len = htons(skey.size());
 329     memcpy((void*)lenprefix.data(), &len, sizeof(len));
 330     std::string stkey = lenprefix + skey + std::string((char*)&id, sizeof(uint32_t));
 331
 332     MDB_val tkey;
 333     MDB_val tdata;
 334
 335     tkey.mv_data = (char*)stkey.c_str();
 336     tkey.mv_size = stkey.size();
 337     tdata.mv_data = (char*)header.c_str();
 338     tdata.mv_size = header.size();
 339
 340     if ((rc = mdb_put(txn, tdbi, &tkey, &tdata, 0)) != 0) {
 341       throw std::runtime_error("mdb_put failed");
 342     }
 343
 344     rc = mdb_cursor_get(cur, &key, &data, MDB_NEXT);
 345   }
 346   if (rc != MDB_NOTFOUND) {
 347     throw std::runtime_error("error while iterating dbi");
 348   }
 349 }
 350
 351 }
 352
 353 bool LMDBBackend::upgradeToSchemav5(std::string& filename)
 354 {
 355   int rc;
 356
 357   auto currentSchemaVersionAndShards = getSchemaVersionAndShards(filename);
 358   uint32_t currentSchemaVersion = currentSchemaVersionAndShards.first;
 359   uint32_t shards = currentSchemaVersionAndShards.second;
 360
 361   if (currentSchemaVersion != 3 && currentSchemaVersion != 4) {
 362     throw std::runtime_error("upgrade to v5 requested but current schema is not v3 or v4, stopping");
 363   }
 364
 365   MDB_env* env = nullptr;
 366
 367   if ((rc = mdb_env_create(&env)) != 0) {
 368     throw std::runtime_error("mdb_env_create failed");
 369   }
 370
 371   if ((rc = mdb_env_set_maxdbs(env, 20)) != 0) {
 372     mdb_env_close(env);
 373     throw std::runtime_error("mdb_env_set_maxdbs failed");
 374   }
 375
 376   if ((rc = mdb_env_open(env, filename.c_str(), MDB_NOSUBDIR, 0600)) != 0) {
 377     mdb_env_close(env);
 378     throw std::runtime_error("mdb_env_open failed");
 379   }
 380
 381   MDB_txn* txn = nullptr;
 382
 383   if ((rc = mdb_txn_begin(env, NULL, 0, &txn)) != 0) {
 384     mdb_env_close(env);
 385     throw std::runtime_error("mdb_txn_begin failed");
 386   }
 387
 388   std::cerr << "migrating shards" << std::endl;
 389   for (uint32_t i = 0; i < shards; i++) {
 390     string shardfile = filename + "-" + std::to_string(i);
 391     if (access(shardfile.c_str(), F_OK) < 0) {
 392       if (errno == ENOENT) {
 393         // apparently this shard doesn't exist yet, moving on
 394         std::cerr << "shard " << shardfile << " not found, continuing" << std::endl;
 395         continue;
 396       }
 397     }
 398
 399     std::cerr << "migrating shard " << shardfile << std::endl;
 400     MDB_env* shenv = nullptr;
 401
 402     if ((rc = mdb_env_create(&shenv)) != 0) {
 403       throw std::runtime_error("mdb_env_create failed");
 404     }
 405
 406     if ((rc = mdb_env_set_maxdbs(shenv, 8)) != 0) {
 407       mdb_env_close(env);
 408       throw std::runtime_error("mdb_env_set_maxdbs failed");
 409     }
 410
 411     if ((rc = mdb_env_open(shenv, shardfile.c_str(), MDB_NOSUBDIR, 0600)) != 0) {
 412       mdb_env_close(env);
 413       throw std::runtime_error("mdb_env_open failed");
 414     }
 415
 416     MDB_txn* shtxn = nullptr;
 417
 418     if ((rc = mdb_txn_begin(shenv, NULL, 0, &shtxn)) != 0) {
 419       mdb_env_close(env);
 420       throw std::runtime_error("mdb_txn_begin failed");
 421     }
 422
 423     MDB_dbi shdbi;
 424
 425     if ((rc = mdb_dbi_open(shtxn, "records", 0, &shdbi)) != 0) {
 426       if (rc == MDB_NOTFOUND) {
 427         mdb_txn_abort(shtxn);
 428         mdb_env_close(shenv);
 429         continue;
 430       }
 431       mdb_txn_abort(shtxn);
 432       mdb_env_close(shenv);
 433       throw std::runtime_error("mdb_dbi_open shard records failed");
 434     }
 435
 436     MDB_dbi shdbi2;
 437
 438     if ((rc = mdb_dbi_open(shtxn, "records_v5", MDB_CREATE, &shdbi2)) != 0) {
 439       mdb_dbi_close(shenv, shdbi);
 440       mdb_txn_abort(shtxn);
 441       mdb_env_close(shenv);
 442       throw std::runtime_error("mdb_dbi_open shard records_v5 failed");
 443     }
 444
 445     try {
 446       copyDBIAndAddLSHeader(shtxn, shdbi, shdbi2);
 447     }
 448     catch (std::exception& e) {
 449       mdb_dbi_close(shenv, shdbi2);
 450       mdb_dbi_close(shenv, shdbi);
 451       mdb_txn_abort(shtxn);
 452       mdb_env_close(shenv);
 453       throw std::runtime_error("copyDBIAndAddLSHeader failed");
 454     }
 455
 456     cerr << "shard mbd_drop=" << mdb_drop(shtxn, shdbi, 1) << endl;
 457     mdb_txn_commit(shtxn);
 458     mdb_dbi_close(shenv, shdbi2);
 459     mdb_env_close(shenv);
 460   }
 461
 462   std::array<MDB_dbi, 4> fromtypeddbi;
 463   std::array<MDB_dbi, 4> totypeddbi;
 464
 465   int index = 0;
 466
 467   for (const std::string dbname : {"domains", "keydata", "tsig", "metadata"}) {
 468     std::cerr << "migrating " << dbname << std::endl;
 469     std::string tdbname = dbname + "_v5";
 470
 471     if ((rc = mdb_dbi_open(txn, dbname.c_str(), 0, &fromtypeddbi[index])) != 0) {
 472       mdb_txn_abort(txn);
 473       mdb_env_close(env);
 474       throw std::runtime_error("mdb_dbi_open typeddbi failed");
 475     }
 476
 477     if ((rc = mdb_dbi_open(txn, tdbname.c_str(), MDB_CREATE, &totypeddbi[index])) != 0) {
 478       mdb_dbi_close(env, fromtypeddbi[index]);
 479       mdb_txn_abort(txn);
 480       mdb_env_close(env);
 481       throw std::runtime_error("mdb_dbi_open typeddbi target failed");
 482     }
 483
 484     try {
 485       copyTypedDBI(txn, fromtypeddbi[index], totypeddbi[index]);
 486     }
 487     catch (std::exception& e) {
 488       mdb_dbi_close(env, totypeddbi[index]);
 489       mdb_dbi_close(env, fromtypeddbi[index]);
 490       mdb_txn_abort(txn);
 491       mdb_env_close(env);
 492       throw std::runtime_error("copyTypedDBI failed");
 493     }
 494
 495     // mdb_dbi_close(env, dbi2);
 496     // mdb_dbi_close(env, dbi);
 497     std::cerr << "migrated " << dbname << std::endl;
 498
 499     index++;
 500   }
 501
 502   std::array<MDB_dbi, 4> fromindexdbi;
 503   std::array<MDB_dbi, 4> toindexdbi;
 504
 505   index = 0;
 506
 507   for (const std::string dbname : {"domains", "keydata", "tsig", "metadata"}) {
 508     std::string fdbname = dbname + "_0";
 509     std::cerr << "migrating " << dbname << std::endl;
 510     std::string tdbname = dbname + "_v5_0";
 511
 512     if ((rc = mdb_dbi_open(txn, fdbname.c_str(), 0, &fromindexdbi[index])) != 0) {
 513       mdb_txn_abort(txn);
 514       mdb_env_close(env);
 515       throw std::runtime_error("mdb_dbi_open indexdbi failed");
 516     }
 517
 518     if ((rc = mdb_dbi_open(txn, tdbname.c_str(), MDB_CREATE, &toindexdbi[index])) != 0) {
 519       mdb_dbi_close(env, fromindexdbi[index]);
 520       mdb_txn_abort(txn);
 521       mdb_env_close(env);
 522       throw std::runtime_error("mdb_dbi_open indexdbi target failed");
 523     }
 524
 525     try {
 526       copyIndexDBI(txn, fromindexdbi[index], toindexdbi[index]);
 527     }
 528     catch (std::exception& e) {
 529       mdb_dbi_close(env, toindexdbi[index]);
 530       mdb_dbi_close(env, fromindexdbi[index]);
 531       mdb_txn_abort(txn);
 532       mdb_env_close(env);
 533       throw std::runtime_error("copyIndexDBI failed");
 534     }
 535
 536     // mdb_dbi_close(env, dbi2);
 537     // mdb_dbi_close(env, dbi);
 538     std::cerr << "migrated " << dbname << std::endl;
 539
 540     index++;
 541   }
 542
 543   MDB_dbi dbi;
 544
 545   // finally, migrate the pdns db
 546   if ((rc = mdb_dbi_open(txn, "pdns", 0, &dbi)) != 0) {
 547     mdb_txn_abort(txn);
 548     mdb_env_close(env);
 549     throw std::runtime_error("mdb_dbi_open pdns failed");
 550   }
 551
 552   MDB_val key, data;
 553
 554   std::string header(LMDBLS::LS_MIN_HEADER_SIZE, '\0');
 555
 556   for (const std::string keyname : {"schemaversion", "shards"}) {
 557     cerr << "migrating pdns." << keyname << endl;
 558
 559     key.mv_data = (char*)keyname.c_str();
 560     key.mv_size = keyname.size();
 561
 562     if ((rc = mdb_get(txn, dbi, &key, &data))) {
 563       throw std::runtime_error("mdb_get pdns.shards failed");
 564     }
 565
 566     uint32_t value;
 567
 568     if (data.mv_size != sizeof(uint32_t)) {
 569       throw std::runtime_error("got non-uint32_t key");
 570     }
 571
 572     memcpy((void*)&value, data.mv_data, sizeof(uint32_t));
 573
 574     value = htonl(value);
 575     if (keyname == "schemaversion") {
 576       value = htonl(5);
 577     }
 578
 579     std::string sdata((char*)data.mv_data, data.mv_size);
 580
 581     std::string stdata = header + std::string((char*)&value, sizeof(uint32_t));
 582     ;
 583
 584     MDB_val tdata;
 585
 586     tdata.mv_data = (char*)stdata.c_str();
 587     tdata.mv_size = stdata.size();
 588
 589     if ((rc = mdb_put(txn, dbi, &key, &tdata, 0)) != 0) {
 590       throw std::runtime_error("mdb_put failed");
 591     }
 592   }
 593
 594   for (const std::string keyname : {"uuid"}) {
 595     cerr << "migrating pdns." << keyname << endl;
 596
 597     key.mv_data = (char*)keyname.c_str();
 598     key.mv_size = keyname.size();
 599
 600     if ((rc = mdb_get(txn, dbi, &key, &data))) {
 601       throw std::runtime_error("mdb_get pdns.shards failed");
 602     }
 603
 604     std::string sdata((char*)data.mv_data, data.mv_size);
 605
 606     std::string stdata = header + sdata;
 607
 608     MDB_val tdata;
 609
 610     tdata.mv_data = (char*)stdata.c_str();
 611     tdata.mv_size = stdata.size();
 612
 613     if ((rc = mdb_put(txn, dbi, &key, &tdata, 0)) != 0) {
 614       throw std::runtime_error("mdb_put failed");
 615     }
 616   }
 617
 618   for (int i = 0; i < 4; i++) {
 619     mdb_drop(txn, fromtypeddbi[i], 1);
 620     mdb_drop(txn, fromindexdbi[i], 1);
 621   }
 622
 623   cerr << "txn commit=" << mdb_txn_commit(txn) << endl;
 624
 625   for (int i = 0; i < 4; i++) {
 626     mdb_dbi_close(env, totypeddbi[i]);
 627     mdb_dbi_close(env, toindexdbi[i]);
 628   }
 629   mdb_env_close(env);
 630
 631   // throw std::runtime_error("migration done");
 632   cerr << "migration done" << endl;
 633   // exit(1);
 634   return true;
 635 }
 636
 637 LMDBBackend::LMDBBackend(const std::string& suffix)
 638 {
 639   // overlapping domain ids in combination with relative names are a recipe for disaster
 640   if (!suffix.empty()) {
 641     throw std::runtime_error("LMDB backend does not support multiple instances");
 642   }
 643
 644   setArgPrefix("lmdb" + suffix);
 645
 646   string syncMode = toLower(getArg("sync-mode"));
 647
 648   d_random_ids = mustDo("random-ids");
 649
 650   if (syncMode == "nosync")
 651     d_asyncFlag = MDB_NOSYNC;
 652   else if (syncMode == "nometasync")
 653     d_asyncFlag = MDB_NOMETASYNC;
 654   else if (syncMode == "mapasync")
 655     d_asyncFlag = MDB_MAPASYNC;
 656   else if (syncMode.empty() || syncMode == "sync")
 657     d_asyncFlag = 0;
 658   else
 659     throw std::runtime_error("Unknown sync mode " + syncMode + " requested for LMDB backend");
 660
 661   uint64_t mapSize = 0;
 662   try {
 663     mapSize = std::stoll(getArg("map-size"));
 664   }
 665   catch (const std::exception& e) {
 666     throw std::runtime_error(std::string("Unable to parse the 'map-size' LMDB value: ") + e.what());
 667   }
 668
 669   LMDBLS::s_flag_deleted = mustDo("flag-deleted");
 670   d_handle_dups = false;
 671
 672   if (mustDo("lightning-stream")) {
 673     d_random_ids = true;
 674     d_handle_dups = true;
 675     LMDBLS::s_flag_deleted = true;
 676
 677     if (atoi(getArg("shards").c_str()) != 1) {
 678       throw std::runtime_error(std::string("running with Lightning Stream support requires shards=1"));
 679     }
 680   }
 681
 682   bool opened = false;
 683
 684   if (s_first) {
 685     std::lock_guard<std::mutex> l(s_lmdbStartupLock);
 686     if (s_first) {
 687       auto filename = getArg("filename");
 688
 689       auto currentSchemaVersionAndShards = getSchemaVersionAndShards(filename);
 690       uint32_t currentSchemaVersion = currentSchemaVersionAndShards.first;
 691       // std::cerr<<"current schema version: "<<currentSchemaVersion<<", shards="<<currentSchemaVersionAndShards.second<<std::endl;
 692
 693       if (getArgAsNum("schema-version") != SCHEMAVERSION) {
 694         throw std::runtime_error("This version of the lmdbbackend only supports schema version 5. Configuration demands a lower version. Not starting up.");
 695       }
 696
 697       if (currentSchemaVersion > 0 && currentSchemaVersion < 3) {
 698         throw std::runtime_error("this version of the lmdbbackend can only upgrade from schema v3/v4 to v5. Upgrading from older schemas is not yet supported.");
 699       }
 700
 701       if (currentSchemaVersion == 0) {
 702         // no database is present yet, we can just create them
 703         currentSchemaVersion = 5;
 704       }
 705
 706       if (currentSchemaVersion == 3 || currentSchemaVersion == 4) {
 707         if (!upgradeToSchemav5(filename)) {
 708           throw std::runtime_error("Failed to perform LMDB schema version upgrade from v4 to v5");
 709         }
 710         currentSchemaVersion = 5;
 711       }
 712
 713       if (currentSchemaVersion != 5) {
 714         throw std::runtime_error("Somehow, we are not at schema version 5. Giving up");
 715       }
 716
 717       d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | d_asyncFlag, 0600, mapSize), "domains_v5");
 718       d_tmeta = std::make_shared<tmeta_t>(d_tdomains->getEnv(), "metadata_v5");
 719       d_tkdb = std::make_shared<tkdb_t>(d_tdomains->getEnv(), "keydata_v5");
 720       d_ttsig = std::make_shared<ttsig_t>(d_tdomains->getEnv(), "tsig_v5");
 721
 722       auto pdnsdbi = d_tdomains->getEnv()->openDB("pdns", MDB_CREATE);
 723
 724       opened = true;
 725
 726       auto txn = d_tdomains->getEnv()->getRWTransaction();
 727
 728       MDBOutVal shards;
 729       if (!txn->get(pdnsdbi, "shards", shards)) {
 730         s_shards = shards.get<uint32_t>();
 731
 732         if (mustDo("lightning-stream") && s_shards != 1) {
 733           throw std::runtime_error(std::string("running with Lightning Stream support enabled requires a database with exactly 1 shard"));
 734         }
 735
 736         if (s_shards != atoi(getArg("shards").c_str())) {
 737           g_log << Logger::Warning << "Note: configured number of lmdb shards (" << atoi(getArg("shards").c_str()) << ") is different from on-disk (" << s_shards << "). Using on-disk shard number" << endl;
 738         }
 739       }
 740       else {
 741         s_shards = atoi(getArg("shards").c_str());
 742         txn->put(pdnsdbi, "shards", s_shards);
 743       }
 744
 745       MDBOutVal gotuuid;
 746       if (txn->get(pdnsdbi, "uuid", gotuuid)) {
 747         const auto uuid = getUniqueID();
 748         const string uuids(uuid.begin(), uuid.end());
 749         txn->put(pdnsdbi, "uuid", uuids);
 750       }
 751
 752       MDBOutVal _schemaversion;
 753       if (txn->get(pdnsdbi, "schemaversion", _schemaversion)) {
 754         // our DB is entirely new, so we need to write the schemaversion
 755         txn->put(pdnsdbi, "schemaversion", currentSchemaVersion);
 756       }
 757       txn->commit();
 758
 759       s_first = false;
 760     }
 761   }
 762
 763   if (!opened) {
 764     d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | d_asyncFlag, 0600, mapSize), "domains_v5");
 765     d_tmeta = std::make_shared<tmeta_t>(d_tdomains->getEnv(), "metadata_v5");
 766     d_tkdb = std::make_shared<tkdb_t>(d_tdomains->getEnv(), "keydata_v5");
 767     d_ttsig = std::make_shared<ttsig_t>(d_tdomains->getEnv(), "tsig_v5");
 768   }
 769   d_trecords.resize(s_shards);
 770   d_dolog = ::arg().mustDo("query-logging");
 771 }
 772
 773 namespace boost
 774 {
 775 namespace serialization
 776 {
 777
 778   template <class Archive>
 779   void save(Archive& ar, const DNSName& g, const unsigned int /* version */)
 780   {
 781     if (g.empty()) {
 782       ar& std::string();
 783     }
 784     else {
 785       ar& g.toDNSStringLC();
 786     }
 787   }
 788
 789   template <class Archive>
 790   void load(Archive& ar, DNSName& g, const unsigned int /* version */)
 791   {
 792     string tmp;
 793     ar& tmp;
 794     if (tmp.empty()) {
 795       g = DNSName();
 796     }
 797     else {
 798       g = DNSName(tmp.c_str(), tmp.size(), 0, false);
 799     }
 800   }
 801
 802   template <class Archive>
 803   void save(Archive& ar, const QType& g, const unsigned int /* version */)
 804   {
 805     ar& g.getCode();
 806   }
 807
 808   template <class Archive>
 809   void load(Archive& ar, QType& g, const unsigned int /* version */)
 810   {
 811     uint16_t tmp;
 812     ar& tmp;
 813     g = QType(tmp);
 814   }
 815
 816   template <class Archive>
 817   void save(Archive& ar, const DomainInfo& g, const unsigned int /* version */)
 818   {
 819     ar& g.zone;
 820     ar& g.last_check;
 821     ar& g.account;
 822     ar& g.masters;
 823     ar& g.id;
 824     ar& g.notified_serial;
 825     ar& g.kind;
 826     ar& g.options;
 827     ar& g.catalog;
 828   }
 829
 830   template <class Archive>
 831   void load(Archive& ar, DomainInfo& g, const unsigned int version)
 832   {
 833     ar& g.zone;
 834     ar& g.last_check;
 835     ar& g.account;
 836     ar& g.masters;
 837     ar& g.id;
 838     ar& g.notified_serial;
 839     ar& g.kind;
 840     if (version >= 1) {
 841       ar& g.options;
 842       ar& g.catalog;
 843     }
 844     else {
 845       g.options.clear();
 846       g.catalog.clear();
 847     }
 848   }
 849
 850   template <class Archive>
 851   void serialize(Archive& ar, LMDBBackend::DomainMeta& g, const unsigned int /* version */)
 852   {
 853     ar& g.domain& g.key& g.value;
 854   }
 855
 856   template <class Archive>
 857   void save(Archive& ar, const LMDBBackend::KeyDataDB& g, const unsigned int /* version */)
 858   {
 859     ar& g.domain& g.content& g.flags& g.active& g.published;
 860   }
 861
 862   template <class Archive>
 863   void load(Archive& ar, LMDBBackend::KeyDataDB& g, const unsigned int version)
 864   {
 865     ar& g.domain& g.content& g.flags& g.active;
 866     if (version >= 1) {
 867       ar& g.published;
 868     }
 869     else {
 870       g.published = true;
 871     }
 872   }
 873
 874   template <class Archive>
 875   void serialize(Archive& ar, TSIGKey& g, const unsigned int /* version */)
 876   {
 877     ar& g.name;
 878     ar& g.algorithm; // this is the ordername
 879     ar& g.key;
 880   }
 881
 882 } // namespace serialization
 883 } // namespace boost
 884
 885 BOOST_SERIALIZATION_SPLIT_FREE(DNSName);
 886 BOOST_SERIALIZATION_SPLIT_FREE(QType);
 887 BOOST_SERIALIZATION_SPLIT_FREE(LMDBBackend::KeyDataDB);
 888 BOOST_SERIALIZATION_SPLIT_FREE(DomainInfo);
 889 BOOST_IS_BITWISE_SERIALIZABLE(ComboAddress);
 890
 891 template <>
 892 std::string serToString(const LMDBBackend::LMDBResourceRecord& lrr)
 893 {
 894   std::string ret;
 895   uint16_t len = lrr.content.length();
 896   ret.reserve(2 + len + 7);
 897
 898   ret.assign((const char*)&len, 2);
 899   ret += lrr.content;
 900   ret.append((const char*)&lrr.ttl, 4);
 901   ret.append(1, (char)lrr.auth);
 902   ret.append(1, (char)lrr.disabled);
 903   ret.append(1, (char)lrr.ordername);
 904   return ret;
 905 }
 906
 907 template <>
 908 std::string serToString(const vector<LMDBBackend::LMDBResourceRecord>& lrrs)
 909 {
 910   std::string ret;
 911   for (const auto& lrr : lrrs) {
 912     ret += serToString(lrr);
 913   }
 914   return ret;
 915 }
 916
 917 static inline size_t serOneRRFromString(const string_view& str, LMDBBackend::LMDBResourceRecord& lrr)
 918 {
 919   uint16_t len;
 920   memcpy(&len, &str[0], 2);
 921   lrr.content.assign(&str[2], len); // len bytes
 922   memcpy(&lrr.ttl, &str[2] + len, 4);
 923   lrr.auth = str[2 + len + 4];
 924   lrr.disabled = str[2 + len + 4 + 1];
 925   lrr.ordername = str[2 + len + 4 + 2];
 926   lrr.wildcardname.clear();
 927
 928   return 2 + len + 7;
 929 }
 930
 931 template <>
 932 void serFromString(const string_view& str, LMDBBackend::LMDBResourceRecord& lrr)
 933 {
 934   serOneRRFromString(str, lrr);
 935 }
 936
 937 template <>
 938 void serFromString(const string_view& str, vector<LMDBBackend::LMDBResourceRecord>& lrrs)
 939 {
 940   auto str_copy = str;
 941   while (str_copy.size() >= 9) { // minimum length for a record is 10
 942     LMDBBackend::LMDBResourceRecord lrr;
 943     auto rrLength = serOneRRFromString(str_copy, lrr);
 944     lrrs.emplace_back(lrr);
 945     str_copy.remove_prefix(rrLength);
 946   }
 947 }
 948
 949 static std::string serializeContent(uint16_t qtype, const DNSName& domain, const std::string& content)
 950 {
 951   auto drc = DNSRecordContent::mastermake(qtype, QClass::IN, content);
 952   return drc->serialize(domain, false);
 953 }
 954
 955 static std::shared_ptr<DNSRecordContent> deserializeContentZR(uint16_t qtype, const DNSName& qname, const std::string& content)
 956 {
 957   if (qtype == QType::A && content.size() == 4) {
 958     return std::make_shared<ARecordContent>(*((uint32_t*)content.c_str()));
 959   }
 960   return DNSRecordContent::deserialize(qname, qtype, content);
 961 }
 962
 963 /* design. If you ask a question without a zone id, we lookup the best
 964    zone id for you, and answer from that. This is different than other backends, but I can't see why it would not work.
 965
 966    The index we use is "zoneid,canonical relative name". This index is also used
 967    for AXFR.
 968
 969    Note - domain_id, name and type are ONLY present on the index!
 970 */
 971
 972 #if BOOST_VERSION >= 106100
 973 #define StringView string_view
 974 #else
 975 #define StringView string
 976 #endif
 977
 978 void LMDBBackend::deleteDomainRecords(RecordsRWTransaction& txn, uint32_t domain_id, uint16_t qtype)
 979 {
 980   compoundOrdername co;
 981   string match = co(domain_id);
 982
 983   auto cursor = txn.txn->getCursor(txn.db->dbi);
 984   MDBOutVal key, val;
 985   //  cout<<"Match: "<<makeHexDump(match);
 986   if (!cursor.lower_bound(match, key, val)) {
 987     while (key.getNoStripHeader<StringView>().rfind(match, 0) == 0) {
 988       if (qtype == QType::ANY || co.getQType(key.getNoStripHeader<StringView>()) == qtype)
 989         cursor.del();
 990       if (cursor.next(key, val))
 991         break;
 992     }
 993   }
 994 }
 995
 996 /* Here's the complicated story. Other backends have just one transaction, which is either
 997    on or not.
 998
 999    You can't call feedRecord without a transaction started with startTransaction.
1000
1001    However, other functions can be called after startTransaction() or without startTransaction()
1002      (like updateDNSSECOrderNameAndAuth)
1003
1004
1005
1006 */
1007
1008 bool LMDBBackend::startTransaction(const DNSName& domain, int domain_id)
1009 {
1010   // cout <<"startTransaction("<<domain<<", "<<domain_id<<")"<<endl;
1011   int real_id = domain_id;
1012   if (real_id < 0) {
1013     auto rotxn = d_tdomains->getROTransaction();
1014     DomainInfo di;
1015     real_id = rotxn.get<0>(domain, di);
1016     // cout<<"real_id = "<<real_id << endl;
1017     if (!real_id)
1018       return false;
1019   }
1020   if (d_rwtxn) {
1021     throw DBException("Attempt to start a transaction while one was open already");
1022   }
1023   d_rwtxn = getRecordsRWTransaction(real_id);
1024
1025   d_transactiondomain = domain;
1026   d_transactiondomainid = real_id;
1027   if (domain_id >= 0) {
1028     deleteDomainRecords(*d_rwtxn, domain_id);
1029   }
1030
1031   return true;
1032 }
1033
1034 bool LMDBBackend::commitTransaction()
1035 {
1036   // cout<<"Commit transaction" <<endl;
1037   if (!d_rwtxn) {
1038     throw DBException("Attempt to commit a transaction while there isn't one open");
1039   }
1040
1041   d_rwtxn->txn->commit();
1042   d_rwtxn.reset();
1043   return true;
1044 }
1045
1046 bool LMDBBackend::abortTransaction()
1047 {
1048   // cout<<"Abort transaction"<<endl;
1049   if (!d_rwtxn) {
1050     throw DBException("Attempt to abort a transaction while there isn't one open");
1051   }
1052
1053   d_rwtxn->txn->abort();
1054   d_rwtxn.reset();
1055
1056   return true;
1057 }
1058
1059 // d_rwtxn must be set here
1060 bool LMDBBackend::feedRecord(const DNSResourceRecord& r, const DNSName& ordername, bool ordernameIsNSEC3)
1061 {
1062   LMDBResourceRecord lrr(r);
1063   lrr.qname.makeUsRelative(d_transactiondomain);
1064   lrr.content = serializeContent(lrr.qtype.getCode(), r.qname, lrr.content);
1065
1066   compoundOrdername co;
1067   string matchName = co(lrr.domain_id, lrr.qname, lrr.qtype.getCode());
1068
1069   string rrs;
1070   MDBOutVal _rrs;
1071   if (!d_rwtxn->txn->get(d_rwtxn->db->dbi, matchName, _rrs)) {
1072     rrs = _rrs.get<string>();
1073   }
1074
1075   rrs += serToString(lrr);
1076
1077   d_rwtxn->txn->put(d_rwtxn->db->dbi, matchName, rrs);
1078
1079   if (ordernameIsNSEC3 && !ordername.empty()) {
1080     MDBOutVal val;
1081     if (d_rwtxn->txn->get(d_rwtxn->db->dbi, co(lrr.domain_id, lrr.qname, QType::NSEC3), val)) {
1082       lrr.ttl = 0;
1083       lrr.content = lrr.qname.toDNSStringLC();
1084       lrr.auth = 0;
1085       string ser = serToString(lrr);
1086       d_rwtxn->txn->put(d_rwtxn->db->dbi, co(lrr.domain_id, ordername, QType::NSEC3), ser);
1087
1088       lrr.ttl = 1;
1089       lrr.content = ordername.toDNSString();
1090       ser = serToString(lrr);
1091       d_rwtxn->txn->put(d_rwtxn->db->dbi, co(lrr.domain_id, lrr.qname, QType::NSEC3), ser);
1092     }
1093   }
1094   return true;
1095 }
1096
1097 bool LMDBBackend::feedEnts(int domain_id, map<DNSName, bool>& nonterm)
1098 {
1099   LMDBResourceRecord lrr;
1100   lrr.ttl = 0;
1101   compoundOrdername co;
1102   for (const auto& nt : nonterm) {
1103     lrr.qname = nt.first.makeRelative(d_transactiondomain);
1104     lrr.auth = nt.second;
1105     lrr.ordername = true;
1106
1107     std::string ser = serToString(lrr);
1108     d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, lrr.qname, QType::ENT), ser);
1109   }
1110   return true;
1111 }
1112
1113 bool LMDBBackend::feedEnts3(int domain_id, const DNSName& domain, map<DNSName, bool>& nonterm, const NSEC3PARAMRecordContent& ns3prc, bool narrow)
1114 {
1115   string ser;
1116   DNSName ordername;
1117   LMDBResourceRecord lrr;
1118   compoundOrdername co;
1119   for (const auto& nt : nonterm) {
1120     lrr.qname = nt.first.makeRelative(domain);
1121     lrr.ttl = 0;
1122     lrr.auth = nt.second;
1123     lrr.ordername = nt.second;
1124     ser = serToString(lrr);
1125     d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, lrr.qname, QType::ENT), ser);
1126
1127     if (!narrow && lrr.auth) {
1128       lrr.content = lrr.qname.toDNSString();
1129       lrr.auth = false;
1130       lrr.ordername = false;
1131       ser = serToString(lrr);
1132
1133       ordername = DNSName(toBase32Hex(hashQNameWithSalt(ns3prc, nt.first)));
1134       d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, ordername, QType::NSEC3), ser);
1135
1136       lrr.ttl = 1;
1137       lrr.content = ordername.toDNSString();
1138       ser = serToString(lrr);
1139       d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, lrr.qname, QType::NSEC3), ser);
1140     }
1141   }
1142   return true;
1143 }
1144
1145 // might be called within a transaction, might also be called alone
1146 bool LMDBBackend::replaceRRSet(uint32_t domain_id, const DNSName& qname, const QType& qt, const vector<DNSResourceRecord>& rrset)
1147 {
1148   // zonk qname/qtype within domain_id (go through qname, check domain_id && qtype)
1149   shared_ptr<RecordsRWTransaction> txn;
1150   bool needCommit = false;
1151   if (d_rwtxn && d_transactiondomainid == domain_id) {
1152     txn = d_rwtxn;
1153     //    cout<<"Reusing open transaction"<<endl;
1154   }
1155   else {
1156     //    cout<<"Making a new RW txn for replace rrset"<<endl;
1157     txn = getRecordsRWTransaction(domain_id);
1158     needCommit = true;
1159   }
1160
1161   DomainInfo di;
1162   if (!d_tdomains->getROTransaction().get(domain_id, di)) {
1163     return false;
1164   }
1165
1166   compoundOrdername co;
1167   auto cursor = txn->txn->getCursor(txn->db->dbi);
1168   MDBOutVal key, val;
1169   string match = co(domain_id, qname.makeRelative(di.zone), qt.getCode());
1170   if (!cursor.find(match, key, val)) {
1171     cursor.del();
1172   }
1173
1174   if (!rrset.empty()) {
1175     vector<LMDBResourceRecord> adjustedRRSet;
1176     for (const auto& rr : rrset) {
1177       LMDBResourceRecord lrr(rr);
1178       lrr.content = serializeContent(lrr.qtype.getCode(), lrr.qname, lrr.content);
1179       lrr.qname.makeUsRelative(di.zone);
1180
1181       adjustedRRSet.emplace_back(lrr);
1182     }
1183     txn->txn->put(txn->db->dbi, match, serToString(adjustedRRSet));
1184   }
1185
1186   if (needCommit)
1187     txn->txn->commit();
1188
1189   return true;
1190 }
1191
1192 bool LMDBBackend::replaceComments([[maybe_unused]] const uint32_t domain_id, [[maybe_unused]] const DNSName& qname, [[maybe_unused]] const QType& qt, const vector<Comment>& comments)
1193 {
1194   // if the vector is empty, good, that's what we do here (LMDB does not store comments)
1195   // if it's not, report failure
1196   return comments.empty();
1197 }
1198
1199 // tempting to templatize these two functions but the pain is not worth it
1200 std::shared_ptr<LMDBBackend::RecordsRWTransaction> LMDBBackend::getRecordsRWTransaction(uint32_t id)
1201 {
1202   auto& shard = d_trecords[id % s_shards];
1203   if (!shard.env) {
1204     shard.env = getMDBEnv((getArg("filename") + "-" + std::to_string(id % s_shards)).c_str(),
1205                           MDB_NOSUBDIR | d_asyncFlag, 0600);
1206     shard.dbi = shard.env->openDB("records_v5", MDB_CREATE);
1207   }
1208   auto ret = std::make_shared<RecordsRWTransaction>(shard.env->getRWTransaction());
1209   ret->db = std::make_shared<RecordsDB>(shard);
1210
1211   return ret;
1212 }
1213
1214 std::shared_ptr<LMDBBackend::RecordsROTransaction> LMDBBackend::getRecordsROTransaction(uint32_t id, std::shared_ptr<LMDBBackend::RecordsRWTransaction> rwtxn)
1215 {
1216   auto& shard = d_trecords[id % s_shards];
1217   if (!shard.env) {
1218     if (rwtxn) {
1219       throw DBException("attempting to start nested transaction without open parent env");
1220     }
1221     shard.env = getMDBEnv((getArg("filename") + "-" + std::to_string(id % s_shards)).c_str(),
1222                           MDB_NOSUBDIR | d_asyncFlag, 0600);
1223     shard.dbi = shard.env->openDB("records_v5", MDB_CREATE);
1224   }
1225
1226   if (rwtxn) {
1227     auto ret = std::make_shared<RecordsROTransaction>(rwtxn->txn->getROTransaction());
1228     ret->db = std::make_shared<RecordsDB>(shard);
1229     return ret;
1230   }
1231   else {
1232     auto ret = std::make_shared<RecordsROTransaction>(shard.env->getROTransaction());
1233     ret->db = std::make_shared<RecordsDB>(shard);
1234     return ret;
1235   }
1236 }
1237
1238 #if 0
1239 // FIXME reinstate soon
1240 bool LMDBBackend::upgradeToSchemav3()
1241 {
1242   g_log << Logger::Warning << "Upgrading LMDB schema" << endl;
1243
1244   for (auto i = 0; i < s_shards; i++) {
1245     string filename = getArg("filename") + "-" + std::to_string(i);
1246     if (rename(filename.c_str(), (filename + "-old").c_str()) < 0) {
1247       if (errno == ENOENT) {
1248         // apparently this shard doesn't exist yet, moving on
1249         continue;
1250       }
1251       unixDie("Rename failed during LMDB upgrade");
1252     }
1253
1254     LMDBBackend::RecordsDB oldShard, newShard;
1255
1256     oldShard.env = getMDBEnv((filename + "-old").c_str(),
1257                              MDB_NOSUBDIR | d_asyncFlag, 0600);
1258     oldShard.dbi = oldShard.env->openDB("records", MDB_CREATE | MDB_DUPSORT);
1259     auto txn = oldShard.env->getROTransaction();
1260     auto cursor = txn->getROCursor(oldShard.dbi);
1261
1262     newShard.env = getMDBEnv((filename).c_str(),
1263                              MDB_NOSUBDIR | d_asyncFlag, 0600);
1264     newShard.dbi = newShard.env->openDB("records", MDB_CREATE);
1265     auto newTxn = newShard.env->getRWTransaction();
1266
1267     MDBOutVal key, val;
1268     if (cursor.first(key, val) != 0) {
1269       cursor.close();
1270       txn->abort();
1271       newTxn->abort();
1272       continue;
1273     }
1274     string_view currentKey;
1275     string value;
1276     for (;;) {
1277       auto newKey = key.getNoStripHeader<string_view>();
1278       if (currentKey.compare(newKey) != 0) {
1279         if (value.size() > 0) {
1280           newTxn->put(newShard.dbi, currentKey, value);
1281         }
1282         currentKey = newKey;
1283         value = "";
1284       }
1285       value += val.get<string>();
1286       if (cursor.next(key, val) != 0) {
1287         if (value.size() > 0) {
1288           newTxn->put(newShard.dbi, currentKey, value);
1289         }
1290         break;
1291       }
1292     }
1293
1294     cursor.close();
1295     txn->commit();
1296     newTxn->commit();
1297   }
1298
1299   return true;
1300 }
1301 #endif
1302
1303 bool LMDBBackend::deleteDomain(const DNSName& domain)
1304 {
1305   if (!d_rwtxn) {
1306     throw DBException(std::string(__PRETTY_FUNCTION__) + " called without a transaction");
1307   }
1308
1309   int transactionDomainId = d_transactiondomainid;
1310   DNSName transactionDomain = d_transactiondomain;
1311
1312   abortTransaction();
1313
1314   LMDBIDvec idvec;
1315
1316   if (!d_handle_dups) {
1317     // get domain id
1318     auto txn = d_tdomains->getROTransaction();
1319
1320     DomainInfo di;
1321     idvec.push_back(txn.get<0>(domain, di));
1322   }
1323   else {
1324     // this transaction used to be RO.
1325     // it is now RW to narrow a race window between PowerDNS and Lightning Stream
1326     // FIXME: turn the entire delete, including this ID scan, into one RW transaction
1327     // when doing that, first do a short RO check to see if we actually have anything to delete
1328     auto txn = d_tdomains->getRWTransaction();
1329
1330     txn.get_multi<0>(domain, idvec);
1331   }
1332
1333   for (auto id : idvec) {
1334
1335     startTransaction(domain, id);
1336
1337     { // Remove metadata
1338       auto txn = d_tmeta->getRWTransaction();
1339       LMDBIDvec ids;
1340
1341       txn.get_multi<0>(domain, ids);
1342
1343       for (auto& _id : ids) {
1344         txn.del(_id);
1345       }
1346
1347       txn.commit();
1348     }
1349
1350     { // Remove cryptokeys
1351       auto txn = d_tkdb->getRWTransaction();
1352       LMDBIDvec ids;
1353       txn.get_multi<0>(domain, ids);
1354
1355       for (auto _id : ids) {
1356         txn.del(_id);
1357       }
1358
1359       txn.commit();
1360     }
1361
1362     // Remove records
1363     commitTransaction();
1364
1365     // Remove zone
1366     auto txn = d_tdomains->getRWTransaction();
1367     txn.del(id);
1368     txn.commit();
1369   }
1370
1371   startTransaction(transactionDomain, transactionDomainId);
1372
1373   return true;
1374 }
1375
1376 bool LMDBBackend::list(const DNSName& target, int /* id */, bool include_disabled)
1377 {
1378   d_includedisabled = include_disabled;
1379
1380   DomainInfo di;
1381   {
1382     auto dtxn = d_tdomains->getROTransaction();
1383     if ((di.id = dtxn.get<0>(target, di))) {
1384       // cerr << "Found domain " << target << " on domain_id " << di.id << ", list requested " << id << endl;
1385     }
1386     else {
1387       // cerr << "Did not find " << target << endl;
1388       return false;
1389     }
1390   }
1391
1392   d_rotxn = getRecordsROTransaction(di.id, d_rwtxn);
1393   d_getcursor = std::make_shared<MDBROCursor>(d_rotxn->txn->getCursor(d_rotxn->db->dbi));
1394
1395   compoundOrdername co;
1396   d_matchkey = co(di.id);
1397
1398   MDBOutVal key, val;
1399   auto a = d_getcursor->lower_bound(d_matchkey, key, val);
1400   auto b0 = key.getNoStripHeader<StringView>();
1401   auto b = b0.rfind(d_matchkey, 0);
1402   if (a || b != 0) {
1403     d_getcursor.reset();
1404   }
1405
1406   d_lookupdomain = target;
1407
1408   // Make sure we start with fresh data
1409   d_currentrrset.clear();
1410   d_currentrrsetpos = 0;
1411
1412   return true;
1413 }
1414
1415 void LMDBBackend::lookup(const QType& type, const DNSName& qdomain, int zoneId, DNSPacket* /* p */)
1416 {
1417   if (d_dolog) {
1418     g_log << Logger::Warning << "Got lookup for " << qdomain << "|" << type.toString() << " in zone " << zoneId << endl;
1419     d_dtime.set();
1420   }
1421
1422   d_includedisabled = false;
1423
1424   DNSName hunt(qdomain);
1425   DomainInfo di;
1426   if (zoneId < 0) {
1427     auto rotxn = d_tdomains->getROTransaction();
1428
1429     do {
1430       zoneId = rotxn.get<0>(hunt, di);
1431     } while (!zoneId && type != QType::SOA && hunt.chopOff());
1432     if (zoneId <= 0) {
1433       //      cout << "Did not find zone for "<< qdomain<<endl;
1434       d_getcursor.reset();
1435       return;
1436     }
1437   }
1438   else {
1439     if (!d_tdomains->getROTransaction().get(zoneId, di)) {
1440       // cout<<"Could not find a zone with id "<<zoneId<<endl;
1441       d_getcursor.reset();
1442       return;
1443     }
1444     hunt = di.zone;
1445   }
1446
1447   DNSName relqname = qdomain.makeRelative(hunt);
1448   if (relqname.empty()) {
1449     return;
1450   }
1451   // cout<<"get will look for "<<relqname<< " in zone "<<hunt<<" with id "<<zoneId<<" and type "<<type.toString()<<endl;
1452   d_rotxn = getRecordsROTransaction(zoneId, d_rwtxn);
1453
1454   compoundOrdername co;
1455   d_getcursor = std::make_shared<MDBROCursor>(d_rotxn->txn->getCursor(d_rotxn->db->dbi));
1456   MDBOutVal key, val;
1457   if (type.getCode() == QType::ANY) {
1458     d_matchkey = co(zoneId, relqname);
1459   }
1460   else {
1461     d_matchkey = co(zoneId, relqname, type.getCode());
1462   }
1463
1464   if (d_getcursor->lower_bound(d_matchkey, key, val) || key.getNoStripHeader<StringView>().rfind(d_matchkey, 0) != 0) {
1465     d_getcursor.reset();
1466     if (d_dolog) {
1467       g_log << Logger::Warning << "Query " << ((long)(void*)this) << ": " << d_dtime.udiffNoReset() << " us to execute (found nothing)" << endl;
1468     }
1469     return;
1470   }
1471
1472   if (d_dolog) {
1473     g_log << Logger::Warning << "Query " << ((long)(void*)this) << ": " << d_dtime.udiffNoReset() << " us to execute" << endl;
1474   }
1475
1476   d_lookupdomain = hunt;
1477
1478   // Make sure we start with fresh data
1479   d_currentrrset.clear();
1480   d_currentrrsetpos = 0;
1481 }
1482
1483 bool LMDBBackend::get(DNSZoneRecord& zr)
1484 {
1485   for (;;) {
1486     // std::cerr<<"d_getcursor="<<d_getcursor<<std::endl;
1487     if (!d_getcursor) {
1488       d_rotxn.reset();
1489       return false;
1490     }
1491
1492     string_view key;
1493
1494     if (d_currentrrset.empty()) {
1495       d_getcursor->current(d_currentKey, d_currentVal);
1496
1497       key = d_currentKey.getNoStripHeader<string_view>();
1498       zr.dr.d_type = compoundOrdername::getQType(key).getCode();
1499
1500       if (zr.dr.d_type == QType::NSEC3) {
1501         // Hit a magic NSEC3 skipping
1502         if (d_getcursor->next(d_currentKey, d_currentVal) || d_currentKey.getNoStripHeader<StringView>().rfind(d_matchkey, 0) != 0) {
1503           // cerr<<"resetting d_getcursor 1"<<endl;
1504           d_getcursor.reset();
1505         }
1506         continue;
1507       }
1508
1509       serFromString(d_currentVal.get<string_view>(), d_currentrrset);
1510       d_currentrrsetpos = 0;
1511     }
1512     else {
1513       key = d_currentKey.getNoStripHeader<string_view>();
1514     }
1515     try {
1516       const auto& lrr = d_currentrrset.at(d_currentrrsetpos++);
1517
1518       zr.disabled = lrr.disabled;
1519       if (!zr.disabled || d_includedisabled) {
1520         zr.dr.d_name = compoundOrdername::getQName(key) + d_lookupdomain;
1521         zr.domain_id = compoundOrdername::getDomainID(key);
1522         zr.dr.d_type = compoundOrdername::getQType(key).getCode();
1523         zr.dr.d_ttl = lrr.ttl;
1524         zr.dr.setContent(deserializeContentZR(zr.dr.d_type, zr.dr.d_name, lrr.content));
1525         zr.auth = lrr.auth;
1526       }
1527
1528       if (d_currentrrsetpos >= d_currentrrset.size()) {
1529         d_currentrrset.clear(); // will invalidate lrr
1530         if (d_getcursor->next(d_currentKey, d_currentVal) || d_currentKey.getNoStripHeader<StringView>().rfind(d_matchkey, 0) != 0) {
1531           // cerr<<"resetting d_getcursor 2"<<endl;
1532           d_getcursor.reset();
1533         }
1534       }
1535
1536       if (zr.disabled && !d_includedisabled) {
1537         continue;
1538       }
1539     }
1540     catch (const std::exception& e) {
1541       throw PDNSException(e.what());
1542     }
1543
1544     break;
1545   }
1546
1547   return true;
1548 }
1549
1550 bool LMDBBackend::get(DNSResourceRecord& rr)
1551 {
1552   DNSZoneRecord zr;
1553   if (!get(zr)) {
1554     return false;
1555   }
1556
1557   rr.qname = zr.dr.d_name;
1558   rr.ttl = zr.dr.d_ttl;
1559   rr.qtype = zr.dr.d_type;
1560   rr.content = zr.dr.getContent()->getZoneRepresentation(true);
1561   rr.domain_id = zr.domain_id;
1562   rr.auth = zr.auth;
1563   rr.disabled = zr.disabled;
1564
1565   return true;
1566 }
1567
1568 bool LMDBBackend::getSerial(DomainInfo& di)
1569 {
1570   auto txn = getRecordsROTransaction(di.id);
1571   compoundOrdername co;
1572   MDBOutVal val;
1573   if (!txn->txn->get(txn->db->dbi, co(di.id, g_rootdnsname, QType::SOA), val)) {
1574     LMDBResourceRecord lrr;
1575     serFromString(val.get<string_view>(), lrr);
1576     if (lrr.content.size() >= 5 * sizeof(uint32_t)) {
1577       uint32_t serial;
1578       // a SOA has five 32 bit fields, the first of which is the serial
1579       // there are two variable length names before the serial, so we calculate from the back
1580       memcpy(&serial, &lrr.content[lrr.content.size() - (5 * sizeof(uint32_t))], sizeof(serial));
1581       di.serial = ntohl(serial);
1582     }
1583     return !lrr.disabled;
1584   }
1585   return false;
1586 }
1587
1588 bool LMDBBackend::getDomainInfo(const DNSName& domain, DomainInfo& di, bool getserial)
1589 {
1590   {
1591     auto txn = d_tdomains->getROTransaction();
1592     // auto range = txn.prefix_range<0>(domain);
1593
1594     // bool found = false;
1595
1596     // for (auto& iter = range.first ; iter != range.second; ++iter) {
1597     //   found = true;
1598     //   di.id = iter.getID();
1599     //   di.backend = this;
1600     // }
1601
1602     // if (!found) {
1603     //   return false;
1604     // }
1605     if (!(di.id = txn.get<0>(domain, di))) {
1606       return false;
1607     }
1608
1609     di.backend = this;
1610   }
1611
1612   if (getserial) {
1613     getSerial(di);
1614   }
1615
1616   return true;
1617 }
1618
1619 int LMDBBackend::genChangeDomain(const DNSName& domain, std::function<void(DomainInfo&)> func)
1620 {
1621   auto txn = d_tdomains->getRWTransaction();
1622
1623   DomainInfo di;
1624
1625   auto id = txn.get<0>(domain, di);
1626   func(di);
1627   txn.put(di, id);
1628
1629   txn.commit();
1630   return true;
1631 }
1632
1633 int LMDBBackend::genChangeDomain(uint32_t id, std::function<void(DomainInfo&)> func)
1634 {
1635   DomainInfo di;
1636
1637   auto txn = d_tdomains->getRWTransaction();
1638
1639   if (!txn.get(id, di))
1640     return false;
1641
1642   func(di);
1643
1644   txn.put(di, id);
1645
1646   txn.commit();
1647   return true;
1648 }
1649
1650 bool LMDBBackend::setKind(const DNSName& domain, const DomainInfo::DomainKind kind)
1651 {
1652   return genChangeDomain(domain, [kind](DomainInfo& di) {
1653     di.kind = kind;
1654   });
1655 }
1656
1657 bool LMDBBackend::setAccount(const DNSName& domain, const std::string& account)
1658 {
1659   return genChangeDomain(domain, [account](DomainInfo& di) {
1660     di.account = account;
1661   });
1662 }
1663
1664 bool LMDBBackend::setMasters(const DNSName& domain, const vector<ComboAddress>& masters)
1665 {
1666   return genChangeDomain(domain, [&masters](DomainInfo& di) {
1667     di.masters = masters;
1668   });
1669 }
1670
1671 bool LMDBBackend::createDomain(const DNSName& domain, const DomainInfo::DomainKind kind, const vector<ComboAddress>& masters, const string& account)
1672 {
1673   DomainInfo di;
1674
1675   {
1676     auto txn = d_tdomains->getRWTransaction();
1677     if (txn.get<0>(domain, di)) {
1678       throw DBException("Domain '" + domain.toLogString() + "' exists already");
1679     }
1680
1681     di.zone = domain;
1682     di.kind = kind;
1683     di.masters = masters;
1684     di.account = account;
1685
1686     txn.put(di, 0, d_random_ids);
1687     txn.commit();
1688   }
1689
1690   return true;
1691 }
1692
1693 void LMDBBackend::getAllDomainsFiltered(vector<DomainInfo>* domains, const std::function<bool(DomainInfo&)>& allow)
1694 {
1695   auto txn = d_tdomains->getROTransaction();
1696   if (d_handle_dups) {
1697     map<DNSName, DomainInfo> zonemap;
1698     set<DNSName> dups;
1699
1700     for (auto iter = txn.begin(); iter != txn.end(); ++iter) {
1701       DomainInfo di = *iter;
1702       di.id = iter.getID();
1703       di.backend = this;
1704
1705       if (!zonemap.emplace(di.zone, di).second) {
1706         dups.insert(di.zone);
1707       }
1708     }
1709
1710     for (const auto& zone : dups) {
1711       DomainInfo di;
1712
1713       // this get grabs the oldest item if there are duplicates
1714       di.id = txn.get<0>(zone, di);
1715
1716       if (di.id == 0) {
1717         // .get actually found nothing for us
1718         continue;
1719       }
1720
1721       di.backend = this;
1722       zonemap[di.zone] = di;
1723     }
1724
1725     for (auto& [k, v] : zonemap) {
1726       if (allow(v)) {
1727         domains->push_back(std::move(v));
1728       }
1729     }
1730   }
1731   else {
1732     for (auto iter = txn.begin(); iter != txn.end(); ++iter) {
1733       DomainInfo di = *iter;
1734       di.id = iter.getID();
1735       di.backend = this;
1736
1737       if (allow(di)) {
1738         domains->push_back(di);
1739       }
1740     }
1741   }
1742 }
1743
1744 void LMDBBackend::getAllDomains(vector<DomainInfo>* domains, bool /* doSerial */, bool include_disabled)
1745 {
1746   domains->clear();
1747
1748   getAllDomainsFiltered(domains, [this, include_disabled](DomainInfo& di) {
1749     if (!getSerial(di) && !include_disabled) {
1750       return false;
1751     }
1752
1753     return true;
1754   });
1755 }
1756
1757 void LMDBBackend::getUnfreshSlaveInfos(vector<DomainInfo>* domains)
1758 {
1759   uint32_t serial;
1760   time_t now = time(0);
1761   LMDBResourceRecord lrr;
1762   soatimes st;
1763
1764   getAllDomainsFiltered(domains, [this, &lrr, &st, &now, &serial](DomainInfo& di) {
1765     if (!di.isSecondaryType()) {
1766       return false;
1767     }
1768
1769     auto txn2 = getRecordsROTransaction(di.id);
1770     compoundOrdername co;
1771     MDBOutVal val;
1772     if (!txn2->txn->get(txn2->db->dbi, co(di.id, g_rootdnsname, QType::SOA), val)) {
1773       serFromString(val.get<string_view>(), lrr);
1774       memcpy(&st, &lrr.content[lrr.content.size() - sizeof(soatimes)], sizeof(soatimes));
1775       if ((time_t)(di.last_check + ntohl(st.refresh)) > now) { // still fresh
1776         return false;
1777       }
1778       serial = ntohl(st.serial);
1779     }
1780     else {
1781       serial = 0;
1782     }
1783
1784     return true;
1785   });
1786 }
1787
1788 void LMDBBackend::setStale(uint32_t domain_id)
1789 {
1790   genChangeDomain(domain_id, [](DomainInfo& di) {
1791     di.last_check = 0;
1792   });
1793 }
1794
1795 void LMDBBackend::setFresh(uint32_t domain_id)
1796 {
1797   genChangeDomain(domain_id, [](DomainInfo& di) {
1798     di.last_check = time(nullptr);
1799   });
1800 }
1801
1802 void LMDBBackend::getUpdatedMasters(vector<DomainInfo>& updatedDomains, std::unordered_set<DNSName>& catalogs, CatalogHashMap& catalogHashes)
1803 {
1804   CatalogInfo ci;
1805
1806   getAllDomainsFiltered(&(updatedDomains), [this, &catalogs, &catalogHashes, &ci](DomainInfo& di) {
1807     if (!di.isPrimaryType()) {
1808       return false;
1809     }
1810
1811     if (di.kind == DomainInfo::Producer) {
1812       catalogs.insert(di.zone);
1813       catalogHashes[di.zone].process("\0");
1814       return false; // Producer fresness check is performed elsewhere
1815     }
1816
1817     if (!di.catalog.empty()) {
1818       ci.fromJson(di.options, CatalogInfo::CatalogType::Producer);
1819       ci.updateHash(catalogHashes, di);
1820     }
1821
1822     if (getSerial(di) && di.serial != di.notified_serial) {
1823       di.backend = this;
1824       return true;
1825     }
1826
1827     return false;
1828   });
1829 }
1830
1831 void LMDBBackend::setNotified(uint32_t domain_id, uint32_t serial)
1832 {
1833   genChangeDomain(domain_id, [serial](DomainInfo& di) {
1834     di.notified_serial = serial;
1835   });
1836 }
1837
1838 class getCatalogMembersReturnFalseException : std::runtime_error
1839 {
1840 public:
1841   getCatalogMembersReturnFalseException() :
1842     std::runtime_error("getCatalogMembers should return false") {}
1843 };
1844
1845 bool LMDBBackend::getCatalogMembers(const DNSName& catalog, vector<CatalogInfo>& members, CatalogInfo::CatalogType type)
1846 {
1847   vector<DomainInfo> scratch;
1848
1849   try {
1850     getAllDomainsFiltered(&scratch, [&catalog, &members, &type](DomainInfo& di) {
1851       if ((type == CatalogInfo::CatalogType::Producer && di.kind != DomainInfo::Master) || (type == CatalogInfo::CatalogType::Consumer && di.kind != DomainInfo::Slave) || di.catalog != catalog) {
1852         return false;
1853       }
1854
1855       CatalogInfo ci;
1856       ci.d_id = di.id;
1857       ci.d_zone = di.zone;
1858       ci.d_primaries = di.masters;
1859       try {
1860         ci.fromJson(di.options, type);
1861       }
1862       catch (const std::runtime_error& e) {
1863         g_log << Logger::Warning << __PRETTY_FUNCTION__ << " options '" << di.options << "' for zone '" << di.zone << "' is no valid JSON: " << e.what() << endl;
1864         members.clear();
1865         throw getCatalogMembersReturnFalseException();
1866       }
1867       members.emplace_back(ci);
1868
1869       return false;
1870     });
1871   }
1872   catch (const getCatalogMembersReturnFalseException& e) {
1873     return false;
1874   }
1875   return true;
1876 }
1877
1878 bool LMDBBackend::setOptions(const DNSName& domain, const std::string& options)
1879 {
1880   return genChangeDomain(domain, [options](DomainInfo& di) {
1881     di.options = options;
1882   });
1883 }
1884
1885 bool LMDBBackend::setCatalog(const DNSName& domain, const DNSName& catalog)
1886 {
1887   return genChangeDomain(domain, [catalog](DomainInfo& di) {
1888     di.catalog = catalog;
1889   });
1890 }
1891
1892 bool LMDBBackend::getAllDomainMetadata(const DNSName& name, std::map<std::string, std::vector<std::string>>& meta)
1893 {
1894   meta.clear();
1895   auto txn = d_tmeta->getROTransaction();
1896   LMDBIDvec ids;
1897   txn.get_multi<0>(name, ids);
1898
1899   DomainMeta dm;
1900   // cerr<<"getAllDomainMetadata start"<<endl;
1901   for (auto id : ids) {
1902     if (txn.get(id, dm)) {
1903       meta[dm.key].push_back(dm.value);
1904     }
1905   }
1906   return true;
1907 }
1908
1909 bool LMDBBackend::setDomainMetadata(const DNSName& name, const std::string& kind, const std::vector<std::string>& meta)
1910 {
1911   auto txn = d_tmeta->getRWTransaction();
1912
1913   LMDBIDvec ids;
1914   txn.get_multi<0>(name, ids);
1915
1916   DomainMeta dmeta;
1917   for (auto id : ids) {
1918     if (txn.get(id, dmeta)) {
1919       if (dmeta.key == kind) {
1920         // cerr<<"delete"<<endl;
1921         txn.del(id);
1922       }
1923     }
1924   }
1925
1926   for (const auto& m : meta) {
1927     DomainMeta dm{name, kind, m};
1928     txn.put(dm, 0, d_random_ids);
1929   }
1930   txn.commit();
1931   return true;
1932 }
1933
1934 bool LMDBBackend::getDomainKeys(const DNSName& name, std::vector<KeyData>& keys)
1935 {
1936   auto txn = d_tkdb->getROTransaction();
1937   LMDBIDvec ids;
1938   txn.get_multi<0>(name, ids);
1939
1940   KeyDataDB key;
1941
1942   for (auto id : ids) {
1943     if (txn.get(id, key)) {
1944       KeyData kd{key.content, id, key.flags, key.active, key.published};
1945       keys.push_back(kd);
1946     }
1947   }
1948
1949   return true;
1950 }
1951
1952 bool LMDBBackend::removeDomainKey(const DNSName& name, unsigned int id)
1953 {
1954   auto txn = d_tkdb->getRWTransaction();
1955   KeyDataDB kdb;
1956   if (txn.get(id, kdb)) {
1957     if (kdb.domain == name) {
1958       txn.del(id);
1959       txn.commit();
1960       return true;
1961     }
1962   }
1963   // cout << "??? wanted to remove domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
1964   return true;
1965 }
1966
1967 bool LMDBBackend::addDomainKey(const DNSName& name, const KeyData& key, int64_t& id)
1968 {
1969   auto txn = d_tkdb->getRWTransaction();
1970   KeyDataDB kdb{name, key.content, key.flags, key.active, key.published};
1971   id = txn.put(kdb, 0, d_random_ids);
1972   txn.commit();
1973
1974   return true;
1975 }
1976
1977 bool LMDBBackend::activateDomainKey(const DNSName& name, unsigned int id)
1978 {
1979   auto txn = d_tkdb->getRWTransaction();
1980   KeyDataDB kdb;
1981   if (txn.get(id, kdb)) {
1982     if (kdb.domain == name) {
1983       txn.modify(id, [](KeyDataDB& kdbarg) {
1984         kdbarg.active = true;
1985       });
1986       txn.commit();
1987       return true;
1988     }
1989   }
1990
1991   // cout << "??? wanted to activate domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
1992   return true;
1993 }
1994
1995 bool LMDBBackend::deactivateDomainKey(const DNSName& name, unsigned int id)
1996 {
1997   auto txn = d_tkdb->getRWTransaction();
1998   KeyDataDB kdb;
1999   if (txn.get(id, kdb)) {
2000     if (kdb.domain == name) {
2001       txn.modify(id, [](KeyDataDB& kdbarg) {
2002         kdbarg.active = false;
2003       });
2004       txn.commit();
2005       return true;
2006     }
2007   }
2008   // cout << "??? wanted to deactivate domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
2009   return true;
2010 }
2011
2012 bool LMDBBackend::publishDomainKey(const DNSName& name, unsigned int id)
2013 {
2014   auto txn = d_tkdb->getRWTransaction();
2015   KeyDataDB kdb;
2016   if (txn.get(id, kdb)) {
2017     if (kdb.domain == name) {
2018       txn.modify(id, [](KeyDataDB& kdbarg) {
2019         kdbarg.published = true;
2020       });
2021       txn.commit();
2022       return true;
2023     }
2024   }
2025
2026   // cout << "??? wanted to hide domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
2027   return true;
2028 }
2029
2030 bool LMDBBackend::unpublishDomainKey(const DNSName& name, unsigned int id)
2031 {
2032   auto txn = d_tkdb->getRWTransaction();
2033   KeyDataDB kdb;
2034   if (txn.get(id, kdb)) {
2035     if (kdb.domain == name) {
2036       txn.modify(id, [](KeyDataDB& kdbarg) {
2037         kdbarg.published = false;
2038       });
2039       txn.commit();
2040       return true;
2041     }
2042   }
2043   // cout << "??? wanted to unhide domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
2044   return true;
2045 }
2046
2047 bool LMDBBackend::getBeforeAndAfterNamesAbsolute(uint32_t id, const DNSName& qname, DNSName& unhashed, DNSName& before, DNSName& after)
2048 {
2049   //  cout << __PRETTY_FUNCTION__<< ": "<<id <<", "<<qname << " " << unhashed<<endl;
2050
2051   DomainInfo di;
2052   if (!d_tdomains->getROTransaction().get(id, di)) {
2053     // domain does not exist, tough luck
2054     return false;
2055   }
2056   // cout <<"Zone: "<<di.zone<<endl;
2057
2058   compoundOrdername co;
2059   auto txn = getRecordsROTransaction(id);
2060
2061   auto cursor = txn->txn->getCursor(txn->db->dbi);
2062   MDBOutVal key, val;
2063
2064   LMDBResourceRecord lrr;
2065
2066   string matchkey = co(id, qname, QType::NSEC3);
2067   if (cursor.lower_bound(matchkey, key, val)) {
2068     // this is beyond the end of the database
2069     // cout << "Beyond end of database!" << endl;
2070     cursor.last(key, val);
2071
2072     for (;;) {
2073       if (co.getDomainID(key.getNoStripHeader<StringView>()) != id) {
2074         //cout<<"Last record also not part of this zone!"<<endl;
2075         // this implies something is wrong in the database, nothing we can do
2076         return false;
2077       }
2078
2079       if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2080         serFromString(val.get<StringView>(), lrr);
2081         if (!lrr.ttl) // the kind of NSEC3 we need
2082           break;
2083       }
2084       if (cursor.prev(key, val)) {
2085         // hit beginning of database, again means something is wrong with it
2086         return false;
2087       }
2088     }
2089     before = co.getQName(key.getNoStripHeader<StringView>());
2090     unhashed = DNSName(lrr.content.c_str(), lrr.content.size(), 0, false) + di.zone;
2091
2092     // now to find after .. at the beginning of the zone
2093     if (cursor.lower_bound(co(id), key, val)) {
2094       // cout<<"hit end of zone find when we shouldn't"<<endl;
2095       return false;
2096     }
2097     for (;;) {
2098       if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2099         serFromString(val.get<StringView>(), lrr);
2100         if (!lrr.ttl)
2101           break;
2102       }
2103
2104       if (cursor.next(key, val) || co.getDomainID(key.getNoStripHeader<StringView>()) != id) {
2105         // cout<<"hit end of zone or database when we shouldn't"<<endl;
2106         return false;
2107       }
2108     }
2109     after = co.getQName(key.getNoStripHeader<StringView>());
2110     // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2111     return true;
2112   }
2113
2114   // cout<<"Ended up at "<<co.getQName(key.get<StringView>()) <<endl;
2115
2116   before = co.getQName(key.getNoStripHeader<StringView>());
2117   if (before == qname) {
2118     // cout << "Ended up on exact right node" << endl;
2119     before = co.getQName(key.getNoStripHeader<StringView>());
2120     // unhashed should be correct now, maybe check?
2121     if (cursor.next(key, val)) {
2122       // xxx should find first hash now
2123
2124       if (cursor.lower_bound(co(id), key, val)) {
2125         // cout<<"hit end of zone find when we shouldn't for id "<<id<< __LINE__<<endl;
2126         return false;
2127       }
2128       for (;;) {
2129         if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2130           serFromString(val.get<StringView>(), lrr);
2131           if (!lrr.ttl)
2132             break;
2133         }
2134
2135         if (cursor.next(key, val) || co.getDomainID(key.getNoStripHeader<StringView>()) != id) {
2136           // cout<<"hit end of zone or database when we shouldn't" << __LINE__<<endl;
2137           return false;
2138         }
2139       }
2140       after = co.getQName(key.getNoStripHeader<StringView>());
2141       // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2142       return true;
2143     }
2144   }
2145   else {
2146     // cout <<"Going backwards to find 'before'"<<endl;
2147     int count = 0;
2148     for (;;) {
2149       if (co.getQName(key.getNoStripHeader<StringView>()).canonCompare(qname) && co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2150         // cout<<"Potentially stopping traverse at "<< co.getQName(key.get<StringView>()) <<", " << (co.getQName(key.get<StringView>()).canonCompare(qname))<<endl;
2151         // cout<<"qname = "<<qname<<endl;
2152         // cout<<"here  = "<<co.getQName(key.get<StringView>())<<endl;
2153         serFromString(val.get<StringView>(), lrr);
2154         if (!lrr.ttl)
2155           break;
2156       }
2157
2158       if (cursor.prev(key, val) || co.getDomainID(key.getNoStripHeader<StringView>()) != id) {
2159         // cout <<"XXX Hit *beginning* of zone or database"<<endl;
2160         // this can happen, must deal with it
2161         // should now find the last hash of the zone
2162
2163         if (cursor.lower_bound(co(id + 1), key, val)) {
2164           // cout << "Could not find the next higher zone, going to the end of the database then"<<endl;
2165           cursor.last(key, val);
2166         }
2167         else
2168           cursor.prev(key, val);
2169
2170         for (;;) {
2171           if (co.getDomainID(key.getNoStripHeader<StringView>()) != id) {
2172             //cout<<"Last record also not part of this zone!"<<endl;
2173             // this implies something is wrong in the database, nothing we can do
2174             return false;
2175           }
2176
2177           if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2178             serFromString(val.get<StringView>(), lrr);
2179             if (!lrr.ttl) // the kind of NSEC3 we need
2180               break;
2181           }
2182           if (cursor.prev(key, val)) {
2183             // hit beginning of database, again means something is wrong with it
2184             return false;
2185           }
2186         }
2187         before = co.getQName(key.getNoStripHeader<StringView>());
2188         unhashed = DNSName(lrr.content.c_str(), lrr.content.size(), 0, false) + di.zone;
2189         // cout <<"Should still find 'after'!"<<endl;
2190         // for 'after', we need to find the first hash of this zone
2191
2192         if (cursor.lower_bound(co(id), key, val)) {
2193           // cout<<"hit end of zone find when we shouldn't"<<endl;
2194           // means database is wrong, nothing we can do
2195           return false;
2196         }
2197         for (;;) {
2198           if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2199             serFromString(val.get<StringView>(), lrr);
2200             if (!lrr.ttl)
2201               break;
2202           }
2203
2204           if (cursor.next(key, val)) {
2205             // means database is wrong, nothing we can do
2206             // cout<<"hit end of zone when we shouldn't 2"<<endl;
2207             return false;
2208           }
2209         }
2210         after = co.getQName(key.getNoStripHeader<StringView>());
2211
2212         // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2213         return true;
2214       }
2215       ++count;
2216     }
2217     before = co.getQName(key.getNoStripHeader<StringView>());
2218     unhashed = DNSName(lrr.content.c_str(), lrr.content.size(), 0, false) + di.zone;
2219     // cout<<"Went backwards, found "<<before<<endl;
2220     // return us to starting point
2221     while (count--)
2222       cursor.next(key, val);
2223   }
2224   //  cout<<"Now going forward"<<endl;
2225   for (int count = 0;; ++count) {
2226     if ((count && cursor.next(key, val)) || co.getDomainID(key.getNoStripHeader<StringView>()) != id) {
2227       // cout <<"Hit end of database or zone, finding first hash then in zone "<<id<<endl;
2228       if (cursor.lower_bound(co(id), key, val)) {
2229         // cout<<"hit end of zone find when we shouldn't"<<endl;
2230         // means database is wrong, nothing we can do
2231         return false;
2232       }
2233       for (;;) {
2234         if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2235           serFromString(val.get<StringView>(), lrr);
2236           if (!lrr.ttl)
2237             break;
2238         }
2239
2240         if (cursor.next(key, val)) {
2241           // means database is wrong, nothing we can do
2242           // cout<<"hit end of zone when we shouldn't 2"<<endl;
2243           return false;
2244         }
2245         // cout << "Next.. "<<endl;
2246       }
2247       after = co.getQName(key.getNoStripHeader<StringView>());
2248
2249       // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2250       return true;
2251     }
2252
2253     // cout<<"After "<<co.getQName(key.get<StringView>()) <<endl;
2254     if (co.getQType(key.getNoStripHeader<StringView>()) == QType::NSEC3) {
2255       serFromString(val.get<StringView>(), lrr);
2256       if (!lrr.ttl) {
2257         break;
2258       }
2259     }
2260   }
2261   after = co.getQName(key.getNoStripHeader<StringView>());
2262   // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2263   return true;
2264 }
2265
2266 bool LMDBBackend::getBeforeAndAfterNames(uint32_t id, const DNSName& zonenameU, const DNSName& qname, DNSName& before, DNSName& after)
2267 {
2268   DNSName zonename = zonenameU.makeLowerCase();
2269   //  cout << __PRETTY_FUNCTION__<< ": "<<id <<", "<<zonename << ", '"<<qname<<"'"<<endl;
2270
2271   auto txn = getRecordsROTransaction(id);
2272   compoundOrdername co;
2273   DNSName qname2 = qname.makeRelative(zonename);
2274   string matchkey = co(id, qname2);
2275   auto cursor = txn->txn->getCursor(txn->db->dbi);
2276   MDBOutVal key, val;
2277   // cout<<"Lower_bound for "<<qname2<<endl;
2278   if (cursor.lower_bound(matchkey, key, val)) {
2279     // cout << "Hit end of database, bummer"<<endl;
2280     cursor.last(key, val);
2281     if (co.getDomainID(key.getNoStripHeader<string_view>()) == id) {
2282       before = co.getQName(key.getNoStripHeader<string_view>()) + zonename;
2283       after = zonename;
2284     }
2285     // else
2286     // cout << "We were at end of database, but this zone is not there?!"<<endl;
2287     return true;
2288   }
2289   // cout<<"Cursor is at "<<co.getQName(key.get<string_view>()) <<", in zone id "<<co.getDomainID(key.get<string_view>())<< endl;
2290
2291   if (co.getQType(key.getNoStripHeader<string_view>()).getCode() && co.getDomainID(key.getNoStripHeader<string_view>()) == id && co.getQName(key.getNoStripHeader<string_view>()) == qname2) { // don't match ENTs
2292     // cout << "Had an exact match!"<<endl;
2293     before = qname2 + zonename;
2294     int rc;
2295     for (;;) {
2296       rc = cursor.next(key, val);
2297       if (rc)
2298         break;
2299
2300       if (co.getDomainID(key.getNoStripHeader<string_view>()) == id && key.getNoStripHeader<StringView>().rfind(matchkey, 0) == 0)
2301         continue;
2302       LMDBResourceRecord lrr;
2303       serFromString(val.get<StringView>(), lrr);
2304       if (co.getQType(key.getNoStripHeader<string_view>()).getCode() && (lrr.auth || co.getQType(key.getNoStripHeader<string_view>()).getCode() == QType::NS))
2305         break;
2306     }
2307     if (rc || co.getDomainID(key.getNoStripHeader<string_view>()) != id) {
2308       // cout << "We hit the end of the zone or database. 'after' is apex" << endl;
2309       after = zonename;
2310       return false;
2311     }
2312     after = co.getQName(key.getNoStripHeader<string_view>()) + zonename;
2313     return true;
2314   }
2315
2316   if (co.getDomainID(key.getNoStripHeader<string_view>()) != id) {
2317     // cout << "Ended up in next zone, 'after' is zonename" <<endl;
2318     after = zonename;
2319     // cout << "Now hunting for previous" << endl;
2320     int rc;
2321     for (;;) {
2322       rc = cursor.prev(key, val);
2323       if (rc) {
2324         // cout<<"Reversed into zone, but got not found from lmdb" <<endl;
2325         return false;
2326       }
2327
2328       if (co.getDomainID(key.getNoStripHeader<string_view>()) != id) {
2329         // cout<<"Reversed into zone, but found wrong zone id " << co.getDomainID(key.getNoStripHeader<string_view>()) << " != "<<id<<endl;
2330         // "this can't happen"
2331         return false;
2332       }
2333       LMDBResourceRecord lrr;
2334       serFromString(val.get<StringView>(), lrr);
2335       if (co.getQType(key.getNoStripHeader<string_view>()).getCode() && (lrr.auth || co.getQType(key.getNoStripHeader<string_view>()).getCode() == QType::NS))
2336         break;
2337     }
2338
2339     before = co.getQName(key.getNoStripHeader<string_view>()) + zonename;
2340     // cout<<"Found: "<< before<<endl;
2341     return true;
2342   }
2343
2344   // cout <<"We ended up after "<<qname<<", on "<<co.getQName(key.getNoStripHeader<string_view>())<<endl;
2345
2346   int skips = 0;
2347   for (;;) {
2348     LMDBResourceRecord lrr;
2349     serFromString(val.get<StringView>(), lrr);
2350     if (co.getQType(key.getNoStripHeader<string_view>()).getCode() && (lrr.auth || co.getQType(key.getNoStripHeader<string_view>()).getCode() == QType::NS)) {
2351       after = co.getQName(key.getNoStripHeader<string_view>()) + zonename;
2352       // cout <<"Found auth ("<<lrr.auth<<") or an NS record "<<after<<", type: "<<co.getQType(key.getNoStripHeader<string_view>()).toString()<<", ttl = "<<lrr.ttl<<endl;
2353       // cout << makeHexDump(val.get<string>()) << endl;
2354       break;
2355     }
2356     // cout <<"  oops, " << co.getQName(key.getNoStripHeader<string_view>()) << " was not auth "<<lrr.auth<< " type=" << lrr.qtype.toString()<<" or NS, so need to skip ahead a bit more" << endl;
2357     int rc = cursor.next(key, val);
2358     if (!rc)
2359       ++skips;
2360     if (rc || co.getDomainID(key.getNoStripHeader<string_view>()) != id) {
2361       // cout << "  oops, hit end of database or zone. This means after is apex" <<endl;
2362       after = zonename;
2363       break;
2364     }
2365   }
2366   // go back to where we were
2367   while (skips--)
2368     cursor.prev(key, val);
2369
2370   for (;;) {
2371     int rc = cursor.prev(key, val);
2372     if (rc || co.getDomainID(key.getNoStripHeader<string_view>()) != id) {
2373       // XX I don't think this case can happen
2374       // cout << "We hit the beginning of the zone or database.. now what" << endl;
2375       return false;
2376     }
2377     before = co.getQName(key.getNoStripHeader<string_view>()) + zonename;
2378     LMDBResourceRecord lrr;
2379     serFromString(val.get<string_view>(), lrr);
2380     // cout<<"And before to "<<before<<", auth = "<<rr.auth<<endl;
2381     if (co.getQType(key.getNoStripHeader<string_view>()).getCode() && (lrr.auth || co.getQType(key.getNoStripHeader<string_view>()) == QType::NS))
2382       break;
2383     // cout << "Oops, that was wrong, go back one more"<<endl;
2384   }
2385
2386   return true;
2387 }
2388
2389 bool LMDBBackend::updateDNSSECOrderNameAndAuth(uint32_t domain_id, const DNSName& qname, const DNSName& ordername, bool auth, const uint16_t qtype)
2390 {
2391   //  cout << __PRETTY_FUNCTION__<< ": "<< domain_id <<", '"<<qname <<"', '"<<ordername<<"', "<<auth<< ", " << qtype << endl;
2392   shared_ptr<RecordsRWTransaction> txn;
2393   bool needCommit = false;
2394   if (d_rwtxn && d_transactiondomainid == domain_id) {
2395     txn = d_rwtxn;
2396     //    cout<<"Reusing open transaction"<<endl;
2397   }
2398   else {
2399     //    cout<<"Making a new RW txn for " << __PRETTY_FUNCTION__ <<endl;
2400     txn = getRecordsRWTransaction(domain_id);
2401     needCommit = true;
2402   }
2403
2404   DomainInfo di;
2405   if (!d_tdomains->getROTransaction().get(domain_id, di)) {
2406     //    cout<<"Could not find domain_id "<<domain_id <<endl;
2407     return false;
2408   }
2409
2410   DNSName rel = qname.makeRelative(di.zone);
2411
2412   compoundOrdername co;
2413   string matchkey = co(domain_id, rel);
2414
2415   auto cursor = txn->txn->getCursor(txn->db->dbi);
2416   MDBOutVal key, val;
2417   if (cursor.lower_bound(matchkey, key, val)) {
2418     // cout << "Could not find anything"<<endl;
2419     return false;
2420   }
2421
2422   bool hasOrderName = !ordername.empty();
2423   bool needNSEC3 = hasOrderName;
2424
2425   for (; key.getNoStripHeader<StringView>().rfind(matchkey, 0) == 0;) {
2426     vector<LMDBResourceRecord> lrrs;
2427
2428     if (co.getQType(key.getNoStripHeader<StringView>()) != QType::NSEC3) {
2429       serFromString(val.get<StringView>(), lrrs);
2430       bool changed = false;
2431       vector<LMDBResourceRecord> newRRs;
2432       for (auto& lrr : lrrs) {
2433         lrr.qtype = co.getQType(key.getNoStripHeader<StringView>());
2434         if (!needNSEC3 && qtype != QType::ANY) {
2435           needNSEC3 = (lrr.ordername && QType(qtype) != lrr.qtype);
2436         }
2437
2438         if ((qtype == QType::ANY || QType(qtype) == lrr.qtype) && (lrr.ordername != hasOrderName || lrr.auth != auth)) {
2439           lrr.auth = auth;
2440           lrr.ordername = hasOrderName;
2441           changed = true;
2442         }
2443         newRRs.push_back(std::move(lrr));
2444       }
2445       if (changed) {
2446         cursor.put(key, serToString(newRRs));
2447       }
2448     }
2449
2450     if (cursor.next(key, val))
2451       break;
2452   }
2453
2454   bool del = false;
2455   LMDBResourceRecord lrr;
2456   matchkey = co(domain_id, rel, QType::NSEC3);
2457   // cerr<<"here qname="<<qname<<" ordername="<<ordername<<" qtype="<<qtype<<" matchkey="<<makeHexDump(matchkey)<<endl;
2458   int txngetrc;
2459   if (!(txngetrc = txn->txn->get(txn->db->dbi, matchkey, val))) {
2460     serFromString(val.get<string_view>(), lrr);
2461
2462     if (needNSEC3) {
2463       if (hasOrderName && lrr.content != ordername.toDNSStringLC()) {
2464         del = true;
2465       }
2466     }
2467     else {
2468       del = true;
2469     }
2470     if (del) {
2471       txn->txn->del(txn->db->dbi, co(domain_id, DNSName(lrr.content.c_str(), lrr.content.size(), 0, false), QType::NSEC3));
2472       txn->txn->del(txn->db->dbi, matchkey);
2473     }
2474   }
2475   else {
2476     del = true;
2477   }
2478
2479   if (hasOrderName && del) {
2480     matchkey = co(domain_id, rel, QType::NSEC3);
2481
2482     lrr.ttl = 0;
2483     lrr.auth = 0;
2484     lrr.content = rel.toDNSStringLC();
2485
2486     string str = serToString(lrr);
2487     txn->txn->put(txn->db->dbi, co(domain_id, ordername, QType::NSEC3), str);
2488     lrr.ttl = 1;
2489     lrr.content = ordername.toDNSStringLC();
2490     str = serToString(lrr);
2491     txn->txn->put(txn->db->dbi, matchkey, str); // 2
2492   }
2493
2494   if (needCommit)
2495     txn->txn->commit();
2496   return false;
2497 }
2498
2499 bool LMDBBackend::updateEmptyNonTerminals(uint32_t domain_id, set<DNSName>& insert, set<DNSName>& erase, bool remove)
2500 {
2501   // cout << __PRETTY_FUNCTION__<< ": "<< domain_id << ", insert.size() "<<insert.size()<<", "<<erase.size()<<", " <<remove<<endl;
2502
2503   bool needCommit = false;
2504   shared_ptr<RecordsRWTransaction> txn;
2505   if (d_rwtxn && d_transactiondomainid == domain_id) {
2506     txn = d_rwtxn;
2507     //    cout<<"Reusing open transaction"<<endl;
2508   }
2509   else {
2510     //    cout<<"Making a new RW txn for delete domain"<<endl;
2511     txn = getRecordsRWTransaction(domain_id);
2512     needCommit = true;
2513   }
2514
2515   // if remove is set, all ENTs should be removed & nothing else should be done
2516   if (remove) {
2517     deleteDomainRecords(*txn, domain_id, 0);
2518   }
2519   else {
2520     DomainInfo di;
2521     auto rotxn = d_tdomains->getROTransaction();
2522     if (!rotxn.get(domain_id, di)) {
2523       // cout <<"No such domain with id "<<domain_id<<endl;
2524       return false;
2525     }
2526     compoundOrdername co;
2527     for (const auto& n : insert) {
2528       LMDBResourceRecord lrr;
2529       lrr.qname = n.makeRelative(di.zone);
2530       lrr.ttl = 0;
2531       lrr.auth = true;
2532
2533       std::string ser = serToString(lrr);
2534
2535       txn->txn->put(txn->db->dbi, co(domain_id, lrr.qname, 0), ser);
2536
2537       // cout <<" +"<<n<<endl;
2538     }
2539     for (auto n : erase) {
2540       // cout <<" -"<<n<<endl;
2541       n.makeUsRelative(di.zone);
2542       txn->txn->del(txn->db->dbi, co(domain_id, n, 0));
2543     }
2544   }
2545   if (needCommit)
2546     txn->txn->commit();
2547   return false;
2548 }
2549
2550 /* TSIG */
2551 bool LMDBBackend::getTSIGKey(const DNSName& name, DNSName& algorithm, string& content)
2552 {
2553   auto txn = d_ttsig->getROTransaction();
2554   LMDBIDvec ids;
2555   txn.get_multi<0>(name, ids);
2556
2557   TSIGKey key;
2558   for (auto id : ids) {
2559     if (txn.get(id, key)) {
2560       if (algorithm.empty() || algorithm == DNSName(key.algorithm)) {
2561         algorithm = DNSName(key.algorithm);
2562         content = key.key;
2563       }
2564     }
2565   }
2566
2567   return true;
2568 }
2569
2570 // this deletes an old key if it has the same algorithm
2571 bool LMDBBackend::setTSIGKey(const DNSName& name, const DNSName& algorithm, const string& content)
2572 {
2573   auto txn = d_ttsig->getRWTransaction();
2574
2575   LMDBIDvec ids;
2576   txn.get_multi<0>(name, ids);
2577
2578   TSIGKey key;
2579   for (auto id : ids) {
2580     if (txn.get(id, key)) {
2581       if (key.algorithm == algorithm) {
2582         txn.del(id);
2583       }
2584     }
2585   }
2586
2587   TSIGKey tk;
2588   tk.name = name;
2589   tk.algorithm = algorithm;
2590   tk.key = content;
2591
2592   txn.put(tk, 0, d_random_ids);
2593   txn.commit();
2594
2595   return true;
2596 }
2597 bool LMDBBackend::deleteTSIGKey(const DNSName& name)
2598 {
2599   auto txn = d_ttsig->getRWTransaction();
2600
2601   LMDBIDvec ids;
2602   txn.get_multi<0>(name, ids);
2603
2604   TSIGKey key;
2605
2606   for (auto id : ids) {
2607     if (txn.get(id, key)) {
2608       txn.del(id);
2609     }
2610   }
2611   txn.commit();
2612   return true;
2613 }
2614 bool LMDBBackend::getTSIGKeys(std::vector<struct TSIGKey>& keys)
2615 {
2616   auto txn = d_ttsig->getROTransaction();
2617
2618   keys.clear();
2619   for (auto iter = txn.begin(); iter != txn.end(); ++iter) {
2620     keys.push_back(*iter);
2621   }
2622   return true;
2623 }
2624
2625 string LMDBBackend::directBackendCmd(const string& query)
2626 {
2627   ostringstream ret, usage;
2628
2629   usage << "info                               show some information about the database" << endl;
2630   usage << "index check domains                check zone<>ID indexes" << endl;
2631   usage << "index refresh domains <ID>         refresh index for zone with this ID" << endl;
2632   usage << "index refresh-all domains          refresh index for all zones with disconnected indexes" << endl;
2633   vector<string> argv;
2634   stringtok(argv, query);
2635
2636   if (argv.empty()) {
2637     return usage.str();
2638   }
2639
2640   string& cmd = argv[0];
2641
2642   if (cmd == "help") {
2643     return usage.str();
2644   }
2645
2646   if (cmd == "info") {
2647     ret << "shards: " << s_shards << endl;
2648     ret << "schemaversion: " << SCHEMAVERSION << endl;
2649
2650     return ret.str();
2651   }
2652
2653   if (cmd == "index") {
2654     if (argv.size() < 2) {
2655       return "need an index subcommand\n";
2656     }
2657
2658     string& subcmd = argv[1];
2659
2660     if (subcmd == "check" || subcmd == "refresh-all") {
2661       bool refresh = false;
2662
2663       if (subcmd == "refresh-all") {
2664         refresh = true;
2665       }
2666
2667       if (argv.size() < 3) {
2668         return "need an index name\n";
2669       }
2670
2671       if (argv[2] != "domains") {
2672         return "can only check the domains index\n";
2673       }
2674
2675       vector<uint32_t> refreshQueue;
2676
2677       {
2678         auto txn = d_tdomains->getROTransaction();
2679
2680         for (auto iter = txn.begin(); iter != txn.end(); ++iter) {
2681           DomainInfo di = *iter;
2682
2683           auto id = iter.getID();
2684
2685           LMDBIDvec ids;
2686           txn.get_multi<0>(di.zone, ids);
2687
2688           if (ids.size() != 1) {
2689             ret << "ID->zone index has " << id << "->" << di.zone << ", ";
2690
2691             if (ids.empty()) {
2692               ret << "zone->ID index has no entry for " << di.zone << endl;
2693               if (refresh) {
2694                 refreshQueue.push_back(id);
2695               }
2696               else {
2697                 ret << "  suggested remedy: index refresh domains " << id << endl;
2698               }
2699             }
2700             else {
2701               // ids.size() > 1
2702               ret << "zone->ID index has multiple entries for " << di.zone << ": ";
2703               for (auto id_ : ids) {
2704                 ret << id_ << " ";
2705               }
2706               ret << endl;
2707             }
2708           }
2709         }
2710       }
2711
2712       if (refresh) {
2713         for (const auto& id : refreshQueue) {
2714           if (genChangeDomain(id, [](DomainInfo& /* di */) {})) {
2715             ret << "refreshed " << id << endl;
2716           }
2717           else {
2718             ret << "failed to refresh " << id << endl;
2719           }
2720         }
2721       }
2722       return ret.str();
2723     }
2724     if (subcmd == "refresh") {
2725       // index refresh domains 12345
2726       if (argv.size() < 4) {
2727         return "usage: index refresh domains <ID>\n";
2728       }
2729
2730       if (argv[2] != "domains") {
2731         return "can only refresh in the domains index\n";
2732       }
2733
2734       uint32_t id = 0;
2735
2736       try {
2737         id = pdns::checked_stoi<uint32_t>(argv[3]);
2738       }
2739       catch (const std::out_of_range& e) {
2740         return "ID out of range\n";
2741       }
2742
2743       if (genChangeDomain(id, [](DomainInfo& /* di */) {})) {
2744         ret << "refreshed" << endl;
2745       }
2746       else {
2747         ret << "failed" << endl;
2748       }
2749       return ret.str();
2750     }
2751   }
2752
2753   return "unknown lmdbbackend command\n";
2754 }
2755
2756 class LMDBFactory : public BackendFactory
2757 {
2758 public:
2759   LMDBFactory() :
2760     BackendFactory("lmdb") {}
2761   void declareArguments(const string& suffix = "") override
2762   {
2763     declare(suffix, "filename", "Filename for lmdb", "./pdns.lmdb");
2764     declare(suffix, "sync-mode", "Synchronisation mode: nosync, nometasync, mapasync, sync", "mapasync");
2765     // there just is no room for more on 32 bit
2766     declare(suffix, "shards", "Records database will be split into this number of shards", (sizeof(void*) == 4) ? "2" : "64");
2767     declare(suffix, "schema-version", "Maximum allowed schema version to run on this DB. If a lower version is found, auto update is performed", std::to_string(SCHEMAVERSION));
2768     declare(suffix, "random-ids", "Numeric IDs inside the database are generated randomly instead of sequentially", "no");
2769     declare(suffix, "map-size", "LMDB map size in megabytes", (sizeof(void*) == 4) ? "100" : "16000");
2770     declare(suffix, "flag-deleted", "Flag entries on deletion instead of deleting them", "no");
2771     declare(suffix, "lightning-stream", "Run in Lightning Stream compatible mode", "no");
2772   }
2773   DNSBackend* make(const string& suffix = "") override
2774   {
2775     return new LMDBBackend(suffix);
2776   }
2777 };
2778
2779 /* THIRD PART */
2780
2781 class LMDBLoader
2782 {
2783 public:
2784   LMDBLoader()
2785   {
2786     BackendMakers().report(new LMDBFactory);
2787     g_log << Logger::Info << "[lmdbbackend] This is the lmdb backend version " VERSION
2788 #ifndef REPRODUCIBLE
2789           << " (" __DATE__ " " __TIME__ ")"
2790 #endif
2791           << " reporting" << endl;
2792   }
2793 };
2794
2795 static LMDBLoader randomLoader;