implement LMDB schema upgrade to v6

diff --git a/docs/backends/lmdb.rst b/docs/backends/lmdb.rst
index 427091ba73196fca34efe8fb6a73518274e669cf..5a916412894a6f62464696ac562e9b02e49d96d3 100644 (file)
--- a/docs/backends/lmdb.rst
+++ b/docs/backends/lmdb.rst
@@ -92,6 +92,7 @@ PowerDNS Version  LMDB Schema version
 4.4.x to 4.6.x    3
 4.7.x and up      4
 4.8.x and up      5
+5.0.x and up      6
 ================  ===================
 
 .. _settings-lmdb-random-ids:

diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc
index a436965798606e5f672642f5cb6d103c6ada839e..acfac59dcf1ddda195a46a0077ed0ddb69c379d5 100644 (file)
--- a/modules/lmdbbackend/lmdbbackend.cc
+++ b/modules/lmdbbackend/lmdbbackend.cc
@@ -54,7 +54,7 @@
 #include <systemd/sd-daemon.h>
 #endif
 
-#define SCHEMAVERSION 5
+constexpr unsigned int SCHEMAVERSION{6};
 
 // List the class version here. Default is 0
 BOOST_CLASS_VERSION(LMDBBackend::KeyDataDB, 1)
@@ -111,9 +111,9 @@ std::pair<uint32_t, uint32_t> LMDBBackend::getSchemaVersionAndShards(std::string
     if (retCode != 0) {
       if (retCode == MDB_NOTFOUND) {
         // this means nothing has been inited yet
-        // we pretend this means 5
+        // we pretend this means the latest schema
         mdb_txn_abort(txn);
-        return {5U, 0U};
+        return {SCHEMAVERSION, 0U};
       }
       mdb_txn_abort(txn);
       throw std::runtime_error("mdb_dbi_open failed");
@@ -130,9 +130,9 @@ std::pair<uint32_t, uint32_t> LMDBBackend::getSchemaVersionAndShards(std::string
     if (retCode != 0) {
       if (retCode == MDB_NOTFOUND) {
         // this means nothing has been inited yet
-        // we pretend this means 5
+        // we pretend this means the latest schema
         mdb_txn_abort(txn);
-        return {5U, 0U};
+        return {SCHEMAVERSION, 0U};
       }
 
       throw std::runtime_error("mdb_get pdns.schemaversion failed");
@@ -145,7 +145,7 @@ std::pair<uint32_t, uint32_t> LMDBBackend::getSchemaVersionAndShards(std::string
     memcpy(&schemaversion, data.mv_data, data.mv_size);
   }
   else if (data.mv_size >= LMDBLS::LS_MIN_HEADER_SIZE + sizeof(schemaversion)) {
-    // schemaversion presumably is 5, stored in 32 bits, network order, after the LS header
+    // schemaversion is >= 5, stored in 32 bits, network order, after the LS header
 
     // FIXME: get actual header size (including extension blocks) instead of just reading from the back
     // FIXME: add a test for reading schemaversion and shards (and actual data, later) when there are variably sized headers
@@ -663,6 +663,15 @@ bool LMDBBackend::upgradeToSchemav5(std::string& filename)
   return true;
 }
 
+bool LMDBBackend::upgradeToSchemav6(std::string& /* filename */)
+{
+  // a v6 reader can read v5 databases just fine
+  // so this function currently does nothing
+  // - except rely on the caller to write '6' to pdns.schemaversion,
+  // as a v5 reader will be unable to handle domain objects once we've touched them
+  return true;
+}
+
 LMDBBackend::LMDBBackend(const std::string& suffix)
 {
   // overlapping domain ids in combination with relative names are a recipe for disaster
@@ -720,16 +729,16 @@ LMDBBackend::LMDBBackend(const std::string& suffix)
       // std::cerr<<"current schema version: "<<currentSchemaVersion<<", shards="<<currentSchemaVersionAndShards.second<<std::endl;
 
       if (getArgAsNum("schema-version") != SCHEMAVERSION) {
-        throw std::runtime_error("This version of the lmdbbackend only supports schema version 5. Configuration demands a lower version. Not starting up.");
+        throw std::runtime_error("This version of the lmdbbackend only supports schema version 6. Configuration demands a lower version. Not starting up.");
       }
 
       if (currentSchemaVersion > 0 && currentSchemaVersion < 3) {
-        throw std::runtime_error("this version of the lmdbbackend can only upgrade from schema v3/v4 to v5. Upgrading from older schemas is not yet supported.");
+        throw std::runtime_error("this version of the lmdbbackend can only upgrade from schema v3 and up. Upgrading from older schemas is not supported.");
       }
 
       if (currentSchemaVersion == 0) {
         // no database is present yet, we can just create them
-        currentSchemaVersion = 5;
+        currentSchemaVersion = 6;
       }
 
       if (currentSchemaVersion == 3 || currentSchemaVersion == 4) {
@@ -739,20 +748,20 @@ LMDBBackend::LMDBBackend(const std::string& suffix)
         currentSchemaVersion = 5;
       }
 
-      if (currentSchemaVersion != 5) {
-        throw std::runtime_error("Somehow, we are not at schema version 5. Giving up");
+      if (currentSchemaVersion == 5) {
+        if (!upgradeToSchemav6(filename)) {
+          throw std::runtime_error("Failed to perform LMDB schema version upgrade from v5 to v6");
+        }
+        currentSchemaVersion = 6;
       }
 
-      d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, mapSize), "domains_v5");
-      d_tmeta = std::make_shared<tmeta_t>(d_tdomains->getEnv(), "metadata_v5");
-      d_tkdb = std::make_shared<tkdb_t>(d_tdomains->getEnv(), "keydata_v5");
-      d_ttsig = std::make_shared<ttsig_t>(d_tdomains->getEnv(), "tsig_v5");
-      d_tnetworks = d_tdomains->getEnv()->openDB("networks_v5", MDB_CREATE);
-      d_tviews = d_tdomains->getEnv()->openDB("views_v5", MDB_CREATE);
-
-      auto pdnsdbi = d_tdomains->getEnv()->openDB("pdns", MDB_CREATE);
+      if (currentSchemaVersion != 6) {
+        throw std::runtime_error("Somehow, we are not at schema version 6. Giving up");
+      }
 
+      openAllTheDatabases(mapSize);
       opened = true;
+      auto pdnsdbi = d_tdomains->getEnv()->openDB("pdns", MDB_CREATE);
 
       auto txn = d_tdomains->getEnv()->getRWTransaction();
 
@@ -796,8 +805,7 @@ LMDBBackend::LMDBBackend(const std::string& suffix)
       }
 
       MDBOutVal _schemaversion{};
-      if (txn->get(pdnsdbi, "schemaversion", _schemaversion) != 0) {
-        // our DB is entirely new, so we need to write the schemaversion
+      if (txn->get(pdnsdbi, "schemaversion", _schemaversion) != 0 || _schemaversion.get<uint32_t>() != currentSchemaVersion) {
         txn->put(pdnsdbi, "schemaversion", currentSchemaVersion);
       }
       txn->commit();
@@ -807,12 +815,7 @@ LMDBBackend::LMDBBackend(const std::string& suffix)
   }
 
   if (!opened) {
-    d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, mapSize), "domains_v5");
-    d_tmeta = std::make_shared<tmeta_t>(d_tdomains->getEnv(), "metadata_v5");
-    d_tkdb = std::make_shared<tkdb_t>(d_tdomains->getEnv(), "keydata_v5");
-    d_ttsig = std::make_shared<ttsig_t>(d_tdomains->getEnv(), "tsig_v5");
-    d_tnetworks = d_tdomains->getEnv()->openDB("networks_v5", MDB_CREATE);
-    d_tviews = d_tdomains->getEnv()->openDB("views_v5", MDB_CREATE);
+    openAllTheDatabases(mapSize);
   }
   d_trecords.resize(s_shards);
   d_dolog = ::arg().mustDo("query-logging");
@@ -835,6 +838,16 @@ LMDBBackend::~LMDBBackend()
   }
 }
 
+void LMDBBackend::openAllTheDatabases(uint64_t mapSize)
+{
+  d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, mapSize), "domains_v5");
+  d_tmeta = std::make_shared<tmeta_t>(d_tdomains->getEnv(), "metadata_v5");
+  d_tkdb = std::make_shared<tkdb_t>(d_tdomains->getEnv(), "keydata_v5");
+  d_ttsig = std::make_shared<ttsig_t>(d_tdomains->getEnv(), "tsig_v5");
+  d_tnetworks = d_tdomains->getEnv()->openDB("networks_v6", MDB_CREATE);
+  d_tviews = d_tdomains->getEnv()->openDB("views_v6", MDB_CREATE);
+}
+
 unsigned int LMDBBackend::getCapabilities()
 {
   unsigned int caps = CAP_DNSSEC | CAP_DIRECT | CAP_LIST | CAP_CREATE;

diff --git a/modules/lmdbbackend/lmdbbackend.hh b/modules/lmdbbackend/lmdbbackend.hh
index 47b28376468f386086aa8eb3c1b6bacbb0e9a2b3..a9aaeb02aa11920ed0dc089dd2f56809eeef5314 100644 (file)
--- a/modules/lmdbbackend/lmdbbackend.hh
+++ b/modules/lmdbbackend/lmdbbackend.hh
@@ -172,6 +172,7 @@ public:
   // functions to use without constructing a backend object
   static std::pair<uint32_t, uint32_t> getSchemaVersionAndShards(std::string& filename);
   static bool upgradeToSchemav5(std::string& filename);
+  static bool upgradeToSchemav6(std::string& filename);
 
 private:
   struct compoundOrdername
@@ -325,6 +326,7 @@ private:
   shared_ptr<RecordsROTransaction> d_rotxn; // for lookup and list
   shared_ptr<RecordsRWTransaction> d_rwtxn; // for feedrecord within begin/aborttransaction
   bool d_txnorder{false}; // whether d_rotxn is more recent than d_rwtxn
+  void openAllTheDatabases(uint64_t mapSize);
   std::shared_ptr<RecordsRWTransaction> getRecordsRWTransaction(domainid_t id);
   std::shared_ptr<RecordsROTransaction> getRecordsROTransaction(domainid_t id, const std::shared_ptr<LMDBBackend::RecordsRWTransaction>& rwtxn = nullptr);
   int genChangeDomain(const ZoneName& domain, const std::function<void(DomainInfo&)>& func);

diff --git a/pdns/pdnsutil.cc b/pdns/pdnsutil.cc
index 12463a79a6c91ebcc554f8c501fdc05bea3a1c4f..c16862105beac52ac05f171762e5298429f41929 100644 (file)
--- a/pdns/pdnsutil.cc
+++ b/pdns/pdnsutil.cc
@@ -2654,7 +2654,7 @@ static int addOrSetMeta(const ZoneName& zone, const string& kind, const vector<s
 
 static int lmdbGetBackendVersion([[maybe_unused]] vector<string>& cmds, [[maybe_unused]] const std::string_view synopsis)
 {
-  cout << "5" << endl; // FIXME this should reuse the constant from lmdbbackend but that is currently a #define in a .cc
+  cout << "6" << endl; // FIXME this should reuse the constant from lmdbbackend but that is currently a #define in a .cc
   return 0;
 }