From: Peter van Dijk Date: Tue, 15 Apr 2025 13:13:09 +0000 (+0200) Subject: implement LMDB schema upgrade to v6 X-Git-Tag: auth-5.0.0-alpha1~1^2~17 X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=6663881054b8f580609e3c982b191863687108f3;p=thirdparty%2Fpdns.git implement LMDB schema upgrade to v6 --- diff --git a/docs/backends/lmdb.rst b/docs/backends/lmdb.rst index 427091ba73..5a91641289 100644 --- a/docs/backends/lmdb.rst +++ b/docs/backends/lmdb.rst @@ -92,6 +92,7 @@ PowerDNS Version LMDB Schema version 4.4.x to 4.6.x 3 4.7.x and up 4 4.8.x and up 5 +5.0.x and up 6 ================ =================== .. _settings-lmdb-random-ids: diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc index a436965798..acfac59dcf 100644 --- a/modules/lmdbbackend/lmdbbackend.cc +++ b/modules/lmdbbackend/lmdbbackend.cc @@ -54,7 +54,7 @@ #include #endif -#define SCHEMAVERSION 5 +constexpr unsigned int SCHEMAVERSION{6}; // List the class version here. Default is 0 BOOST_CLASS_VERSION(LMDBBackend::KeyDataDB, 1) @@ -111,9 +111,9 @@ std::pair LMDBBackend::getSchemaVersionAndShards(std::string if (retCode != 0) { if (retCode == MDB_NOTFOUND) { // this means nothing has been inited yet - // we pretend this means 5 + // we pretend this means the latest schema mdb_txn_abort(txn); - return {5U, 0U}; + return {SCHEMAVERSION, 0U}; } mdb_txn_abort(txn); throw std::runtime_error("mdb_dbi_open failed"); @@ -130,9 +130,9 @@ std::pair LMDBBackend::getSchemaVersionAndShards(std::string if (retCode != 0) { if (retCode == MDB_NOTFOUND) { // this means nothing has been inited yet - // we pretend this means 5 + // we pretend this means the latest schema mdb_txn_abort(txn); - return {5U, 0U}; + return {SCHEMAVERSION, 0U}; } throw std::runtime_error("mdb_get pdns.schemaversion failed"); @@ -145,7 +145,7 @@ std::pair LMDBBackend::getSchemaVersionAndShards(std::string memcpy(&schemaversion, data.mv_data, data.mv_size); } else if (data.mv_size >= LMDBLS::LS_MIN_HEADER_SIZE + sizeof(schemaversion)) { - // schemaversion presumably is 5, stored in 32 bits, network order, after the LS header + // schemaversion is >= 5, stored in 32 bits, network order, after the LS header // FIXME: get actual header size (including extension blocks) instead of just reading from the back // FIXME: add a test for reading schemaversion and shards (and actual data, later) when there are variably sized headers @@ -663,6 +663,15 @@ bool LMDBBackend::upgradeToSchemav5(std::string& filename) return true; } +bool LMDBBackend::upgradeToSchemav6(std::string& /* filename */) +{ + // a v6 reader can read v5 databases just fine + // so this function currently does nothing + // - except rely on the caller to write '6' to pdns.schemaversion, + // as a v5 reader will be unable to handle domain objects once we've touched them + return true; +} + LMDBBackend::LMDBBackend(const std::string& suffix) { // overlapping domain ids in combination with relative names are a recipe for disaster @@ -720,16 +729,16 @@ LMDBBackend::LMDBBackend(const std::string& suffix) // std::cerr<<"current schema version: "< 0 && currentSchemaVersion < 3) { - throw std::runtime_error("this version of the lmdbbackend can only upgrade from schema v3/v4 to v5. Upgrading from older schemas is not yet supported."); + throw std::runtime_error("this version of the lmdbbackend can only upgrade from schema v3 and up. Upgrading from older schemas is not supported."); } if (currentSchemaVersion == 0) { // no database is present yet, we can just create them - currentSchemaVersion = 5; + currentSchemaVersion = 6; } if (currentSchemaVersion == 3 || currentSchemaVersion == 4) { @@ -739,20 +748,20 @@ LMDBBackend::LMDBBackend(const std::string& suffix) currentSchemaVersion = 5; } - if (currentSchemaVersion != 5) { - throw std::runtime_error("Somehow, we are not at schema version 5. Giving up"); + if (currentSchemaVersion == 5) { + if (!upgradeToSchemav6(filename)) { + throw std::runtime_error("Failed to perform LMDB schema version upgrade from v5 to v6"); + } + currentSchemaVersion = 6; } - d_tdomains = std::make_shared(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, mapSize), "domains_v5"); - d_tmeta = std::make_shared(d_tdomains->getEnv(), "metadata_v5"); - d_tkdb = std::make_shared(d_tdomains->getEnv(), "keydata_v5"); - d_ttsig = std::make_shared(d_tdomains->getEnv(), "tsig_v5"); - d_tnetworks = d_tdomains->getEnv()->openDB("networks_v5", MDB_CREATE); - d_tviews = d_tdomains->getEnv()->openDB("views_v5", MDB_CREATE); - - auto pdnsdbi = d_tdomains->getEnv()->openDB("pdns", MDB_CREATE); + if (currentSchemaVersion != 6) { + throw std::runtime_error("Somehow, we are not at schema version 6. Giving up"); + } + openAllTheDatabases(mapSize); opened = true; + auto pdnsdbi = d_tdomains->getEnv()->openDB("pdns", MDB_CREATE); auto txn = d_tdomains->getEnv()->getRWTransaction(); @@ -796,8 +805,7 @@ LMDBBackend::LMDBBackend(const std::string& suffix) } MDBOutVal _schemaversion{}; - if (txn->get(pdnsdbi, "schemaversion", _schemaversion) != 0) { - // our DB is entirely new, so we need to write the schemaversion + if (txn->get(pdnsdbi, "schemaversion", _schemaversion) != 0 || _schemaversion.get() != currentSchemaVersion) { txn->put(pdnsdbi, "schemaversion", currentSchemaVersion); } txn->commit(); @@ -807,12 +815,7 @@ LMDBBackend::LMDBBackend(const std::string& suffix) } if (!opened) { - d_tdomains = std::make_shared(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, mapSize), "domains_v5"); - d_tmeta = std::make_shared(d_tdomains->getEnv(), "metadata_v5"); - d_tkdb = std::make_shared(d_tdomains->getEnv(), "keydata_v5"); - d_ttsig = std::make_shared(d_tdomains->getEnv(), "tsig_v5"); - d_tnetworks = d_tdomains->getEnv()->openDB("networks_v5", MDB_CREATE); - d_tviews = d_tdomains->getEnv()->openDB("views_v5", MDB_CREATE); + openAllTheDatabases(mapSize); } d_trecords.resize(s_shards); d_dolog = ::arg().mustDo("query-logging"); @@ -835,6 +838,16 @@ LMDBBackend::~LMDBBackend() } } +void LMDBBackend::openAllTheDatabases(uint64_t mapSize) +{ + d_tdomains = std::make_shared(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, mapSize), "domains_v5"); + d_tmeta = std::make_shared(d_tdomains->getEnv(), "metadata_v5"); + d_tkdb = std::make_shared(d_tdomains->getEnv(), "keydata_v5"); + d_ttsig = std::make_shared(d_tdomains->getEnv(), "tsig_v5"); + d_tnetworks = d_tdomains->getEnv()->openDB("networks_v6", MDB_CREATE); + d_tviews = d_tdomains->getEnv()->openDB("views_v6", MDB_CREATE); +} + unsigned int LMDBBackend::getCapabilities() { unsigned int caps = CAP_DNSSEC | CAP_DIRECT | CAP_LIST | CAP_CREATE; diff --git a/modules/lmdbbackend/lmdbbackend.hh b/modules/lmdbbackend/lmdbbackend.hh index 47b2837646..a9aaeb02aa 100644 --- a/modules/lmdbbackend/lmdbbackend.hh +++ b/modules/lmdbbackend/lmdbbackend.hh @@ -172,6 +172,7 @@ public: // functions to use without constructing a backend object static std::pair getSchemaVersionAndShards(std::string& filename); static bool upgradeToSchemav5(std::string& filename); + static bool upgradeToSchemav6(std::string& filename); private: struct compoundOrdername @@ -325,6 +326,7 @@ private: shared_ptr d_rotxn; // for lookup and list shared_ptr d_rwtxn; // for feedrecord within begin/aborttransaction bool d_txnorder{false}; // whether d_rotxn is more recent than d_rwtxn + void openAllTheDatabases(uint64_t mapSize); std::shared_ptr getRecordsRWTransaction(domainid_t id); std::shared_ptr getRecordsROTransaction(domainid_t id, const std::shared_ptr& rwtxn = nullptr); int genChangeDomain(const ZoneName& domain, const std::function& func); diff --git a/pdns/pdnsutil.cc b/pdns/pdnsutil.cc index 12463a79a6..c16862105b 100644 --- a/pdns/pdnsutil.cc +++ b/pdns/pdnsutil.cc @@ -2654,7 +2654,7 @@ static int addOrSetMeta(const ZoneName& zone, const string& kind, const vector& cmds, [[maybe_unused]] const std::string_view synopsis) { - cout << "5" << endl; // FIXME this should reuse the constant from lmdbbackend but that is currently a #define in a .cc + cout << "6" << endl; // FIXME this should reuse the constant from lmdbbackend but that is currently a #define in a .cc return 0; }