From: Peter van Dijk Date: Fri, 4 Feb 2022 15:08:18 +0000 (+0100) Subject: auth lmdb: add random ID generation feature X-Git-Tag: auth-4.7.0-alpha1~2^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a824a629ab13d2d5516221ec47a849755b221e5f;p=thirdparty%2Fpdns.git auth lmdb: add random ID generation feature --- diff --git a/docs/backends/lmdb.rst b/docs/backends/lmdb.rst index 61b2b3cefd..dc0d96fba2 100644 --- a/docs/backends/lmdb.rst +++ b/docs/backends/lmdb.rst @@ -84,6 +84,16 @@ PowerDNS Version LMDB Schema version 4.4.x and up 3 ================ =================== +.. _settings-lmdb-random-ids: + +``lmdb-random-ids`` +^^^^^^^^^^^^^^^^^^^ + + .. versionadded:: 4.7.0 + +Numeric IDs inside the database are generated randomly instead of sequentially. +If some external process is synchronising databases between systems, this will avoid conflicts when objects (domains, keys, etc.) get added. + LMDB Structure -------------- diff --git a/ext/lmdb-safe/lmdb-typed.cc b/ext/lmdb-safe/lmdb-typed.cc index b4ecd38746..a366da2276 100644 --- a/ext/lmdb-safe/lmdb-typed.cc +++ b/ext/lmdb-safe/lmdb-typed.cc @@ -1,4 +1,5 @@ #include "lmdb-typed.hh" +#include "pdns/dns_random.hh" unsigned int MDBGetMaxID(MDBRWTransaction& txn, MDBDbi& dbi) { @@ -11,4 +12,21 @@ unsigned int MDBGetMaxID(MDBRWTransaction& txn, MDBDbi& dbi) return maxid; } +unsigned int MDBGetRandomID(MDBRWTransaction& txn, MDBDbi& dbi) +{ + auto cursor = txn->getRWCursor(dbi); + unsigned int id; + for(int attempts=0; attempts<20; attempts++) { + MDBOutVal key, content; + + // dns_random generates a random number in [0..type_max-1]. We add 1 to avoid 0 and allow type_max. + // 0 is avoided because the put() interface uses it to mean "please allocate a number for me" + id = dns_random(std::numeric_limits::max()) + 1; + if(cursor.find(MDBInVal(id), key, content)) { + return id; + } + } + throw std::runtime_error("MDBGetRandomID() could not assign an unused random ID"); +} + diff --git a/ext/lmdb-safe/lmdb-typed.hh b/ext/lmdb-safe/lmdb-typed.hh index 4887aa7e6a..8fdce35227 100644 --- a/ext/lmdb-safe/lmdb-typed.hh +++ b/ext/lmdb-safe/lmdb-typed.hh @@ -35,6 +35,12 @@ */ unsigned int MDBGetMaxID(MDBRWTransaction& txn, MDBDbi& dbi); +/** Return a randomly generated ID that is unique and not zero. + May throw if the database is very full. +*/ +unsigned int MDBGetRandomID(MDBRWTransaction& txn, MDBDbi& dbi); + + /** This is our serialization interface. You can define your own serToString for your type if you know better */ @@ -588,12 +594,17 @@ public: } // insert something, with possibly a specific id - uint32_t put(const T& t, uint32_t id=0) + uint32_t put(const T& t, uint32_t id, bool random_ids=false) { int flags = 0; if(!id) { - id = MDBGetMaxID(*d_txn, d_parent->d_main) + 1; - flags = MDB_APPEND; + if(random_ids) { + id = MDBGetRandomID(*d_txn, d_parent->d_main); + } + else { + id = MDBGetMaxID(*d_txn, d_parent->d_main) + 1; + flags = MDB_APPEND; + } } (*d_txn)->put(d_parent->d_main, id, serToString(t), flags); diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc index 7c3173a174..52c47cafaa 100644 --- a/modules/lmdbbackend/lmdbbackend.cc +++ b/modules/lmdbbackend/lmdbbackend.cc @@ -69,6 +69,8 @@ LMDBBackend::LMDBBackend(const std::string& suffix) string syncMode = toLower(getArg("sync-mode")); + d_random_ids = mustDo("random-ids"); + if (syncMode == "nosync") d_asyncFlag = MDB_NOSYNC; else if (syncMode == "nometasync") @@ -978,7 +980,7 @@ bool LMDBBackend::createDomain(const DNSName& domain, const DomainInfo::DomainKi di.masters = masters; di.account = account; - txn.put(di); + txn.put(di, 0, d_random_ids); txn.commit(); } @@ -1069,7 +1071,7 @@ bool LMDBBackend::setDomainMetadata(const DNSName& name, const std::string& kind for (const auto& m : meta) { DomainMeta dm{name, kind, m}; - txn.put(dm); + txn.put(dm, 0, d_random_ids); } txn.commit(); return true; @@ -1106,7 +1108,7 @@ bool LMDBBackend::addDomainKey(const DNSName& name, const KeyData& key, int64_t& { auto txn = d_tkdb->getRWTransaction(); KeyDataDB kdb{name, key.content, key.flags, key.active, key.published}; - id = txn.put(kdb); + id = txn.put(kdb, 0, d_random_ids); txn.commit(); return true; @@ -1712,7 +1714,7 @@ bool LMDBBackend::setTSIGKey(const DNSName& name, const DNSName& algorithm, cons tk.algorithm = algorithm; tk.key = content; - txn.put(tk); + txn.put(tk, 0, d_random_ids); txn.commit(); return true; @@ -1751,6 +1753,7 @@ public: // there just is no room for more on 32 bit declare(suffix, "shards", "Records database will be split into this number of shards", (sizeof(long) == 4) ? "2" : "64"); declare(suffix, "schema-version", "Maximum allowed schema version to run on this DB. If a lower version is found, auto update is performed", std::to_string(SCHEMAVERSION)); + declare(suffix, "random-ids", "Numeric IDs inside the database are generated randomly instead of sequentially", "no"); } DNSBackend* make(const string& suffix = "") override { diff --git a/modules/lmdbbackend/lmdbbackend.hh b/modules/lmdbbackend/lmdbbackend.hh index 6e26e99b24..99244a9ef8 100644 --- a/modules/lmdbbackend/lmdbbackend.hh +++ b/modules/lmdbbackend/lmdbbackend.hh @@ -311,5 +311,6 @@ private: DNSName d_transactiondomain; uint32_t d_transactiondomainid; bool d_dolog; + bool d_random_ids; DTime d_dtime; // used only for logging }; diff --git a/regression-tests.nobackend/lmdb-id-generation/command b/regression-tests.nobackend/lmdb-id-generation/command new file mode 100755 index 0000000000..a6775684d2 --- /dev/null +++ b/regression-tests.nobackend/lmdb-id-generation/command @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -e +if [ "${PDNS_DEBUG}" = "YES" ]; then + set -x +fi + +rootPath=$(readlink -f $(dirname $0)) + +for random in no yes +do + workdir=$(mktemp -d) + + cat << EOF > "${workdir}/pdns-lmdb.conf" + module-dir=../regression-tests/modules + launch=lmdb + lmdb-filename=${workdir}/pdns.lmdb + lmdb-shards=2 + lmdb-random-ids=${random} +EOF + + $PDNSUTIL --config-dir="${workdir}" --config-name=lmdb create-zone example.com + ids=$( for i in a b c ; do $PDNSUTIL --config-dir="${workdir}" --config-name=lmdb add-zone-key example.com ksk ; done | xargs ) + if [ $random = no ] + then + if [ "$ids" = "1 2 3" ] + then + echo "sequential generator generated 1 2 3 correctly" + else + echo "sequential generator: expected 1 2 3, but got $ids" + fi + else + if [ "$ids" = "1 2 3" ] + then + echo "random generator generated 1 2 3, this is unlikely" + else + echo "random generator generated something other than 1 2 3, good" + fi + fi +done diff --git a/regression-tests.nobackend/lmdb-id-generation/description b/regression-tests.nobackend/lmdb-id-generation/description new file mode 100644 index 0000000000..1bbcb42549 --- /dev/null +++ b/regression-tests.nobackend/lmdb-id-generation/description @@ -0,0 +1 @@ +This test verifies that LMDB ID generation is sequential, unless configured with lmdb-random-ids=yes diff --git a/regression-tests.nobackend/lmdb-id-generation/expected_result b/regression-tests.nobackend/lmdb-id-generation/expected_result new file mode 100644 index 0000000000..127b73f596 --- /dev/null +++ b/regression-tests.nobackend/lmdb-id-generation/expected_result @@ -0,0 +1,2 @@ +sequential generator generated 1 2 3 correctly +random generator generated something other than 1 2 3, good