auth lmdb: add random ID generation feature

diff --git a/docs/backends/lmdb.rst b/docs/backends/lmdb.rst
index 61b2b3cefdf196137c3ae476d4e9180703ea88f0..dc0d96fba2e6b000ff65dccd47a532e4dc6d5815 100644 (file)
--- a/docs/backends/lmdb.rst
+++ b/docs/backends/lmdb.rst
@@ -84,6 +84,16 @@ PowerDNS Version  LMDB Schema version
 4.4.x and up      3
 ================  ===================
 
+.. _settings-lmdb-random-ids:
+
+``lmdb-random-ids``
+^^^^^^^^^^^^^^^^^^^
+
+  .. versionadded:: 4.7.0
+
+Numeric IDs inside the database are generated randomly instead of sequentially.
+If some external process is synchronising databases between systems, this will avoid conflicts when objects (domains, keys, etc.) get added.
+
 LMDB Structure
 --------------
 

diff --git a/ext/lmdb-safe/lmdb-typed.cc b/ext/lmdb-safe/lmdb-typed.cc
index b4ecd3874624f1207482a452160aacd635cb2357..a366da227600743c70befb43338ae38bf2a59c45 100644 (file)
--- a/ext/lmdb-safe/lmdb-typed.cc
+++ b/ext/lmdb-safe/lmdb-typed.cc
@@ -1,4 +1,5 @@
 #include "lmdb-typed.hh"
+#include "pdns/dns_random.hh"
 
 unsigned int MDBGetMaxID(MDBRWTransaction& txn, MDBDbi& dbi)
 {
@@ -11,4 +12,21 @@ unsigned int MDBGetMaxID(MDBRWTransaction& txn, MDBDbi& dbi)
   return maxid;
 }
 
+unsigned int MDBGetRandomID(MDBRWTransaction& txn, MDBDbi& dbi)
+{
+  auto cursor = txn->getRWCursor(dbi);
+  unsigned int id;
+  for(int attempts=0; attempts<20; attempts++) {
+    MDBOutVal key, content;
+
+    // dns_random generates a random number in [0..type_max-1]. We add 1 to avoid 0 and allow type_max.
+    // 0 is avoided because the put() interface uses it to mean "please allocate a number for me"
+    id = dns_random(std::numeric_limits<decltype(id)>::max()) + 1;
+    if(cursor.find(MDBInVal(id), key, content)) {
+      return id;
+    }
+  }
+  throw std::runtime_error("MDBGetRandomID() could not assign an unused random ID");
+}
+
 

diff --git a/ext/lmdb-safe/lmdb-typed.hh b/ext/lmdb-safe/lmdb-typed.hh
index 4887aa7e6aa4c3ab90c6679458be89897657f517..8fdce352278b366aa67dc12abb98d299f19956e7 100644 (file)
--- a/ext/lmdb-safe/lmdb-typed.hh
+++ b/ext/lmdb-safe/lmdb-typed.hh
@@ -35,6 +35,12 @@
 */
 unsigned int MDBGetMaxID(MDBRWTransaction& txn, MDBDbi& dbi);
 
+/** Return a randomly generated ID that is unique and not zero.
+    May throw if the database is very full.
+*/
+unsigned int MDBGetRandomID(MDBRWTransaction& txn, MDBDbi& dbi);
+
+
 /** This is our serialization interface.
     You can define your own serToString for your type if you know better
 */
@@ -588,12 +594,17 @@ public:
     }
 
     // insert something, with possibly a specific id
-    uint32_t put(const T& t, uint32_t id=0)
+    uint32_t put(const T& t, uint32_t id, bool random_ids=false)
     {
       int flags = 0;
       if(!id) {
-        id = MDBGetMaxID(*d_txn, d_parent->d_main) + 1;
-        flags = MDB_APPEND;
+        if(random_ids) {
+          id = MDBGetRandomID(*d_txn, d_parent->d_main);
+        }
+        else {
+          id = MDBGetMaxID(*d_txn, d_parent->d_main) + 1;
+          flags = MDB_APPEND;
+        }
       }
       (*d_txn)->put(d_parent->d_main, id, serToString(t), flags);
 

diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc
index 7c3173a1747b4d8fcdd57ec6cceb5059283a1510..52c47cafaa0eb57c64823482360d16b927511a91 100644 (file)
--- a/modules/lmdbbackend/lmdbbackend.cc
+++ b/modules/lmdbbackend/lmdbbackend.cc
@@ -69,6 +69,8 @@ LMDBBackend::LMDBBackend(const std::string& suffix)
 
   string syncMode = toLower(getArg("sync-mode"));
 
+  d_random_ids = mustDo("random-ids");
+
   if (syncMode == "nosync")
     d_asyncFlag = MDB_NOSYNC;
   else if (syncMode == "nometasync")
@@ -978,7 +980,7 @@ bool LMDBBackend::createDomain(const DNSName& domain, const DomainInfo::DomainKi
     di.masters = masters;
     di.account = account;
 
-    txn.put(di);
+    txn.put(di, 0, d_random_ids);
     txn.commit();
   }
 
@@ -1069,7 +1071,7 @@ bool LMDBBackend::setDomainMetadata(const DNSName& name, const std::string& kind
 
   for (const auto& m : meta) {
     DomainMeta dm{name, kind, m};
-    txn.put(dm);
+    txn.put(dm, 0, d_random_ids);
   }
   txn.commit();
   return true;
@@ -1106,7 +1108,7 @@ bool LMDBBackend::addDomainKey(const DNSName& name, const KeyData& key, int64_t&
 {
   auto txn = d_tkdb->getRWTransaction();
   KeyDataDB kdb{name, key.content, key.flags, key.active, key.published};
-  id = txn.put(kdb);
+  id = txn.put(kdb, 0, d_random_ids);
   txn.commit();
 
   return true;
@@ -1712,7 +1714,7 @@ bool LMDBBackend::setTSIGKey(const DNSName& name, const DNSName& algorithm, cons
   tk.algorithm = algorithm;
   tk.key = content;
 
-  txn.put(tk);
+  txn.put(tk, 0, d_random_ids);
   txn.commit();
 
   return true;
@@ -1751,6 +1753,7 @@ public:
     // there just is no room for more on 32 bit
     declare(suffix, "shards", "Records database will be split into this number of shards", (sizeof(long) == 4) ? "2" : "64");
     declare(suffix, "schema-version", "Maximum allowed schema version to run on this DB. If a lower version is found, auto update is performed", std::to_string(SCHEMAVERSION));
+    declare(suffix, "random-ids", "Numeric IDs inside the database are generated randomly instead of sequentially", "no");
   }
   DNSBackend* make(const string& suffix = "") override
   {

diff --git a/modules/lmdbbackend/lmdbbackend.hh b/modules/lmdbbackend/lmdbbackend.hh
index 6e26e99b2452ef76afa37b6719bdbe425fc52e45..99244a9ef86f626ed5e5dede5f1b9e6a8bc63573 100644 (file)
--- a/modules/lmdbbackend/lmdbbackend.hh
+++ b/modules/lmdbbackend/lmdbbackend.hh
@@ -311,5 +311,6 @@ private:
   DNSName d_transactiondomain;
   uint32_t d_transactiondomainid;
   bool d_dolog;
+  bool d_random_ids;
   DTime d_dtime; // used only for logging
 };

diff --git a/regression-tests.nobackend/lmdb-id-generation/command b/regression-tests.nobackend/lmdb-id-generation/command
new file mode 100755 (executable)
index 0000000..a677568
--- /dev/null
+++ b/regression-tests.nobackend/lmdb-id-generation/command
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -e
+if [ "${PDNS_DEBUG}" = "YES" ]; then
+  set -x
+fi
+
+rootPath=$(readlink -f $(dirname $0))
+
+for random in no yes
+do
+  workdir=$(mktemp -d)
+
+  cat << EOF > "${workdir}/pdns-lmdb.conf"
+  module-dir=../regression-tests/modules
+  launch=lmdb
+  lmdb-filename=${workdir}/pdns.lmdb
+  lmdb-shards=2
+  lmdb-random-ids=${random}
+EOF
+
+  $PDNSUTIL --config-dir="${workdir}" --config-name=lmdb create-zone example.com
+  ids=$( for i in a b c ; do $PDNSUTIL --config-dir="${workdir}" --config-name=lmdb add-zone-key example.com ksk ; done | xargs )
+  if [ $random = no ]
+  then
+    if [ "$ids" = "1 2 3" ]
+    then
+      echo "sequential generator generated 1 2 3 correctly"
+    else
+      echo "sequential generator: expected 1 2 3, but got $ids"
+    fi
+  else
+    if [ "$ids" = "1 2 3" ]
+    then
+      echo "random generator generated 1 2 3, this is unlikely"
+    else
+      echo "random generator generated something other than 1 2 3, good"
+    fi
+  fi
+done

diff --git a/regression-tests.nobackend/lmdb-id-generation/description b/regression-tests.nobackend/lmdb-id-generation/description
new file mode 100644 (file)
index 0000000..1bbcb42
--- /dev/null
+++ b/regression-tests.nobackend/lmdb-id-generation/description
@@ -0,0 +1 @@
+This test verifies that LMDB ID generation is sequential, unless configured with lmdb-random-ids=yes

diff --git a/regression-tests.nobackend/lmdb-id-generation/expected_result b/regression-tests.nobackend/lmdb-id-generation/expected_result
new file mode 100644 (file)
index 0000000..127b73f
--- /dev/null
+++ b/regression-tests.nobackend/lmdb-id-generation/expected_result
@@ -0,0 +1,2 @@
+sequential generator generated 1 2 3 correctly
+random generator generated something other than 1 2 3, good