auth lmdb: make map size configurable

diff --git a/docs/backends/lmdb.rst b/docs/backends/lmdb.rst
index dc0d96fba2e6b000ff65dccd47a532e4dc6d5815..936e13d413e72777b7dc88e109da980e70762230 100644 (file)
--- a/docs/backends/lmdb.rst
+++ b/docs/backends/lmdb.rst
@@ -94,6 +94,17 @@ PowerDNS Version  LMDB Schema version
 Numeric IDs inside the database are generated randomly instead of sequentially.
 If some external process is synchronising databases between systems, this will avoid conflicts when objects (domains, keys, etc.) get added.
 
+.. _settings-lmdb-map-size:
+
+``lmdb-random-ids``
+^^^^^^^^^^^^^^^^^^^
+
+  .. versionadded:: 4.7.0
+
+Size, in megabytes, of each LMDB database.
+This number can be increased later, but never decreased.
+Defaults to 100 on 32 bit systems, and 16000 on 64 bit systems.
+
 LMDB Structure
 --------------
 

diff --git a/ext/lmdb-safe/lmdb-safe.cc b/ext/lmdb-safe/lmdb-safe.cc
index 7a023fd05afcf56e3a597faeda92ade1c8af56d7..f159eafafa217861bbb1f5d1309d7849e4c1a7f6 100644 (file)
--- a/ext/lmdb-safe/lmdb-safe.cc
+++ b/ext/lmdb-safe/lmdb-safe.cc
@@ -27,11 +27,9 @@ MDBDbi::MDBDbi(MDB_env* env, MDB_txn* txn, const string_view dbname, int flags)
   // Database names are keys in the unnamed database, and may be read but not written.
 }
 
-MDBEnv::MDBEnv(const char* fname, int flags, int mode)
+MDBEnv::MDBEnv(const char* fname, int flags, int mode, uint64_t mapsizeMB)
 {
   mdb_env_create(&d_env);
-  uint64_t mapsizeMB = (sizeof(long)==4) ? 100 : 16000;
-  // on 32 bit platforms, there is just no room for more
   if(mdb_env_set_mapsize(d_env, mapsizeMB * 1048576))
     throw std::runtime_error("setting map size");
     /*
@@ -90,7 +88,7 @@ int MDBEnv::getROTX()
 }
 
 
-std::shared_ptr<MDBEnv> getMDBEnv(const char* fname, int flags, int mode)
+std::shared_ptr<MDBEnv> getMDBEnv(const char* fname, int flags, int mode, uint64_t mapsizeMB)
 {
   struct Value
   {
@@ -107,7 +105,7 @@ std::shared_ptr<MDBEnv> getMDBEnv(const char* fname, int flags, int mode)
       throw std::runtime_error("Unable to stat prospective mdb database: "+string(strerror(errno)));
     else {
       std::lock_guard<std::mutex> l(mut);
-      auto fresh = std::make_shared<MDBEnv>(fname, flags, mode);
+      auto fresh = std::make_shared<MDBEnv>(fname, flags, mode, mapsizeMB);
       if(stat(fname, &statbuf))
         throw std::runtime_error("Unable to stat prospective mdb database: "+string(strerror(errno)));
       auto key = std::tie(statbuf.st_dev, statbuf.st_ino);
@@ -132,7 +130,7 @@ std::shared_ptr<MDBEnv> getMDBEnv(const char* fname, int flags, int mode)
     }
   }
 
-  auto fresh = std::make_shared<MDBEnv>(fname, flags, mode);
+  auto fresh = std::make_shared<MDBEnv>(fname, flags, mode, mapsizeMB);
   s_envs[key] = {fresh, flags};
   
   return fresh;

diff --git a/ext/lmdb-safe/lmdb-safe.hh b/ext/lmdb-safe/lmdb-safe.hh
index 6844909e96e127a7c7b9df023a55f690caf79d9b..33bfa8f131202de4bd2f74771a9b206197569ab1 100644 (file)
--- a/ext/lmdb-safe/lmdb-safe.hh
+++ b/ext/lmdb-safe/lmdb-safe.hh
@@ -54,7 +54,7 @@ using MDBRWTransaction = std::unique_ptr<MDBRWTransactionImpl>;
 class MDBEnv
 {
 public:
-  MDBEnv(const char* fname, int flags, int mode);
+  MDBEnv(const char* fname, int flags, int mode, uint64_t mapsizeMB);
 
   ~MDBEnv()
   {
@@ -87,7 +87,7 @@ private:
   std::map<std::thread::id, int> d_ROtransactionsOut;
 };
 
-std::shared_ptr<MDBEnv> getMDBEnv(const char* fname, int flags, int mode);
+std::shared_ptr<MDBEnv> getMDBEnv(const char* fname, int flags, int mode, uint64_t mapsizeMB=(sizeof(void *)==4) ? 100 : 16000);
 
 
 

diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc
index 52c47cafaa0eb57c64823482360d16b927511a91..e287a0b892390c00068faaf319e76645b6fa7f07 100644 (file)
--- a/modules/lmdbbackend/lmdbbackend.cc
+++ b/modules/lmdbbackend/lmdbbackend.cc
@@ -82,7 +82,14 @@ LMDBBackend::LMDBBackend(const std::string& suffix)
   else
     throw std::runtime_error("Unknown sync mode " + syncMode + " requested for LMDB backend");
 
-  d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | d_asyncFlag, 0600), "domains");
+  uint64_t mapSize = 0;
+  try {
+    mapSize = std::stoll(getArg("map-size"));
+  }
+  catch (const std::exception& e) {
+    throw std::runtime_error(std::string("Unable to parse the 'map-size' LMDB value: ") + e.what());
+  }
+  d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | d_asyncFlag, 0600, mapSize), "domains");
   d_tmeta = std::make_shared<tmeta_t>(d_tdomains->getEnv(), "metadata");
   d_tkdb = std::make_shared<tkdb_t>(d_tdomains->getEnv(), "keydata");
   d_ttsig = std::make_shared<ttsig_t>(d_tdomains->getEnv(), "tsig");
@@ -1751,9 +1758,10 @@ public:
     declare(suffix, "filename", "Filename for lmdb", "./pdns.lmdb");
     declare(suffix, "sync-mode", "Synchronisation mode: nosync, nometasync, mapasync, sync", "mapasync");
     // there just is no room for more on 32 bit
-    declare(suffix, "shards", "Records database will be split into this number of shards", (sizeof(long) == 4) ? "2" : "64");
+    declare(suffix, "shards", "Records database will be split into this number of shards", (sizeof(void*) == 4) ? "2" : "64");
     declare(suffix, "schema-version", "Maximum allowed schema version to run on this DB. If a lower version is found, auto update is performed", std::to_string(SCHEMAVERSION));
     declare(suffix, "random-ids", "Numeric IDs inside the database are generated randomly instead of sequentially", "no");
+    declare(suffix, "map-size", "LMDB map size in megabytes", (sizeof(void*) == 4) ? "100" : "16000");
   }
   DNSBackend* make(const string& suffix = "") override
   {

diff --git a/pdns/dnsdistdist/dnsdist-kvs.hh b/pdns/dnsdistdist/dnsdist-kvs.hh
index 14d0bed079b06c939867c21217c841d99b1d009c..9f9392cdc118bb2506afe718d65933de38677850 100644 (file)
--- a/pdns/dnsdistdist/dnsdist-kvs.hh
+++ b/pdns/dnsdistdist/dnsdist-kvs.hh
@@ -175,7 +175,7 @@ public:
 class LMDBKVStore: public KeyValueStore
 {
 public:
-  LMDBKVStore(const std::string& fname, const std::string& dbName, bool noLock=false): d_env(fname.c_str(), noLock ? MDB_NOSUBDIR|MDB_RDONLY|MDB_NOLOCK : MDB_NOSUBDIR|MDB_RDONLY, 0600), d_dbi(d_env.openDB(dbName, 0)), d_fname(fname), d_dbName(dbName)
+  LMDBKVStore(const std::string& fname, const std::string& dbName, bool noLock=false): d_env(fname.c_str(), noLock ? MDB_NOSUBDIR|MDB_RDONLY|MDB_NOLOCK : MDB_NOSUBDIR|MDB_RDONLY, 0600, 0), d_dbi(d_env.openDB(dbName, 0)), d_fname(fname), d_dbName(dbName)
   {
   }
 

diff --git a/pdns/dnsdistdist/test-dnsdistkvs_cc.cc b/pdns/dnsdistdist/test-dnsdistkvs_cc.cc
index 5b68bd923e109ee4bb0edd449d68b9109cd700b9..5ed25c2a1020a52ba5ef430f498a4218811e8c95 100644 (file)
--- a/pdns/dnsdistdist/test-dnsdistkvs_cc.cc
+++ b/pdns/dnsdistdist/test-dnsdistkvs_cc.cc
@@ -327,7 +327,7 @@ BOOST_AUTO_TEST_CASE(test_LMDB) {
 
   string dbPath("/tmp/test_lmdb.XXXXXX");
   {
-    MDBEnv env(dbPath.c_str(), MDB_NOSUBDIR, 0600);
+    MDBEnv env(dbPath.c_str(), MDB_NOSUBDIR, 0600, 50);
     auto transaction = env.getRWTransaction();
     auto dbi = transaction->openDB("db-name", MDB_CREATE);
     transaction->put(dbi, MDBInVal(std::string(reinterpret_cast<const char*>(&rem.sin4.sin_addr.s_addr), sizeof(rem.sin4.sin_addr.s_addr))), MDBInVal("this is the value for the remote addr"));
@@ -341,7 +341,7 @@ BOOST_AUTO_TEST_CASE(test_LMDB) {
   }
 
   {
-    MDBEnv env(dbPath.c_str(), MDB_NOSUBDIR, 0600);
+    MDBEnv env(dbPath.c_str(), MDB_NOSUBDIR, 0600, 50);
     auto transaction = env.getRWTransaction();
     auto dbi = transaction->openDB("range-db-name", MDB_CREATE);
     /* range-based lookups */