Allow different mapsize values for main and shards.

Signed-off-by: Miod Vallat <miod.vallat@powerdns.com>

diff --git a/docs/backends/lmdb.rst b/docs/backends/lmdb.rst
index ea8911e1228b7cfe2cc18a8be2dd4a7ae38d7009..524afd2359c5b772a14d1cd5e4ff7f1f8efeffc3 100644 (file)
--- a/docs/backends/lmdb.rst
+++ b/docs/backends/lmdb.rst
@@ -120,6 +120,23 @@ Size, in megabytes, of each LMDB database.
 This number can be increased later, but never decreased.
 Defaults to 100 on 32 bit systems, and 16000 on 64 bit systems.
 
+  .. versionchanged:: 5.1.0
+
+From version 5.1.0 onwards, this settings only applies to the main database
+file; shards use :ref:`settings-lmdb-shards-map-size` instead.
+
+.. _settings-lmdb-shards-map-size:
+
+``lmdb-shards-map-size``
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+  .. versionadded:: 5.1.0
+
+Size, in megabytes, of each LMDB shard database.
+This number can be increased later, but never decreased.
+If set to zero (which is its default value), the same value as
+:ref:`settings-lmdb-map-size` will be used.
+
 .. _settings-lmdb-flag-deleted:
 
 ``lmdb-flag-deleted``

diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc
index ccb2cc5ca1bb56e9ec9b3500b7a6dc2364766d13..af0bd352a31080bcbf35bb380aa89b03597eac86 100644 (file)
--- a/modules/lmdbbackend/lmdbbackend.cc
+++ b/modules/lmdbbackend/lmdbbackend.cc
@@ -731,13 +731,23 @@ LMDBBackend::LMDBBackend(const std::string& suffix)
   else
     throw std::runtime_error("Unknown sync mode " + syncMode + " requested for LMDB backend");
 
-  d_mapsize = 0;
+  d_mapsize_main = d_mapsize_shards = 0;
   try {
-    d_mapsize = std::stoll(getArg("map-size"));
+    d_mapsize_main = std::stoll(getArg("map-size"));
   }
   catch (const std::exception& e) {
     throw std::runtime_error(std::string("Unable to parse the 'map-size' LMDB value: ") + e.what());
   }
+  try {
+    d_mapsize_shards = std::stoll(getArg("shards-map-size"));
+  }
+  catch (const std::exception& e) {
+    throw std::runtime_error(std::string("Unable to parse the 'shards-map-size' LMDB value: ") + e.what());
+  }
+  if (d_mapsize_shards == 0) {
+    // Old configuration with only one settings for main and shards.
+    d_mapsize_shards = d_mapsize_main;
+  }
 
   d_write_notification_update = mustDo("write-notification-update");
 
@@ -879,7 +889,7 @@ LMDBBackend::~LMDBBackend()
 
 void LMDBBackend::openAllTheDatabases()
 {
-  d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, d_mapsize), "domains_v5");
+  d_tdomains = std::make_shared<tdomains_t>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, d_mapsize_main), "domains_v5");
   d_tmeta = std::make_shared<tmeta_t>(d_tdomains->getEnv(), "metadata_v5");
   d_tkdb = std::make_shared<tkdb_t>(d_tdomains->getEnv(), "keydata_v5");
   d_ttsig = std::make_shared<ttsig_t>(d_tdomains->getEnv(), "tsig_v5");
@@ -1775,7 +1785,7 @@ std::shared_ptr<LMDBBackend::RecordsRWTransaction> LMDBBackend::getRecordsRWTran
   auto& shard = d_trecords[id % s_shards];
   if (!shard.env) {
     shard.env = getMDBEnv((getArg("filename") + "-" + std::to_string(id % s_shards)).c_str(),
-                          MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, d_mapsize);
+                          MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, d_mapsize_shards);
     shard.dbi = shard.env->openDB("records_v5", MDB_CREATE);
   }
   auto ret = std::make_shared<RecordsRWTransaction>(shard.env->getRWTransaction());
@@ -1793,7 +1803,7 @@ std::shared_ptr<LMDBBackend::RecordsROTransaction> LMDBBackend::getRecordsROTran
       throw DBException("attempting to start nested transaction without open parent env");
     }
     shard.env = getMDBEnv((getArg("filename") + "-" + std::to_string(id % s_shards)).c_str(),
-                          MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, d_mapsize);
+                          MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, d_mapsize_shards);
     shard.dbi = shard.env->openDB("records_v5", MDB_CREATE);
   }
 
@@ -3488,7 +3498,8 @@ public:
     declare(suffix, "shards", "Records database will be split into this number of shards", (sizeof(void*) == 4) ? "2" : "64");
     declare(suffix, "schema-version", "Maximum allowed schema version to run on this DB. If a lower version is found, auto update is performed", std::to_string(SCHEMAVERSION));
     declare(suffix, "random-ids", "Numeric IDs inside the database are generated randomly instead of sequentially", "no");
-    declare(suffix, "map-size", "LMDB map size in megabytes", (sizeof(void*) == 4) ? "100" : "16000");
+    declare(suffix, "map-size", "main LMDB map size in megabytes", (sizeof(void*) == 4) ? "100" : "16000");
+    declare(suffix, "shards-map-size", "shard LMDB map size in megabytes, zero to use the same size as main", "0");
     declare(suffix, "flag-deleted", "Flag entries on deletion instead of deleting them", "no");
     declare(suffix, "write-notification-update", "Do not update domain table upon notification", "yes");
     declare(suffix, "lightning-stream", "Run in Lightning Stream compatible mode", "no");

diff --git a/modules/lmdbbackend/lmdbbackend.hh b/modules/lmdbbackend/lmdbbackend.hh
index c51ea7fe311af35f473b546b91f06923dc50b360..9aa1a9c132d81d4835032cf3d664223cb7931990 100644 (file)
--- a/modules/lmdbbackend/lmdbbackend.hh
+++ b/modules/lmdbbackend/lmdbbackend.hh
@@ -424,5 +424,6 @@ private:
   bool d_views;
   bool d_write_notification_update;
   DTime d_dtime; // used only for logging
-  uint64_t d_mapsize;
+  uint64_t d_mapsize_main;
+  uint64_t d_mapsize_shards;
 };