From: Miod Vallat Date: Thu, 23 Oct 2025 12:12:07 +0000 (+0200) Subject: Allow different mapsize values for main and shards. X-Git-Tag: rec-5.4.0-alpha1~112^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cc40a28e935a9308890cfae64c0d10b9310d739a;p=thirdparty%2Fpdns.git Allow different mapsize values for main and shards. Signed-off-by: Miod Vallat --- diff --git a/docs/backends/lmdb.rst b/docs/backends/lmdb.rst index ea8911e122..524afd2359 100644 --- a/docs/backends/lmdb.rst +++ b/docs/backends/lmdb.rst @@ -120,6 +120,23 @@ Size, in megabytes, of each LMDB database. This number can be increased later, but never decreased. Defaults to 100 on 32 bit systems, and 16000 on 64 bit systems. + .. versionchanged:: 5.1.0 + +From version 5.1.0 onwards, this settings only applies to the main database +file; shards use :ref:`settings-lmdb-shards-map-size` instead. + +.. _settings-lmdb-shards-map-size: + +``lmdb-shards-map-size`` +^^^^^^^^^^^^^^^^^^^^^^^^ + + .. versionadded:: 5.1.0 + +Size, in megabytes, of each LMDB shard database. +This number can be increased later, but never decreased. +If set to zero (which is its default value), the same value as +:ref:`settings-lmdb-map-size` will be used. + .. _settings-lmdb-flag-deleted: ``lmdb-flag-deleted`` diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc index ccb2cc5ca1..af0bd352a3 100644 --- a/modules/lmdbbackend/lmdbbackend.cc +++ b/modules/lmdbbackend/lmdbbackend.cc @@ -731,13 +731,23 @@ LMDBBackend::LMDBBackend(const std::string& suffix) else throw std::runtime_error("Unknown sync mode " + syncMode + " requested for LMDB backend"); - d_mapsize = 0; + d_mapsize_main = d_mapsize_shards = 0; try { - d_mapsize = std::stoll(getArg("map-size")); + d_mapsize_main = std::stoll(getArg("map-size")); } catch (const std::exception& e) { throw std::runtime_error(std::string("Unable to parse the 'map-size' LMDB value: ") + e.what()); } + try { + d_mapsize_shards = std::stoll(getArg("shards-map-size")); + } + catch (const std::exception& e) { + throw std::runtime_error(std::string("Unable to parse the 'shards-map-size' LMDB value: ") + e.what()); + } + if (d_mapsize_shards == 0) { + // Old configuration with only one settings for main and shards. + d_mapsize_shards = d_mapsize_main; + } d_write_notification_update = mustDo("write-notification-update"); @@ -879,7 +889,7 @@ LMDBBackend::~LMDBBackend() void LMDBBackend::openAllTheDatabases() { - d_tdomains = std::make_shared(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, d_mapsize), "domains_v5"); + d_tdomains = std::make_shared(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, d_mapsize_main), "domains_v5"); d_tmeta = std::make_shared(d_tdomains->getEnv(), "metadata_v5"); d_tkdb = std::make_shared(d_tdomains->getEnv(), "keydata_v5"); d_ttsig = std::make_shared(d_tdomains->getEnv(), "tsig_v5"); @@ -1775,7 +1785,7 @@ std::shared_ptr LMDBBackend::getRecordsRWTran auto& shard = d_trecords[id % s_shards]; if (!shard.env) { shard.env = getMDBEnv((getArg("filename") + "-" + std::to_string(id % s_shards)).c_str(), - MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, d_mapsize); + MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, d_mapsize_shards); shard.dbi = shard.env->openDB("records_v5", MDB_CREATE); } auto ret = std::make_shared(shard.env->getRWTransaction()); @@ -1793,7 +1803,7 @@ std::shared_ptr LMDBBackend::getRecordsROTran throw DBException("attempting to start nested transaction without open parent env"); } shard.env = getMDBEnv((getArg("filename") + "-" + std::to_string(id % s_shards)).c_str(), - MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, d_mapsize); + MDB_NOSUBDIR | MDB_NORDAHEAD | d_asyncFlag, 0600, d_mapsize_shards); shard.dbi = shard.env->openDB("records_v5", MDB_CREATE); } @@ -3488,7 +3498,8 @@ public: declare(suffix, "shards", "Records database will be split into this number of shards", (sizeof(void*) == 4) ? "2" : "64"); declare(suffix, "schema-version", "Maximum allowed schema version to run on this DB. If a lower version is found, auto update is performed", std::to_string(SCHEMAVERSION)); declare(suffix, "random-ids", "Numeric IDs inside the database are generated randomly instead of sequentially", "no"); - declare(suffix, "map-size", "LMDB map size in megabytes", (sizeof(void*) == 4) ? "100" : "16000"); + declare(suffix, "map-size", "main LMDB map size in megabytes", (sizeof(void*) == 4) ? "100" : "16000"); + declare(suffix, "shards-map-size", "shard LMDB map size in megabytes, zero to use the same size as main", "0"); declare(suffix, "flag-deleted", "Flag entries on deletion instead of deleting them", "no"); declare(suffix, "write-notification-update", "Do not update domain table upon notification", "yes"); declare(suffix, "lightning-stream", "Run in Lightning Stream compatible mode", "no"); diff --git a/modules/lmdbbackend/lmdbbackend.hh b/modules/lmdbbackend/lmdbbackend.hh index c51ea7fe31..9aa1a9c132 100644 --- a/modules/lmdbbackend/lmdbbackend.hh +++ b/modules/lmdbbackend/lmdbbackend.hh @@ -424,5 +424,6 @@ private: bool d_views; bool d_write_notification_update; DTime d_dtime; // used only for logging - uint64_t d_mapsize; + uint64_t d_mapsize_main; + uint64_t d_mapsize_shards; };