From aded4962afb7232d7ef878b6197eaeb273eef871 Mon Sep 17 00:00:00 2001 From: Peter van Dijk Date: Tue, 15 Feb 2022 16:21:32 +0100 Subject: [PATCH] auth lmdb: make map size configurable --- docs/backends/lmdb.rst | 11 +++++++++++ ext/lmdb-safe/lmdb-safe.cc | 10 ++++------ ext/lmdb-safe/lmdb-safe.hh | 4 ++-- modules/lmdbbackend/lmdbbackend.cc | 12 ++++++++++-- pdns/dnsdistdist/dnsdist-kvs.hh | 2 +- pdns/dnsdistdist/test-dnsdistkvs_cc.cc | 4 ++-- 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/docs/backends/lmdb.rst b/docs/backends/lmdb.rst index dc0d96fba2..936e13d413 100644 --- a/docs/backends/lmdb.rst +++ b/docs/backends/lmdb.rst @@ -94,6 +94,17 @@ PowerDNS Version LMDB Schema version Numeric IDs inside the database are generated randomly instead of sequentially. If some external process is synchronising databases between systems, this will avoid conflicts when objects (domains, keys, etc.) get added. +.. _settings-lmdb-map-size: + +``lmdb-random-ids`` +^^^^^^^^^^^^^^^^^^^ + + .. versionadded:: 4.7.0 + +Size, in megabytes, of each LMDB database. +This number can be increased later, but never decreased. +Defaults to 100 on 32 bit systems, and 16000 on 64 bit systems. + LMDB Structure -------------- diff --git a/ext/lmdb-safe/lmdb-safe.cc b/ext/lmdb-safe/lmdb-safe.cc index 7a023fd05a..f159eafafa 100644 --- a/ext/lmdb-safe/lmdb-safe.cc +++ b/ext/lmdb-safe/lmdb-safe.cc @@ -27,11 +27,9 @@ MDBDbi::MDBDbi(MDB_env* env, MDB_txn* txn, const string_view dbname, int flags) // Database names are keys in the unnamed database, and may be read but not written. } -MDBEnv::MDBEnv(const char* fname, int flags, int mode) +MDBEnv::MDBEnv(const char* fname, int flags, int mode, uint64_t mapsizeMB) { mdb_env_create(&d_env); - uint64_t mapsizeMB = (sizeof(long)==4) ? 100 : 16000; - // on 32 bit platforms, there is just no room for more if(mdb_env_set_mapsize(d_env, mapsizeMB * 1048576)) throw std::runtime_error("setting map size"); /* @@ -90,7 +88,7 @@ int MDBEnv::getROTX() } -std::shared_ptr getMDBEnv(const char* fname, int flags, int mode) +std::shared_ptr getMDBEnv(const char* fname, int flags, int mode, uint64_t mapsizeMB) { struct Value { @@ -107,7 +105,7 @@ std::shared_ptr getMDBEnv(const char* fname, int flags, int mode) throw std::runtime_error("Unable to stat prospective mdb database: "+string(strerror(errno))); else { std::lock_guard l(mut); - auto fresh = std::make_shared(fname, flags, mode); + auto fresh = std::make_shared(fname, flags, mode, mapsizeMB); if(stat(fname, &statbuf)) throw std::runtime_error("Unable to stat prospective mdb database: "+string(strerror(errno))); auto key = std::tie(statbuf.st_dev, statbuf.st_ino); @@ -132,7 +130,7 @@ std::shared_ptr getMDBEnv(const char* fname, int flags, int mode) } } - auto fresh = std::make_shared(fname, flags, mode); + auto fresh = std::make_shared(fname, flags, mode, mapsizeMB); s_envs[key] = {fresh, flags}; return fresh; diff --git a/ext/lmdb-safe/lmdb-safe.hh b/ext/lmdb-safe/lmdb-safe.hh index 6844909e96..33bfa8f131 100644 --- a/ext/lmdb-safe/lmdb-safe.hh +++ b/ext/lmdb-safe/lmdb-safe.hh @@ -54,7 +54,7 @@ using MDBRWTransaction = std::unique_ptr; class MDBEnv { public: - MDBEnv(const char* fname, int flags, int mode); + MDBEnv(const char* fname, int flags, int mode, uint64_t mapsizeMB); ~MDBEnv() { @@ -87,7 +87,7 @@ private: std::map d_ROtransactionsOut; }; -std::shared_ptr getMDBEnv(const char* fname, int flags, int mode); +std::shared_ptr getMDBEnv(const char* fname, int flags, int mode, uint64_t mapsizeMB=(sizeof(void *)==4) ? 100 : 16000); diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc index 52c47cafaa..e287a0b892 100644 --- a/modules/lmdbbackend/lmdbbackend.cc +++ b/modules/lmdbbackend/lmdbbackend.cc @@ -82,7 +82,14 @@ LMDBBackend::LMDBBackend(const std::string& suffix) else throw std::runtime_error("Unknown sync mode " + syncMode + " requested for LMDB backend"); - d_tdomains = std::make_shared(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | d_asyncFlag, 0600), "domains"); + uint64_t mapSize = 0; + try { + mapSize = std::stoll(getArg("map-size")); + } + catch (const std::exception& e) { + throw std::runtime_error(std::string("Unable to parse the 'map-size' LMDB value: ") + e.what()); + } + d_tdomains = std::make_shared(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR | d_asyncFlag, 0600, mapSize), "domains"); d_tmeta = std::make_shared(d_tdomains->getEnv(), "metadata"); d_tkdb = std::make_shared(d_tdomains->getEnv(), "keydata"); d_ttsig = std::make_shared(d_tdomains->getEnv(), "tsig"); @@ -1751,9 +1758,10 @@ public: declare(suffix, "filename", "Filename for lmdb", "./pdns.lmdb"); declare(suffix, "sync-mode", "Synchronisation mode: nosync, nometasync, mapasync, sync", "mapasync"); // there just is no room for more on 32 bit - declare(suffix, "shards", "Records database will be split into this number of shards", (sizeof(long) == 4) ? "2" : "64"); + declare(suffix, "shards", "Records database will be split into this number of shards", (sizeof(void*) == 4) ? "2" : "64"); declare(suffix, "schema-version", "Maximum allowed schema version to run on this DB. If a lower version is found, auto update is performed", std::to_string(SCHEMAVERSION)); declare(suffix, "random-ids", "Numeric IDs inside the database are generated randomly instead of sequentially", "no"); + declare(suffix, "map-size", "LMDB map size in megabytes", (sizeof(void*) == 4) ? "100" : "16000"); } DNSBackend* make(const string& suffix = "") override { diff --git a/pdns/dnsdistdist/dnsdist-kvs.hh b/pdns/dnsdistdist/dnsdist-kvs.hh index 14d0bed079..9f9392cdc1 100644 --- a/pdns/dnsdistdist/dnsdist-kvs.hh +++ b/pdns/dnsdistdist/dnsdist-kvs.hh @@ -175,7 +175,7 @@ public: class LMDBKVStore: public KeyValueStore { public: - LMDBKVStore(const std::string& fname, const std::string& dbName, bool noLock=false): d_env(fname.c_str(), noLock ? MDB_NOSUBDIR|MDB_RDONLY|MDB_NOLOCK : MDB_NOSUBDIR|MDB_RDONLY, 0600), d_dbi(d_env.openDB(dbName, 0)), d_fname(fname), d_dbName(dbName) + LMDBKVStore(const std::string& fname, const std::string& dbName, bool noLock=false): d_env(fname.c_str(), noLock ? MDB_NOSUBDIR|MDB_RDONLY|MDB_NOLOCK : MDB_NOSUBDIR|MDB_RDONLY, 0600, 0), d_dbi(d_env.openDB(dbName, 0)), d_fname(fname), d_dbName(dbName) { } diff --git a/pdns/dnsdistdist/test-dnsdistkvs_cc.cc b/pdns/dnsdistdist/test-dnsdistkvs_cc.cc index 5b68bd923e..5ed25c2a10 100644 --- a/pdns/dnsdistdist/test-dnsdistkvs_cc.cc +++ b/pdns/dnsdistdist/test-dnsdistkvs_cc.cc @@ -327,7 +327,7 @@ BOOST_AUTO_TEST_CASE(test_LMDB) { string dbPath("/tmp/test_lmdb.XXXXXX"); { - MDBEnv env(dbPath.c_str(), MDB_NOSUBDIR, 0600); + MDBEnv env(dbPath.c_str(), MDB_NOSUBDIR, 0600, 50); auto transaction = env.getRWTransaction(); auto dbi = transaction->openDB("db-name", MDB_CREATE); transaction->put(dbi, MDBInVal(std::string(reinterpret_cast(&rem.sin4.sin_addr.s_addr), sizeof(rem.sin4.sin_addr.s_addr))), MDBInVal("this is the value for the remote addr")); @@ -341,7 +341,7 @@ BOOST_AUTO_TEST_CASE(test_LMDB) { } { - MDBEnv env(dbPath.c_str(), MDB_NOSUBDIR, 0600); + MDBEnv env(dbPath.c_str(), MDB_NOSUBDIR, 0600, 50); auto transaction = env.getRWTransaction(); auto dbi = transaction->openDB("range-db-name", MDB_CREATE); /* range-based lookups */ -- 2.47.2