From: Miod Vallat Date: Mon, 15 Sep 2025 13:56:20 +0000 (+0200) Subject: lmdb: allow domain notification timestamps to be kept in memory only. X-Git-Tag: auth-4.9.9~2^2~1 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=372227d6b0698e231788b1e25aa5646da0b82297;p=thirdparty%2Fpdns.git lmdb: allow domain notification timestamps to be kept in memory only. They will get synchronized on disk only when another DomainInfo field gets modified. Signed-off-by: Miod Vallat (cherry picked from commit 51d5dbc558e36f3bb6d487c5c46737e58e2f4b19) --- diff --git a/docs/backends/lmdb.rst b/docs/backends/lmdb.rst index db573f447b..642604eba1 100644 --- a/docs/backends/lmdb.rst +++ b/docs/backends/lmdb.rst @@ -129,6 +129,21 @@ Defaults to 100 on 32 bit systems, and 16000 on 64 bit systems. Instead of deleting items from the database, flag them as deleted in the item's `Lightning Stream `_ header. Only enable this if you are using Lightning Stream. +``lmdb-skip-notification-update`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + .. versionadded:: 5.1.0 + +- Boolean +- Default: no + +Do not update the domains table in the database when the last notification +timestamp is modified. These timestamps will only be written back to the +database when other changes to the domain (such as accounts) occur. + +**Warning**: Running with this flag enabled will cause spurious notifications +to be sent upon startup. + ``lmdb-lightning-stream`` ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc index 07441464c1..657f860f4f 100644 --- a/modules/lmdbbackend/lmdbbackend.cc +++ b/modules/lmdbbackend/lmdbbackend.cc @@ -645,6 +645,30 @@ bool LMDBBackend::upgradeToSchemav5(std::string& filename) return true; } +// Serial number cache +bool LMDBBackend::SerialCache::get(uint32_t domainid, uint32_t& serial) const +{ + if (auto iter = d_serials.find(domainid); iter != d_serials.end()) { + serial = iter->second; + return true; + } + return false; +} + +void LMDBBackend::SerialCache::remove(uint32_t domainid) +{ + if (auto iter = d_serials.find(domainid); iter != d_serials.end()) { + d_serials.erase(iter); + } +} + +void LMDBBackend::SerialCache::update(uint32_t domainid, uint32_t serial) +{ + d_serials.insert_or_assign(domainid, serial); +} + +SharedLockGuarded LMDBBackend::s_notified_serial; + LMDBBackend::LMDBBackend(const std::string& suffix) { // overlapping domain ids in combination with relative names are a recipe for disaster @@ -673,6 +697,8 @@ LMDBBackend::LMDBBackend(const std::string& suffix) throw std::runtime_error(std::string("Unable to parse the 'map-size' LMDB value: ") + e.what()); } + d_skip_notification_update = mustDo("skip-notification-update"); + if (mustDo("lightning-stream")) { d_random_ids = true; d_handle_dups = true; @@ -1002,7 +1028,7 @@ void LMDBBackend::deleteDomainRecords(RecordsRWTransaction& txn, uint32_t domain } } -bool LMDBBackend::findDomain(const DNSName& domain, DomainInfo& info) +bool LMDBBackend::findDomain(const DNSName& domain, DomainInfo& info) const { auto rotxn = d_tdomains->getROTransaction(); auto domain_id = rotxn.get<0>(domain, info); @@ -1013,7 +1039,7 @@ bool LMDBBackend::findDomain(const DNSName& domain, DomainInfo& info) return true; } -bool LMDBBackend::findDomain(uint32_t domainid, DomainInfo& info) +bool LMDBBackend::findDomain(uint32_t domainid, DomainInfo& info) const { auto rotxn = d_tdomains->getROTransaction(); if (!rotxn.get(domainid, info)) { @@ -1023,6 +1049,32 @@ bool LMDBBackend::findDomain(uint32_t domainid, DomainInfo& info) return true; } +void LMDBBackend::consolidateDomainInfo(DomainInfo& info) const +{ + // Update the notified_serial value if we have a cached value in memory. + if (d_skip_notification_update) { + auto container = s_notified_serial.read_lock(); + container->get(info.id, info.notified_serial); + } +} + +void LMDBBackend::writeDomainInfo(const DomainInfo& info) +{ + if (d_skip_notification_update) { + uint32_t last_notified_serial{0}; + auto container = s_notified_serial.write_lock(); + container->get(info.id, last_notified_serial); + // Only remove the in-memory value if it has not been modified since the + // DomainInfo data was set up. + if (last_notified_serial == info.notified_serial) { + container->remove(info.id); + } + } + auto txn = d_tdomains->getRWTransaction(); + txn.put(info, info.id); + txn.commit(); +} + /* Here's the complicated story. Other backends have just one transaction, which is either on or not. @@ -1392,6 +1444,10 @@ bool LMDBBackend::deleteDomain(const DNSName& domain) commitTransaction(); // Remove zone + { + auto container = s_notified_serial.write_lock(); + container->remove(static_cast(id)); + } auto txn = d_tdomains->getRWTransaction(); txn.del(id); txn.commit(); @@ -1614,6 +1670,7 @@ bool LMDBBackend::getDomainInfo(const DNSName& domain, DomainInfo& di, bool gets return false; } di.backend = this; + consolidateDomainInfo(di); if (getserial) { getSerial(di); @@ -1628,10 +1685,9 @@ int LMDBBackend::genChangeDomain(const DNSName& domain, const std::function(false); } + consolidateDomainInfo(info); func(info); - auto txn = d_tdomains->getRWTransaction(); - txn.put(info, info.id); - txn.commit(); + writeDomainInfo(info); return true; } @@ -1641,10 +1697,9 @@ int LMDBBackend::genChangeDomain(uint32_t id, const std::function(false); } + consolidateDomainInfo(info); func(info); - auto txn = d_tdomains->getRWTransaction(); - txn.put(info, id); - txn.commit(); + writeDomainInfo(info); return true; } @@ -1720,6 +1775,7 @@ void LMDBBackend::getAllDomainsFiltered(vector* domains, const std:: for (auto& [k, v] : zonemap) { if (allow(v)) { + consolidateDomainInfo(v); domains->push_back(std::move(v)); } } @@ -1731,6 +1787,7 @@ void LMDBBackend::getAllDomainsFiltered(vector* domains, const std:: di.backend = this; if (allow(di)) { + consolidateDomainInfo(di); domains->push_back(di); } } @@ -1826,9 +1883,19 @@ void LMDBBackend::getUpdatedPrimaries(vector& updatedDomains, std::u void LMDBBackend::setNotified(uint32_t domain_id, uint32_t serial) { - genChangeDomain(domain_id, [serial](DomainInfo& info) { - info.notified_serial = serial; - }); + if (!d_skip_notification_update) { + genChangeDomain(domain_id, [serial](DomainInfo& info) { + info.notified_serial = serial; + }); + return; + } + + DomainInfo info; + if (findDomain(domain_id, info)) { + auto container = s_notified_serial.write_lock(); + container->update(info.id, serial); + } + // else throw something? this should be a "can't happen" situation. } class getCatalogMembersReturnFalseException : std::runtime_error @@ -2763,6 +2830,7 @@ public: declare(suffix, "random-ids", "Numeric IDs inside the database are generated randomly instead of sequentially", "no"); declare(suffix, "map-size", "LMDB map size in megabytes", (sizeof(void*) == 4) ? "100" : "16000"); declare(suffix, "flag-deleted", "Flag entries on deletion instead of deleting them", "no"); + declare(suffix, "skip-notification-update", "Do not update domain table upon notification", "no"); declare(suffix, "lightning-stream", "Run in Lightning Stream compatible mode", "no"); } DNSBackend* make(const string& suffix = "") override diff --git a/modules/lmdbbackend/lmdbbackend.hh b/modules/lmdbbackend/lmdbbackend.hh index 71cf8e2bbd..813cde3e15 100644 --- a/modules/lmdbbackend/lmdbbackend.hh +++ b/modules/lmdbbackend/lmdbbackend.hh @@ -310,8 +310,10 @@ private: int genChangeDomain(uint32_t id, const std::function& func); void deleteDomainRecords(RecordsRWTransaction& txn, uint32_t domain_id, uint16_t qtype = QType::ANY); - bool findDomain(const DNSName& domain, DomainInfo& info); - bool findDomain(uint32_t domainid, DomainInfo& info); + bool findDomain(const DNSName& domain, DomainInfo& info) const; + bool findDomain(uint32_t domainid, DomainInfo& info) const; + void consolidateDomainInfo(DomainInfo& info) const; + void writeDomainInfo(const DomainInfo& info); void getAllDomainsFiltered(vector* domains, const std::function& allow); @@ -324,6 +326,19 @@ private: std::string d_matchkey; DNSName d_lookupdomain; + // Cache of DomainInfo notified_serial values + class SerialCache : public boost::noncopyable + { + public: + bool get(uint32_t domainid, uint32_t& serial) const; + void remove(uint32_t domainid); + void update(uint32_t domainid, uint32_t serial); + + private: + std::unordered_map d_serials; + }; + static SharedLockGuarded s_notified_serial; + vector d_currentrrset; size_t d_currentrrsetpos; time_t d_currentrrsettime; @@ -336,6 +351,7 @@ private: bool d_dolog; bool d_random_ids; bool d_handle_dups; + bool d_skip_notification_update; DTime d_dtime; // used only for logging uint64_t d_mapsize; };