lmdb: allow domain notification timestamps to be kept in memory only.

They will get synchronized on disk only when another DomainInfo field
gets modified.

Signed-off-by: Miod Vallat <miod.vallat@powerdns.com>
(cherry picked from commit 51d5dbc558e36f3bb6d487c5c46737e58e2f4b19)

diff --git a/docs/backends/lmdb.rst b/docs/backends/lmdb.rst
index db573f447b5596097dbcaa0561bd9085e892bfd2..642604eba11754a54d5d831548b7138efa8f75ed 100644 (file)
--- a/docs/backends/lmdb.rst
+++ b/docs/backends/lmdb.rst
@@ -129,6 +129,21 @@ Defaults to 100 on 32 bit systems, and 16000 on 64 bit systems.
 Instead of deleting items from the database, flag them as deleted in the item's `Lightning Stream <https://doc.powerdns.com/lightningstream>`_ header.
 Only enable this if you are using Lightning Stream.
 
+``lmdb-skip-notification-update``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  .. versionadded:: 5.1.0
+
+-  Boolean
+-  Default: no
+
+Do not update the domains table in the database when the last notification
+timestamp is modified. These timestamps will only be written back to the
+database when other changes to the domain (such as accounts) occur.
+
+**Warning**: Running with this flag enabled will cause spurious notifications
+to be sent upon startup.
+
 ``lmdb-lightning-stream``
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 

diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc
index 07441464c1b5f6b22ed1bdc80a72b32decccca07..657f860f4fa0bdea0d092aa9eecc83fcc99756c9 100644 (file)
--- a/modules/lmdbbackend/lmdbbackend.cc
+++ b/modules/lmdbbackend/lmdbbackend.cc
@@ -645,6 +645,30 @@ bool LMDBBackend::upgradeToSchemav5(std::string& filename)
   return true;
 }
 
+// Serial number cache
+bool LMDBBackend::SerialCache::get(uint32_t domainid, uint32_t& serial) const
+{
+  if (auto iter = d_serials.find(domainid); iter != d_serials.end()) {
+    serial = iter->second;
+    return true;
+  }
+  return false;
+}
+
+void LMDBBackend::SerialCache::remove(uint32_t domainid)
+{
+  if (auto iter = d_serials.find(domainid); iter != d_serials.end()) {
+    d_serials.erase(iter);
+  }
+}
+
+void LMDBBackend::SerialCache::update(uint32_t domainid, uint32_t serial)
+{
+  d_serials.insert_or_assign(domainid, serial);
+}
+
+SharedLockGuarded<LMDBBackend::SerialCache> LMDBBackend::s_notified_serial;
+
 LMDBBackend::LMDBBackend(const std::string& suffix)
 {
   // overlapping domain ids in combination with relative names are a recipe for disaster
@@ -673,6 +697,8 @@ LMDBBackend::LMDBBackend(const std::string& suffix)
     throw std::runtime_error(std::string("Unable to parse the 'map-size' LMDB value: ") + e.what());
   }
 
+  d_skip_notification_update = mustDo("skip-notification-update");
+
   if (mustDo("lightning-stream")) {
     d_random_ids = true;
     d_handle_dups = true;
@@ -1002,7 +1028,7 @@ void LMDBBackend::deleteDomainRecords(RecordsRWTransaction& txn, uint32_t domain
   }
 }
 
-bool LMDBBackend::findDomain(const DNSName& domain, DomainInfo& info)
+bool LMDBBackend::findDomain(const DNSName& domain, DomainInfo& info) const
 {
   auto rotxn = d_tdomains->getROTransaction();
   auto domain_id = rotxn.get<0>(domain, info);
@@ -1013,7 +1039,7 @@ bool LMDBBackend::findDomain(const DNSName& domain, DomainInfo& info)
   return true;
 }
 
-bool LMDBBackend::findDomain(uint32_t domainid, DomainInfo& info)
+bool LMDBBackend::findDomain(uint32_t domainid, DomainInfo& info) const
 {
   auto rotxn = d_tdomains->getROTransaction();
   if (!rotxn.get(domainid, info)) {
@@ -1023,6 +1049,32 @@ bool LMDBBackend::findDomain(uint32_t domainid, DomainInfo& info)
   return true;
 }
 
+void LMDBBackend::consolidateDomainInfo(DomainInfo& info) const
+{
+  // Update the notified_serial value if we have a cached value in memory.
+  if (d_skip_notification_update) {
+    auto container = s_notified_serial.read_lock();
+    container->get(info.id, info.notified_serial);
+  }
+}
+
+void LMDBBackend::writeDomainInfo(const DomainInfo& info)
+{
+  if (d_skip_notification_update) {
+    uint32_t last_notified_serial{0};
+    auto container = s_notified_serial.write_lock();
+    container->get(info.id, last_notified_serial);
+    // Only remove the in-memory value if it has not been modified since the
+    // DomainInfo data was set up.
+    if (last_notified_serial == info.notified_serial) {
+      container->remove(info.id);
+    }
+  }
+  auto txn = d_tdomains->getRWTransaction();
+  txn.put(info, info.id);
+  txn.commit();
+}
+
 /* Here's the complicated story. Other backends have just one transaction, which is either
    on or not.
 
@@ -1392,6 +1444,10 @@ bool LMDBBackend::deleteDomain(const DNSName& domain)
     commitTransaction();
 
     // Remove zone
+    {
+      auto container = s_notified_serial.write_lock();
+      container->remove(static_cast<uint32_t>(id));
+    }
     auto txn = d_tdomains->getRWTransaction();
     txn.del(id);
     txn.commit();
@@ -1614,6 +1670,7 @@ bool LMDBBackend::getDomainInfo(const DNSName& domain, DomainInfo& di, bool gets
     return false;
   }
   di.backend = this;
+  consolidateDomainInfo(di);
 
   if (getserial) {
     getSerial(di);
@@ -1628,10 +1685,9 @@ int LMDBBackend::genChangeDomain(const DNSName& domain, const std::function<void
   if (!findDomain(domain, info)) {
     return static_cast<int>(false);
   }
+  consolidateDomainInfo(info);
   func(info);
-  auto txn = d_tdomains->getRWTransaction();
-  txn.put(info, info.id);
-  txn.commit();
+  writeDomainInfo(info);
   return true;
 }
 
@@ -1641,10 +1697,9 @@ int LMDBBackend::genChangeDomain(uint32_t id, const std::function<void(DomainInf
   if (!findDomain(id, info)) {
     return static_cast<int>(false);
   }
+  consolidateDomainInfo(info);
   func(info);
-  auto txn = d_tdomains->getRWTransaction();
-  txn.put(info, id);
-  txn.commit();
+  writeDomainInfo(info);
   return true;
 }
 
@@ -1720,6 +1775,7 @@ void LMDBBackend::getAllDomainsFiltered(vector<DomainInfo>* domains, const std::
 
     for (auto& [k, v] : zonemap) {
       if (allow(v)) {
+        consolidateDomainInfo(v);
         domains->push_back(std::move(v));
       }
     }
@@ -1731,6 +1787,7 @@ void LMDBBackend::getAllDomainsFiltered(vector<DomainInfo>* domains, const std::
       di.backend = this;
 
       if (allow(di)) {
+        consolidateDomainInfo(di);
         domains->push_back(di);
       }
     }
@@ -1826,9 +1883,19 @@ void LMDBBackend::getUpdatedPrimaries(vector<DomainInfo>& updatedDomains, std::u
 
 void LMDBBackend::setNotified(uint32_t domain_id, uint32_t serial)
 {
-  genChangeDomain(domain_id, [serial](DomainInfo& info) {
-    info.notified_serial = serial;
-  });
+  if (!d_skip_notification_update) {
+    genChangeDomain(domain_id, [serial](DomainInfo& info) {
+      info.notified_serial = serial;
+    });
+    return;
+  }
+
+  DomainInfo info;
+  if (findDomain(domain_id, info)) {
+    auto container = s_notified_serial.write_lock();
+    container->update(info.id, serial);
+  }
+  // else throw something? this should be a "can't happen" situation.
 }
 
 class getCatalogMembersReturnFalseException : std::runtime_error
@@ -2763,6 +2830,7 @@ public:
     declare(suffix, "random-ids", "Numeric IDs inside the database are generated randomly instead of sequentially", "no");
     declare(suffix, "map-size", "LMDB map size in megabytes", (sizeof(void*) == 4) ? "100" : "16000");
     declare(suffix, "flag-deleted", "Flag entries on deletion instead of deleting them", "no");
+    declare(suffix, "skip-notification-update", "Do not update domain table upon notification", "no");
     declare(suffix, "lightning-stream", "Run in Lightning Stream compatible mode", "no");
   }
   DNSBackend* make(const string& suffix = "") override

diff --git a/modules/lmdbbackend/lmdbbackend.hh b/modules/lmdbbackend/lmdbbackend.hh
index 71cf8e2bbd01f7963c465e99dcc9e159f373c134..813cde3e15ac95cffe03e3e1999120a7b985a689 100644 (file)
--- a/modules/lmdbbackend/lmdbbackend.hh
+++ b/modules/lmdbbackend/lmdbbackend.hh
@@ -310,8 +310,10 @@ private:
   int genChangeDomain(uint32_t id, const std::function<void(DomainInfo&)>& func);
   void deleteDomainRecords(RecordsRWTransaction& txn, uint32_t domain_id, uint16_t qtype = QType::ANY);
 
-  bool findDomain(const DNSName& domain, DomainInfo& info);
-  bool findDomain(uint32_t domainid, DomainInfo& info);
+  bool findDomain(const DNSName& domain, DomainInfo& info) const;
+  bool findDomain(uint32_t domainid, DomainInfo& info) const;
+  void consolidateDomainInfo(DomainInfo& info) const;
+  void writeDomainInfo(const DomainInfo& info);
 
   void getAllDomainsFiltered(vector<DomainInfo>* domains, const std::function<bool(DomainInfo&)>& allow);
 
@@ -324,6 +326,19 @@ private:
   std::string d_matchkey;
   DNSName d_lookupdomain;
 
+  // Cache of DomainInfo notified_serial values
+  class SerialCache : public boost::noncopyable
+  {
+  public:
+    bool get(uint32_t domainid, uint32_t& serial) const;
+    void remove(uint32_t domainid);
+    void update(uint32_t domainid, uint32_t serial);
+
+  private:
+    std::unordered_map<uint32_t, uint32_t> d_serials;
+  };
+  static SharedLockGuarded<SerialCache> s_notified_serial;
+
   vector<LMDBResourceRecord> d_currentrrset;
   size_t d_currentrrsetpos;
   time_t d_currentrrsettime;
@@ -336,6 +351,7 @@ private:
   bool d_dolog;
   bool d_random_ids;
   bool d_handle_dups;
+  bool d_skip_notification_update;
   DTime d_dtime; // used only for logging
   uint64_t d_mapsize;
 };