From: Otto Moerbeek <otto.moerbeek@open-xchange.com>
Date: Wed, 20 Mar 2024 13:52:38 +0000 (+0100)
Subject: Make snapshot write interval settable
X-Git-Tag: rec-5.1.0-alpha1~77^2~3
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=60be7ecb53c4715c8b8d832ec2268b100b827cd3;p=thirdparty%2Fpdns.git

Make snapshot write interval settable
---

diff --git a/pdns/recursordist/nod.cc b/pdns/recursordist/nod.cc
index 6a1f4277dd..4aae73d97a 100644
--- a/pdns/recursordist/nod.cc
+++ b/pdns/recursordist/nod.cc
@@ -43,7 +43,7 @@ std::mutex PersistentSBF::d_cachedir_mutex;
 void PersistentSBF::remove_tmp_files(const filesystem::path& path, std::lock_guard<std::mutex>& /* lock */)
 {
   Regex file_regex(d_prefix + ".*\\." + bf_suffix + "\\..{8}$");
-  for (const auto& file : filesystem::directory_iterator (path)) {
+  for (const auto& file : filesystem::directory_iterator(path)) {
     if (filesystem::is_regular_file(file.path()) && file_regex.match(file.path().filename().string())) {
       filesystem::remove(file);
     }
@@ -67,11 +67,11 @@ bool PersistentSBF::init(bool ignore_pid)
         filesystem::path newest_file;
         std::time_t newest_time = 0;
         Regex file_regex(d_prefix + ".*\\." + bf_suffix + "$");
-        for (const auto& file : filesystem::directory_iterator (path)) {
+        for (const auto& file : filesystem::directory_iterator(path)) {
           if (filesystem::is_regular_file(file.path()) && file_regex.match(file.path().filename().string())) {
             if (ignore_pid || (file.path().filename().string().find(std::to_string(getpid())) == std::string::npos)) {
               // look for the newest file matching the regex
-              if (last_write_time(file.path()) > newest_time) { 
+              if (last_write_time(file.path()) > newest_time) {
                 newest_time = last_write_time(file.path());
                 newest_file = file.path();
               }
@@ -204,8 +204,6 @@ bool NODDB::isNewDomain(const std::string& domain)
 bool NODDB::isNewDomain(const DNSName& dname)
 {
   std::string dname_lc = dname.toDNSStringLC();
-  // The only time this should block is when snapshotting from the
-  // housekeeping thread
   // the result is always the inverse of what is returned by the SBF
   return !d_psbf.testAndAdd(dname_lc);
 }
diff --git a/pdns/recursordist/nod.hh b/pdns/recursordist/nod.hh
index 4e6d1e7d7d..2a5bb3edf4 100644
--- a/pdns/recursordist/nod.hh
+++ b/pdns/recursordist/nod.hh
@@ -33,7 +33,7 @@ namespace nod
 const float c_fp_rate = 0.01;
 const size_t c_num_cells = 67108864;
 const uint8_t c_num_dec = 10;
-  const unsigned int snapshot_interval_default = 30; // XXX 600;
+const unsigned int snapshot_interval_default = 600;
 const std::string bf_suffix = "bf";
 const std::string sbf_prefix = "sbf";
 
diff --git a/pdns/recursordist/rec-main.cc b/pdns/recursordist/rec-main.cc
index 075fbec059..3f451d3008 100644
--- a/pdns/recursordist/rec-main.cc
+++ b/pdns/recursordist/rec-main.cc
@@ -796,10 +796,13 @@ static void setupNODThread(Logr::log_t log)
            log->info(Logr::Error, "Could not initialize domain tracking"));
       _exit(1);
     }
-    std::thread thread([tid = std::this_thread::get_id()]() {
-      g_nodDBp->housekeepingThread(tid);
-    });
-    thread.detach();
+    if (::arg().asNum("new-domain-db-snapshot-interval") > 0) {
+      g_nodDBp->setSnapshotInterval(::arg().asNum("new-domain-db-snapshot-interval"));
+      std::thread thread([tid = std::this_thread::get_id()]() {
+        g_nodDBp->housekeepingThread(tid);
+      });
+      thread.detach();
+    }
   }
   if (g_udrEnabled) {
     uint32_t num_cells = ::arg().asNum("unique-response-db-size");
@@ -817,10 +820,13 @@ static void setupNODThread(Logr::log_t log)
            log->info(Logr::Error, "Could not initialize unique response tracking"));
       _exit(1);
     }
-    std::thread thread([tid = std::this_thread::get_id()]() {
-      g_udrDBp->housekeepingThread(tid);
-    });
-    thread.detach();
+    if (::arg().asNum("new-domain-db-snapshot-interval") > 0) {
+      g_udrDBp->setSnapshotInterval(::arg().asNum("new-domain-db-snapshot-interval"));
+      std::thread thread([tid = std::this_thread::get_id()]() {
+        g_udrDBp->housekeepingThread(tid);
+      });
+      thread.detach();
+    }
   }
 }
 
diff --git a/pdns/recursordist/settings/table.py b/pdns/recursordist/settings/table.py
index 5028b4c5f4..7e374aeb66 100644
--- a/pdns/recursordist/settings/table.py
+++ b/pdns/recursordist/settings/table.py
@@ -1737,6 +1737,19 @@ from this directory.
  ''',
     'versionadded': '4.2.0'
     },
+    {
+        'name' : 'db_snapshot_interval',
+        'section' : 'nod',
+        'oldname' : 'new-domain-db-snapshot-interval',
+        'type' : LType.Uint64,
+        'default' : '600',
+        'help' : 'Interval (in seconds) to write the NOD and UDR DB snapshots',
+        'doc' : '''
+Interval (in seconds) to write the NOD and UDR DB snapshots.
+Set to zero to disable snapshot writing.',
+ ''',
+        'versionadded': '5.1.0'
+    },
     {
         'name' : 'whitelist',
         'section' : 'nod',
diff --git a/pdns/recursordist/stable-bloom.hh b/pdns/recursordist/stable-bloom.hh
index 1c1b7b1e1b..a025eb8e8d 100644
--- a/pdns/recursordist/stable-bloom.hh
+++ b/pdns/recursordist/stable-bloom.hh
@@ -29,7 +29,6 @@
 #include <boost/dynamic_bitset.hpp>
 #include "misc.hh"
 #include "noinitvector.hh"
-#include "views.hh"
 #include "ext/probds/murmur3.h"
 
 namespace bf