Base of a working mechanism using a multi-index

Signed-off-by: Otto Moerbeek <otto.moerbeek@open-xchange.com>

diff --git a/pdns/recursordist/rec-keepwarm.hh b/pdns/recursordist/rec-keepwarm.hh
new file mode 100644 (file)
index 0000000..03076e0
--- /dev/null
+++ b/pdns/recursordist/rec-keepwarm.hh
@@ -0,0 +1,100 @@
+/*
+ * This file is part of PowerDNS or dnsdist.
+ * Copyright -- PowerDNS.COM B.V. and its contributors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * In addition, for the avoidance of any doubt, permission is granted to
+ * link this program with OpenSSL and to (re)distribute the binaries
+ * produced as the result of such linking.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+#pragma once
+
+#include <ctime>
+
+#include <boost/multi_index_container.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/key_extractors.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/sequenced_index.hpp>
+#include <boost/multi_index/tag.hpp>
+#include <utility>
+
+#include "dnsname.hh"
+#include "qtype.hh"
+
+namespace rec
+{
+using namespace ::boost::multi_index;
+
+struct KeepWarmEntry
+{
+  KeepWarmEntry(DNSName name, QType qtype, time_t ttd = 0) : d_qname(std::move(name)), d_ttd(ttd), d_qtype(qtype) {}
+  DNSName d_qname;
+  time_t d_ttd;
+  uint16_t d_qtype;
+};
+
+class KeepWarm
+{
+public:
+  struct QNameQTypeTag
+  {
+  };
+
+  struct TTDTag
+  {
+  };
+
+  using Queue = multi_index_container<
+    KeepWarmEntry,
+    indexed_by<ordered_unique<tag<QNameQTypeTag>,
+                              composite_key<KeepWarmEntry,
+                                            member<KeepWarmEntry, DNSName, &KeepWarmEntry::d_qname>,
+                                            member<KeepWarmEntry, uint16_t, &KeepWarmEntry::d_qtype>>>,
+               ordered_non_unique<tag<TTDTag>, member<KeepWarmEntry, time_t, &KeepWarmEntry::d_ttd>, std::less<>>>>;
+
+  [[nodiscard]] const Queue& get() const
+  {
+    return d_queue;
+  }
+  void modifyTTD(const DNSName& qname, uint16_t qtype, uint32_t ttd)
+  {
+      auto item = d_queue.find(std::tie(qname, qtype));
+      if (item != d_queue.end()) {
+        d_queue.modify(item, [ttd](rec::KeepWarmEntry& entry) { entry.d_ttd = ttd; });
+      }
+    
+  }
+  void emplace(const DNSName& name, uint16_t qtype)
+  {
+    d_queue.emplace(name, qtype);
+  }
+  Queue::iterator erase(Queue::iterator iter)
+  {
+    return d_queue.erase(iter);
+  }
+  Queue::iterator begin()
+  {
+    return d_queue.begin();
+  }
+  Queue::iterator end()
+  {
+    return d_queue.end();
+  }
+
+private:
+  Queue d_queue;
+};
+}

diff --git a/pdns/recursordist/rec-main.cc b/pdns/recursordist/rec-main.cc
index f9a53e24d70b23c9610f8b453650303cc528689f..41318f1c497910437c9b95170d8f6c23246c8f43 100644 (file)
--- a/pdns/recursordist/rec-main.cc
+++ b/pdns/recursordist/rec-main.cc
@@ -40,6 +40,7 @@
 #include "threadname.hh"
 #include "version.hh"
 #include "ws-recursor.hh"
+#include "rec-keepwarm.hh"
 
 #ifdef NOD_ENABLED
 #include "nod.hh"
@@ -2454,68 +2455,103 @@ static void handleRCC(int fileDesc, FDMultiplexer::funcparam_t& /* var */)
 static time_t keepCacheWarm(const timeval& now, LocalStateHolder<LuaConfigItems>& luaconfsLocal)
 {
   auto log = g_slog->withName("cachewarmer");
-  time_t wait = 60;
-
-  for (const auto& [qname, qtype] : luaconfsLocal->keepWarm) {
-    SyncRes resolver(now);
-    resolver.setQNameMinimization(true);
-    resolver.setCacheOnly(true);
-    resolver.setDoDNSSEC(g_dnssecmode != DNSSECMode::Off);
-    resolver.setDNSSECValidationRequested(g_dnssecmode != DNSSECMode::Off && g_dnssecmode != DNSSECMode::ProcessNoValidate);
-    std::vector<DNSRecord> ret;
-    int res = -1;
-    const std::string msg = "Exception while resolving";
-    try {
-      res = resolver.beginResolve(qname, qtype, QClass::IN, ret, 0);
-    }
-    catch (const PDNSException& e) {
-      log->error(Logr::Warning, e.reason, msg, "exception", Logging::Loggable("PDNSException"));
-      ret.clear();
-    }
-    catch (const ImmediateServFailException& e) {
-      log->error(Logr::Warning, e.reason, msg, "exception", Logging::Loggable("ImmediateServFailException"));
-      ret.clear();
-    }
-    catch (const PolicyHitException& e) {
-      log->info(Logr::Warning, msg, "exception", Logging::Loggable("PolicyHitException"));
-      ret.clear();
+
+  static LockGuarded<rec::KeepWarm> s_keepwarm;
+  static uint64_t lastgeneration = 0;
+
+  auto lock = s_keepwarm.lock();
+
+  if (lastgeneration != luaconfsLocal->generation) {
+    lastgeneration = luaconfsLocal->generation;
+    for (const auto& [qname, qtype] : luaconfsLocal->keepWarm) {
+      lock->emplace(qname, qtype);
     }
-    catch (const std::exception& e) {
-      log->error(Logr::Warning, e.what(), msg, "exception", Logging::Loggable("std::exception"));
-      ret.clear();
+    std::set<std::pair<DNSName, QType>> all;
+    std::copy(luaconfsLocal->keepWarm.begin(), luaconfsLocal->keepWarm.end(), std::inserter(all, all.end()));
+    for (auto iter = lock->begin(); iter != lock->end();) {
+      if (all.count({iter->d_qname, QType(iter->d_qtype)}) == 0) {
+        iter = lock->erase(iter);
+      }
+      else {
+        ++iter;
+      }
     }
-    catch (...) {
-      log->info(Logr::Warning, msg);
-      ret.clear();
+  }
+
+  std::vector<rec::KeepWarmEntry> toBeHandled;
+
+  auto& sidx = lock->get().template get<rec::KeepWarm::TTDTag>();
+  auto siter = sidx.begin();
+
+  const int batchSize = 100;
+  const time_t specialTime = 1;
+  const time_t cooldown = 60;
+  const time_t almost = 5;
+  for (int i = 0; i < batchSize && siter != sidx.end(); i++, siter++) {
+    if (siter->d_ttd > now.tv_sec + almost) {
+      break;
     }
+    toBeHandled.emplace_back(*siter);
+  }
 
-    if (res == RCode::NoError && ret.size() > 0) {
-      uint32_t minttl = std::numeric_limits<uint32_t>::max();
-      for (const auto& record : ret) {
-        minttl = std::min(minttl, record.d_ttl);
+
+  for (auto& element : toBeHandled) {
+    if (element.d_ttd == specialTime) {
+      SyncRes resolver(now);
+      resolver.setQNameMinimization(true);
+      resolver.setCacheOnly(true);
+      resolver.setDoDNSSEC(g_dnssecmode != DNSSECMode::Off);
+      resolver.setDNSSECValidationRequested(g_dnssecmode != DNSSECMode::Off && g_dnssecmode != DNSSECMode::ProcessNoValidate);
+      std::vector<DNSRecord> ret;
+      const std::string msg = "Exception while resolving";
+      try {
+        resolver.beginResolve(element.d_qname, element.d_qtype, QClass::IN, ret, 0);
       }
-      if (minttl > 5) {
-        wait = std::min(wait, static_cast<time_t>(minttl - 5));
-        continue;
+      catch (const PDNSException& e) {
+        log->error(Logr::Warning, e.reason, msg, "exception", Logging::Loggable("PDNSException"));
+        ret.clear();
+      }
+      catch (const ImmediateServFailException& e) {
+        log->error(Logr::Warning, e.reason, msg, "exception", Logging::Loggable("ImmediateServFailException"));
+        ret.clear();
+      }
+      catch (const PolicyHitException& e) {
+        log->info(Logr::Warning, msg, "exception", Logging::Loggable("PolicyHitException"));
+        ret.clear();
+      }
+      catch (const std::exception& e) {
+        log->error(Logr::Warning, e.what(), msg, "exception", Logging::Loggable("std::exception"));
+        ret.clear();
+      }
+      catch (...) {
+        log->info(Logr::Warning, msg);
+        ret.clear();
       }
-      wait = std::min(wait, static_cast<time_t>(1));
-    }
 
-    NegCache::NegCacheEntry negEntry;
-    bool inNegCache = g_negCache->get(qname, qtype, now, negEntry, false);
-    if (!inNegCache) {
-      log->info(Logr::Debug, "Absent or expiring and not in negache, pushing task", "qname", Logging::Loggable(qname),
-                "qtype", Logging::Loggable(qtype),
-                "res", Logging::Loggable(res), "size", Logging::Loggable(ret.size()));
-      // Work to be done
-      pushAlmostExpiredTask(qname, qtype, now.tv_sec + 60, ComboAddress("255.255.255.255"), true);
-      wait = std::min(wait, static_cast<time_t>(1));
+      uint32_t minttl = cooldown; // If no records found, either it did not resolve at all, or it did
+                                  // not resolve yet. In both cases, pace the work.
+      if (ret.size() > 0) {
+        minttl = std::numeric_limits<uint32_t>::max();
+        for (const auto& record : ret) {
+          minttl = std::min(minttl, record.d_ttl);
+        }
+      }
+      lock->modifyTTD(element.d_qname, element.d_qtype, now.tv_sec + minttl);
     }
-    else {
-      auto expiring = negEntry.d_ttd - now.tv_sec;
-      wait = std::min(wait, expiring);
+    else if (element.d_ttd == 0 || element.d_ttd <= now.tv_sec + almost) {
+      pushAlmostExpiredTask(element.d_qname, element.d_qtype, now.tv_sec + cooldown, ComboAddress("255.255.255.255"), true);
+        lock->modifyTTD(element.d_qname, element.d_qtype, specialTime);
     }
   }
+
+  time_t wait = cooldown;
+  siter = sidx.begin();
+  if (siter != sidx.end()) {
+    wait = siter->d_ttd - now.tv_sec - 1;
+    wait = std::max(static_cast<time_t>(1), wait);
+    wait = std::min(static_cast<time_t>(cooldown), wait);
+  }
+
   log->info(Logr::Debug, "Wait", "interval", Logging::Loggable(wait));
   return wait;
 }

diff --git a/pdns/recursordist/rec_channel_rec.cc b/pdns/recursordist/rec_channel_rec.cc
index 1b5f6dd57a0c77e94187a1cf18fce369a2555525..a05d0f8fe492cf90d722390026a996b688c6aca9 100644 (file)
--- a/pdns/recursordist/rec_channel_rec.cc
+++ b/pdns/recursordist/rec_channel_rec.cc
@@ -2130,6 +2130,8 @@ RecursorControlChannel::Answer luaconfig(bool broadcast)
         pdns::settings::rec::fromBridgeStructToLuaConfig(settings, dummyLuaConfig, dummyProxyMapping, conditions);
         TCPOutConnectionManager::setupOutgoingTLSConfigTables(settings);
         lci.keepWarm = dummyLuaConfig.keepWarm; // XXX
+        auto generation = g_luaconfs.getLocal()->generation;
+        lci.generation = generation + 1;
       }
       if (!::arg()["lua-config-file"].empty()) {
         loadRecursorLuaConfig(::arg()["lua-config-file"], proxyMapping, lci); // will bump generation