- Print shard info in cache dumps

- change # of shard of negcache to be 1/8th of main record cache, same for size

diff --git a/pdns/recursordist/negcache.cc b/pdns/recursordist/negcache.cc
index 4da1d92021179d1fdcbebf06cf3194b11f114c9a..9370c531cba1af93716406aeb1d700374af728f6 100644 (file)
--- a/pdns/recursordist/negcache.cc
+++ b/pdns/recursordist/negcache.cc
@@ -258,7 +258,10 @@ void NegCache::clear()
 void NegCache::prune(size_t maxEntries)
 {
   size_t cacheSize = size();
+  cerr << "======= NegCache =======" << endl;
   pruneMutexCollectionsVector<SequenceTag>(*this, d_maps, maxEntries, cacheSize);
+  cerr << "NegCache size is now " << size() << endl;
+  cerr << "========================" << endl;
 }
 
 /*!
@@ -266,30 +269,51 @@ void NegCache::prune(size_t maxEntries)
  *
  * \param fp A pointer to an open FILE object
  */
-size_t NegCache::dumpToFile(FILE* fp, const struct timeval& now)
+size_t NegCache::doDump(int fd, size_t maxCacheEntries)
 {
+  int newfd = dup(fd);
+  if (newfd == -1) {
+    return 0;
+  }
+  auto fp = std::unique_ptr<FILE, int (*)(FILE*)>(fdopen(newfd, "w"), fclose);
+  if (!fp) {
+    return 0;
+  }
+  fprintf(fp.get(), "; negcache dump follows\n;\n");
+
+  struct timeval now;
+  Utility::gettimeofday(&now, nullptr);
+
   size_t ret = 0;
 
+  size_t shard = 0;
+  size_t min = std::numeric_limits<size_t>::max();
+  size_t max = 0;
   for (auto& mc : d_maps) {
     auto m = mc.lock();
+    const auto shardSize = m->d_map.size();
+    min = std::min(min, shardSize);
+    max = std::max(max, shardSize);
+    shard++;
     auto& sidx = m->d_map.get<SequenceTag>();
     for (const NegCacheEntry& ne : sidx) {
       ret++;
       int64_t ttl = ne.d_ttd - now.tv_sec;
-      fprintf(fp, "%s %" PRId64 " IN %s VIA %s ; (%s) origttl=%" PRIu32 " ss=%hu\n", ne.d_name.toString().c_str(), ttl, ne.d_qtype.toString().c_str(), ne.d_auth.toString().c_str(), vStateToString(ne.d_validationState).c_str(), ne.d_orig_ttl, ne.d_servedStale);
+      fprintf(fp.get(), "%s %" PRId64 " IN %s VIA %s ; (%s) origttl=%" PRIu32 " ss=%hu\n", ne.d_name.toString().c_str(), ttl, ne.d_qtype.toString().c_str(), ne.d_auth.toString().c_str(), vStateToString(ne.d_validationState).c_str(), ne.d_orig_ttl, ne.d_servedStale);
       for (const auto& rec : ne.authoritySOA.records) {
-        fprintf(fp, "%s %" PRId64 " IN %s %s ; (%s)\n", rec.d_name.toString().c_str(), ttl, DNSRecordContent::NumberToType(rec.d_type).c_str(), rec.d_content->getZoneRepresentation().c_str(), vStateToString(ne.d_validationState).c_str());
+        fprintf(fp.get(), "%s %" PRId64 " IN %s %s ; (%s)\n", rec.d_name.toString().c_str(), ttl, DNSRecordContent::NumberToType(rec.d_type).c_str(), rec.d_content->getZoneRepresentation().c_str(), vStateToString(ne.d_validationState).c_str());
       }
       for (const auto& sig : ne.authoritySOA.signatures) {
-        fprintf(fp, "%s %" PRId64 " IN RRSIG %s ;\n", sig.d_name.toString().c_str(), ttl, sig.d_content->getZoneRepresentation().c_str());
+        fprintf(fp.get(), "%s %" PRId64 " IN RRSIG %s ;\n", sig.d_name.toString().c_str(), ttl, sig.d_content->getZoneRepresentation().c_str());
       }
       for (const auto& rec : ne.DNSSECRecords.records) {
-        fprintf(fp, "%s %" PRId64 " IN %s %s ; (%s)\n", rec.d_name.toString().c_str(), ttl, DNSRecordContent::NumberToType(rec.d_type).c_str(), rec.d_content->getZoneRepresentation().c_str(), vStateToString(ne.d_validationState).c_str());
+        fprintf(fp.get(), "%s %" PRId64 " IN %s %s ; (%s)\n", rec.d_name.toString().c_str(), ttl, DNSRecordContent::NumberToType(rec.d_type).c_str(), rec.d_content->getZoneRepresentation().c_str(), vStateToString(ne.d_validationState).c_str());
       }
       for (const auto& sig : ne.DNSSECRecords.signatures) {
-        fprintf(fp, "%s %" PRId64 " IN RRSIG %s ;\n", sig.d_name.toString().c_str(), ttl, sig.d_content->getZoneRepresentation().c_str());
+        fprintf(fp.get(), "%s %" PRId64 " IN RRSIG %s ;\n", sig.d_name.toString().c_str(), ttl, sig.d_content->getZoneRepresentation().c_str());
       }
     }
   }
+  fprintf(fp.get(), "; negcache size: %zu/%zu shards: %zu min/max shard size: %zu/%zu\n", size(), maxCacheEntries, d_maps.size(), min, max);
   return ret;
 }

diff --git a/pdns/recursordist/negcache.hh b/pdns/recursordist/negcache.hh
index 4afe1664146d0161fb2a544c51d05ab90eebeff0..01a4e402b665a3ecb6e7362220fe787be56d65e0 100644 (file)
--- a/pdns/recursordist/negcache.hh
+++ b/pdns/recursordist/negcache.hh
@@ -54,7 +54,7 @@ typedef struct
 class NegCache : public boost::noncopyable
 {
 public:
-  NegCache(size_t mapsCount = 1024);
+  NegCache(size_t mapsCount = 128);
 
   // For a description on how ServeStale works, see recursor_cache.cc, the general structure is the same.
   // The number of times a stale cache entry is extended
@@ -94,7 +94,7 @@ public:
   size_t count(const DNSName& qname, const QType qtype);
   void prune(size_t maxEntries);
   void clear();
-  size_t dumpToFile(FILE* fd, const struct timeval& now);
+  size_t doDump(int fd, size_t maxCacheEntries);
   size_t wipe(const DNSName& name, bool subtree = false);
   size_t size() const;
 

diff --git a/pdns/recursordist/rec-main.cc b/pdns/recursordist/rec-main.cc
index b30f7fa26266e371781933ccbd3a9d93c61a0c81..db234628aa7842703421fd115ffe22a8b62b261c 100644 (file)
--- a/pdns/recursordist/rec-main.cc
+++ b/pdns/recursordist/rec-main.cc
@@ -2187,7 +2187,7 @@ static void houseKeeping(void*)
 
       static PeriodicTask negCachePruneTask{"NegCachePrunteTask", 5};
       negCachePruneTask.runIfDue(now, []() {
-        g_negCache->prune(g_maxCacheEntries / 10);
+        g_negCache->prune(g_maxCacheEntries / 8);
       });
 
       static PeriodicTask aggrNSECPruneTask{"AggrNSECPruneTask", 5};
@@ -3027,7 +3027,7 @@ int main(int argc, char** argv)
     }
 
     g_recCache = std::make_unique<MemRecursorCache>(::arg().asNum("record-cache-shards"));
-    g_negCache = std::make_unique<NegCache>(::arg().asNum("record-cache-shards"));
+    g_negCache = std::make_unique<NegCache>(::arg().asNum("record-cache-shards") / 8);
 
     ret = serviceMain(argc, argv, startupLog);
   }

diff --git a/pdns/recursordist/rec_channel_rec.cc b/pdns/recursordist/rec_channel_rec.cc
index 6db7e91d8a88f3c8cd943fde9ea4099c5233ba05..dd8f7f266a20c39b7546025e58bcdd4b7588689c 100644 (file)
--- a/pdns/recursordist/rec_channel_rec.cc
+++ b/pdns/recursordist/rec_channel_rec.cc
@@ -313,23 +313,6 @@ getfd(int s)
   return FDWrapper(fd);
 }
 
-static uint64_t dumpNegCache(int fd)
-{
-  int newfd = dup(fd);
-  if (newfd == -1) {
-    return 0;
-  }
-  auto fp = std::unique_ptr<FILE, int (*)(FILE*)>(fdopen(newfd, "w"), fclose);
-  if (!fp) {
-    return 0;
-  }
-  fprintf(fp.get(), "; negcache dump follows\n;\n");
-
-  struct timeval now;
-  Utility::gettimeofday(&now, nullptr);
-  return g_negCache->dumpToFile(fp.get(), now);
-}
-
 static uint64_t dumpAggressiveNSECCache(int fd)
 {
   if (!g_aggressiveNSECCache) {
@@ -433,7 +416,7 @@ static RecursorControlChannel::Answer doDumpCache(int s)
   uint64_t total = 0;
   try {
     int fd = fdw;
-    total = g_recCache->doDump(fd) + dumpNegCache(fd) + broadcastAccFunction<uint64_t>([fd] { return pleaseDump(fd); }) + dumpAggressiveNSECCache(fd);
+    total = g_recCache->doDump(fd, g_maxCacheEntries.load()) + g_negCache->doDump(fd, g_maxCacheEntries.load() / 8) + broadcastAccFunction<uint64_t>([fd] { return pleaseDump(fd); }) + dumpAggressiveNSECCache(fd);
   }
   catch (...) {
   }

diff --git a/pdns/recursordist/recursor_cache.cc b/pdns/recursordist/recursor_cache.cc
index 8f9e440f4dadd9f7c802b4bc430a2ca9d746f23f..3796c70f4408ea2041b9cc282bcca47c71684149 100644 (file)
--- a/pdns/recursordist/recursor_cache.cc
+++ b/pdns/recursordist/recursor_cache.cc
@@ -761,7 +761,7 @@ bool MemRecursorCache::updateValidationStatus(time_t now, const DNSName& qname,
   return updated;
 }
 
-uint64_t MemRecursorCache::doDump(int fd)
+uint64_t MemRecursorCache::doDump(int fd, size_t maxCacheEntries)
 {
   int newfd = dup(fd);
   if (newfd == -1) {
@@ -775,11 +775,16 @@ uint64_t MemRecursorCache::doDump(int fd)
 
   fprintf(fp.get(), "; main record cache dump follows\n;\n");
   uint64_t count = 0;
-
+  size_t shard = 0;
+  size_t min = std::numeric_limits<size_t>::max();
+  size_t max = 0;
   for (auto& mc : d_maps) {
     auto map = mc.lock();
+    const auto shardSize = map->d_map.size();
+    min = std::min(min, shardSize);
+    max = std::max(max, shardSize);
+    shard++;
     const auto& sidx = map->d_map.get<SequencedTag>();
-
     time_t now = time(nullptr);
     for (const auto& i : sidx) {
       for (const auto& j : i.d_records) {
@@ -802,13 +807,17 @@ uint64_t MemRecursorCache::doDump(int fd)
       }
     }
   }
+  fprintf(fp.get(), "; main record cache size: %zu/%zu shards: %zu min/max shard size: %zu/%zu\n", size(), maxCacheEntries, d_maps.size(), min, max);
   return count;
 }
 
 void MemRecursorCache::doPrune(size_t keep)
 {
   size_t cacheSize = size();
+  cerr << "=====-Cache=============" << endl;
   pruneMutexCollectionsVector<SequencedTag>(*this, d_maps, keep, cacheSize);
+  cerr << "Size is now " << size() << endl;
+  cerr << "========================" << endl;
 }
 
 namespace boost

diff --git a/pdns/recursordist/recursor_cache.hh b/pdns/recursordist/recursor_cache.hh
index bd3b0eb859e788bb00f19f59bae611dbbb5d4541..91102d99ae2e8fe66f57ab72043f542545d51b2f 100644 (file)
--- a/pdns/recursordist/recursor_cache.hh
+++ b/pdns/recursordist/recursor_cache.hh
@@ -72,7 +72,7 @@ public:
   void replace(time_t, const DNSName& qname, const QType qt, const vector<DNSRecord>& content, const vector<shared_ptr<RRSIGRecordContent>>& signatures, const std::vector<std::shared_ptr<DNSRecord>>& authorityRecs, bool auth, const DNSName& authZone, boost::optional<Netmask> ednsmask = boost::none, const OptTag& routingTag = boost::none, vState state = vState::Indeterminate, boost::optional<ComboAddress> from = boost::none, bool refresh = false);
 
   void doPrune(size_t keep);
-  uint64_t doDump(int fd);
+  uint64_t doDump(int fd, size_t maxCacheEntries);
 
   size_t doWipeCache(const DNSName& name, bool sub, QType qtype = 0xffff);
   bool doAgeCache(time_t now, const DNSName& name, QType qtype, uint32_t newTTL);

diff --git a/pdns/recursordist/test-negcache_cc.cc b/pdns/recursordist/test-negcache_cc.cc
index ae448a5b8912bac5e8752021e7c03fa3c5381619..4bbce4ababc1c6023459eb2552ffc29b2945d8e8 100644 (file)
--- a/pdns/recursordist/test-negcache_cc.cc
+++ b/pdns/recursordist/test-negcache_cc.cc
@@ -440,17 +440,21 @@ BOOST_AUTO_TEST_CASE(test_clear)
 BOOST_AUTO_TEST_CASE(test_dumpToFile)
 {
   NegCache cache(1);
-  vector<string> expected;
-  expected.push_back("www1.powerdns.com. 600 IN TYPE0 VIA powerdns.com. ; (Indeterminate) origttl=600 ss=0\n");
-  expected.push_back("powerdns.com. 600 IN SOA ns1. hostmaster. 1 2 3 4 5 ; (Indeterminate)\n");
-  expected.push_back("powerdns.com. 600 IN RRSIG SOA 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n");
-  expected.push_back("powerdns.com. 600 IN NSEC deadbeef. ; (Indeterminate)\n");
-  expected.push_back("powerdns.com. 600 IN RRSIG NSEC 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n");
-  expected.push_back("www2.powerdns.com. 600 IN TYPE0 VIA powerdns.com. ; (Indeterminate) origttl=600 ss=0\n");
-  expected.push_back("powerdns.com. 600 IN SOA ns1. hostmaster. 1 2 3 4 5 ; (Indeterminate)\n");
-  expected.push_back("powerdns.com. 600 IN RRSIG SOA 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n");
-  expected.push_back("powerdns.com. 600 IN NSEC deadbeef. ; (Indeterminate)\n");
-  expected.push_back("powerdns.com. 600 IN RRSIG NSEC 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n");
+  vector<string> expected = {
+    "; negcache dump follows\n",
+    ";\n",
+    "www1.powerdns.com. 600 IN TYPE0 VIA powerdns.com. ; (Indeterminate) origttl=600 ss=0\n",
+    "powerdns.com. 600 IN SOA ns1. hostmaster. 1 2 3 4 5 ; (Indeterminate)\n",
+    "powerdns.com. 600 IN RRSIG SOA 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n",
+    "powerdns.com. 600 IN NSEC deadbeef. ; (Indeterminate)\n",
+    "powerdns.com. 600 IN RRSIG NSEC 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n",
+    "www2.powerdns.com. 600 IN TYPE0 VIA powerdns.com. ; (Indeterminate) origttl=600 ss=0\n",
+    "powerdns.com. 600 IN SOA ns1. hostmaster. 1 2 3 4 5 ; (Indeterminate)\n",
+    "powerdns.com. 600 IN RRSIG SOA 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n",
+    "powerdns.com. 600 IN NSEC deadbeef. ; (Indeterminate)\n",
+    "powerdns.com. 600 IN RRSIG NSEC 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n",
+    "; negcache size: 2/0 shards: 1 min/max shard size: 2/2\n"
+    };
 
   struct timeval now;
   Utility::gettimeofday(&now, 0);
@@ -462,7 +466,7 @@ BOOST_AUTO_TEST_CASE(test_dumpToFile)
   if (!fp)
     BOOST_FAIL("Temporary file could not be opened");
 
-  cache.dumpToFile(fp.get(), now);
+  cache.doDump(fileno(fp.get()), 0);
 
   rewind(fp.get());
   char* line = nullptr;