From: Otto Moerbeek Date: Tue, 3 Jan 2023 10:35:14 +0000 (+0100) Subject: - Print shard info in cache dumps X-Git-Tag: dnsdist-1.8.0-rc1~137^2~2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=911738a77f716ed460c46767dfe17e348371fda8;p=thirdparty%2Fpdns.git - Print shard info in cache dumps - change # of shard of negcache to be 1/8th of main record cache, same for size --- diff --git a/pdns/recursordist/negcache.cc b/pdns/recursordist/negcache.cc index 4da1d92021..9370c531cb 100644 --- a/pdns/recursordist/negcache.cc +++ b/pdns/recursordist/negcache.cc @@ -258,7 +258,10 @@ void NegCache::clear() void NegCache::prune(size_t maxEntries) { size_t cacheSize = size(); + cerr << "======= NegCache =======" << endl; pruneMutexCollectionsVector(*this, d_maps, maxEntries, cacheSize); + cerr << "NegCache size is now " << size() << endl; + cerr << "========================" << endl; } /*! @@ -266,30 +269,51 @@ void NegCache::prune(size_t maxEntries) * * \param fp A pointer to an open FILE object */ -size_t NegCache::dumpToFile(FILE* fp, const struct timeval& now) +size_t NegCache::doDump(int fd, size_t maxCacheEntries) { + int newfd = dup(fd); + if (newfd == -1) { + return 0; + } + auto fp = std::unique_ptr(fdopen(newfd, "w"), fclose); + if (!fp) { + return 0; + } + fprintf(fp.get(), "; negcache dump follows\n;\n"); + + struct timeval now; + Utility::gettimeofday(&now, nullptr); + size_t ret = 0; + size_t shard = 0; + size_t min = std::numeric_limits::max(); + size_t max = 0; for (auto& mc : d_maps) { auto m = mc.lock(); + const auto shardSize = m->d_map.size(); + min = std::min(min, shardSize); + max = std::max(max, shardSize); + shard++; auto& sidx = m->d_map.get(); for (const NegCacheEntry& ne : sidx) { ret++; int64_t ttl = ne.d_ttd - now.tv_sec; - fprintf(fp, "%s %" PRId64 " IN %s VIA %s ; (%s) origttl=%" PRIu32 " ss=%hu\n", ne.d_name.toString().c_str(), ttl, ne.d_qtype.toString().c_str(), ne.d_auth.toString().c_str(), vStateToString(ne.d_validationState).c_str(), ne.d_orig_ttl, ne.d_servedStale); + fprintf(fp.get(), "%s %" PRId64 " IN %s VIA %s ; (%s) origttl=%" PRIu32 " ss=%hu\n", ne.d_name.toString().c_str(), ttl, ne.d_qtype.toString().c_str(), ne.d_auth.toString().c_str(), vStateToString(ne.d_validationState).c_str(), ne.d_orig_ttl, ne.d_servedStale); for (const auto& rec : ne.authoritySOA.records) { - fprintf(fp, "%s %" PRId64 " IN %s %s ; (%s)\n", rec.d_name.toString().c_str(), ttl, DNSRecordContent::NumberToType(rec.d_type).c_str(), rec.d_content->getZoneRepresentation().c_str(), vStateToString(ne.d_validationState).c_str()); + fprintf(fp.get(), "%s %" PRId64 " IN %s %s ; (%s)\n", rec.d_name.toString().c_str(), ttl, DNSRecordContent::NumberToType(rec.d_type).c_str(), rec.d_content->getZoneRepresentation().c_str(), vStateToString(ne.d_validationState).c_str()); } for (const auto& sig : ne.authoritySOA.signatures) { - fprintf(fp, "%s %" PRId64 " IN RRSIG %s ;\n", sig.d_name.toString().c_str(), ttl, sig.d_content->getZoneRepresentation().c_str()); + fprintf(fp.get(), "%s %" PRId64 " IN RRSIG %s ;\n", sig.d_name.toString().c_str(), ttl, sig.d_content->getZoneRepresentation().c_str()); } for (const auto& rec : ne.DNSSECRecords.records) { - fprintf(fp, "%s %" PRId64 " IN %s %s ; (%s)\n", rec.d_name.toString().c_str(), ttl, DNSRecordContent::NumberToType(rec.d_type).c_str(), rec.d_content->getZoneRepresentation().c_str(), vStateToString(ne.d_validationState).c_str()); + fprintf(fp.get(), "%s %" PRId64 " IN %s %s ; (%s)\n", rec.d_name.toString().c_str(), ttl, DNSRecordContent::NumberToType(rec.d_type).c_str(), rec.d_content->getZoneRepresentation().c_str(), vStateToString(ne.d_validationState).c_str()); } for (const auto& sig : ne.DNSSECRecords.signatures) { - fprintf(fp, "%s %" PRId64 " IN RRSIG %s ;\n", sig.d_name.toString().c_str(), ttl, sig.d_content->getZoneRepresentation().c_str()); + fprintf(fp.get(), "%s %" PRId64 " IN RRSIG %s ;\n", sig.d_name.toString().c_str(), ttl, sig.d_content->getZoneRepresentation().c_str()); } } } + fprintf(fp.get(), "; negcache size: %zu/%zu shards: %zu min/max shard size: %zu/%zu\n", size(), maxCacheEntries, d_maps.size(), min, max); return ret; } diff --git a/pdns/recursordist/negcache.hh b/pdns/recursordist/negcache.hh index 4afe166414..01a4e402b6 100644 --- a/pdns/recursordist/negcache.hh +++ b/pdns/recursordist/negcache.hh @@ -54,7 +54,7 @@ typedef struct class NegCache : public boost::noncopyable { public: - NegCache(size_t mapsCount = 1024); + NegCache(size_t mapsCount = 128); // For a description on how ServeStale works, see recursor_cache.cc, the general structure is the same. // The number of times a stale cache entry is extended @@ -94,7 +94,7 @@ public: size_t count(const DNSName& qname, const QType qtype); void prune(size_t maxEntries); void clear(); - size_t dumpToFile(FILE* fd, const struct timeval& now); + size_t doDump(int fd, size_t maxCacheEntries); size_t wipe(const DNSName& name, bool subtree = false); size_t size() const; diff --git a/pdns/recursordist/rec-main.cc b/pdns/recursordist/rec-main.cc index b30f7fa262..db234628aa 100644 --- a/pdns/recursordist/rec-main.cc +++ b/pdns/recursordist/rec-main.cc @@ -2187,7 +2187,7 @@ static void houseKeeping(void*) static PeriodicTask negCachePruneTask{"NegCachePrunteTask", 5}; negCachePruneTask.runIfDue(now, []() { - g_negCache->prune(g_maxCacheEntries / 10); + g_negCache->prune(g_maxCacheEntries / 8); }); static PeriodicTask aggrNSECPruneTask{"AggrNSECPruneTask", 5}; @@ -3027,7 +3027,7 @@ int main(int argc, char** argv) } g_recCache = std::make_unique(::arg().asNum("record-cache-shards")); - g_negCache = std::make_unique(::arg().asNum("record-cache-shards")); + g_negCache = std::make_unique(::arg().asNum("record-cache-shards") / 8); ret = serviceMain(argc, argv, startupLog); } diff --git a/pdns/recursordist/rec_channel_rec.cc b/pdns/recursordist/rec_channel_rec.cc index 6db7e91d8a..dd8f7f266a 100644 --- a/pdns/recursordist/rec_channel_rec.cc +++ b/pdns/recursordist/rec_channel_rec.cc @@ -313,23 +313,6 @@ getfd(int s) return FDWrapper(fd); } -static uint64_t dumpNegCache(int fd) -{ - int newfd = dup(fd); - if (newfd == -1) { - return 0; - } - auto fp = std::unique_ptr(fdopen(newfd, "w"), fclose); - if (!fp) { - return 0; - } - fprintf(fp.get(), "; negcache dump follows\n;\n"); - - struct timeval now; - Utility::gettimeofday(&now, nullptr); - return g_negCache->dumpToFile(fp.get(), now); -} - static uint64_t dumpAggressiveNSECCache(int fd) { if (!g_aggressiveNSECCache) { @@ -433,7 +416,7 @@ static RecursorControlChannel::Answer doDumpCache(int s) uint64_t total = 0; try { int fd = fdw; - total = g_recCache->doDump(fd) + dumpNegCache(fd) + broadcastAccFunction([fd] { return pleaseDump(fd); }) + dumpAggressiveNSECCache(fd); + total = g_recCache->doDump(fd, g_maxCacheEntries.load()) + g_negCache->doDump(fd, g_maxCacheEntries.load() / 8) + broadcastAccFunction([fd] { return pleaseDump(fd); }) + dumpAggressiveNSECCache(fd); } catch (...) { } diff --git a/pdns/recursordist/recursor_cache.cc b/pdns/recursordist/recursor_cache.cc index 8f9e440f4d..3796c70f44 100644 --- a/pdns/recursordist/recursor_cache.cc +++ b/pdns/recursordist/recursor_cache.cc @@ -761,7 +761,7 @@ bool MemRecursorCache::updateValidationStatus(time_t now, const DNSName& qname, return updated; } -uint64_t MemRecursorCache::doDump(int fd) +uint64_t MemRecursorCache::doDump(int fd, size_t maxCacheEntries) { int newfd = dup(fd); if (newfd == -1) { @@ -775,11 +775,16 @@ uint64_t MemRecursorCache::doDump(int fd) fprintf(fp.get(), "; main record cache dump follows\n;\n"); uint64_t count = 0; - + size_t shard = 0; + size_t min = std::numeric_limits::max(); + size_t max = 0; for (auto& mc : d_maps) { auto map = mc.lock(); + const auto shardSize = map->d_map.size(); + min = std::min(min, shardSize); + max = std::max(max, shardSize); + shard++; const auto& sidx = map->d_map.get(); - time_t now = time(nullptr); for (const auto& i : sidx) { for (const auto& j : i.d_records) { @@ -802,13 +807,17 @@ uint64_t MemRecursorCache::doDump(int fd) } } } + fprintf(fp.get(), "; main record cache size: %zu/%zu shards: %zu min/max shard size: %zu/%zu\n", size(), maxCacheEntries, d_maps.size(), min, max); return count; } void MemRecursorCache::doPrune(size_t keep) { size_t cacheSize = size(); + cerr << "=====-Cache=============" << endl; pruneMutexCollectionsVector(*this, d_maps, keep, cacheSize); + cerr << "Size is now " << size() << endl; + cerr << "========================" << endl; } namespace boost diff --git a/pdns/recursordist/recursor_cache.hh b/pdns/recursordist/recursor_cache.hh index bd3b0eb859..91102d99ae 100644 --- a/pdns/recursordist/recursor_cache.hh +++ b/pdns/recursordist/recursor_cache.hh @@ -72,7 +72,7 @@ public: void replace(time_t, const DNSName& qname, const QType qt, const vector& content, const vector>& signatures, const std::vector>& authorityRecs, bool auth, const DNSName& authZone, boost::optional ednsmask = boost::none, const OptTag& routingTag = boost::none, vState state = vState::Indeterminate, boost::optional from = boost::none, bool refresh = false); void doPrune(size_t keep); - uint64_t doDump(int fd); + uint64_t doDump(int fd, size_t maxCacheEntries); size_t doWipeCache(const DNSName& name, bool sub, QType qtype = 0xffff); bool doAgeCache(time_t now, const DNSName& name, QType qtype, uint32_t newTTL); diff --git a/pdns/recursordist/test-negcache_cc.cc b/pdns/recursordist/test-negcache_cc.cc index ae448a5b89..4bbce4abab 100644 --- a/pdns/recursordist/test-negcache_cc.cc +++ b/pdns/recursordist/test-negcache_cc.cc @@ -440,17 +440,21 @@ BOOST_AUTO_TEST_CASE(test_clear) BOOST_AUTO_TEST_CASE(test_dumpToFile) { NegCache cache(1); - vector expected; - expected.push_back("www1.powerdns.com. 600 IN TYPE0 VIA powerdns.com. ; (Indeterminate) origttl=600 ss=0\n"); - expected.push_back("powerdns.com. 600 IN SOA ns1. hostmaster. 1 2 3 4 5 ; (Indeterminate)\n"); - expected.push_back("powerdns.com. 600 IN RRSIG SOA 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n"); - expected.push_back("powerdns.com. 600 IN NSEC deadbeef. ; (Indeterminate)\n"); - expected.push_back("powerdns.com. 600 IN RRSIG NSEC 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n"); - expected.push_back("www2.powerdns.com. 600 IN TYPE0 VIA powerdns.com. ; (Indeterminate) origttl=600 ss=0\n"); - expected.push_back("powerdns.com. 600 IN SOA ns1. hostmaster. 1 2 3 4 5 ; (Indeterminate)\n"); - expected.push_back("powerdns.com. 600 IN RRSIG SOA 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n"); - expected.push_back("powerdns.com. 600 IN NSEC deadbeef. ; (Indeterminate)\n"); - expected.push_back("powerdns.com. 600 IN RRSIG NSEC 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n"); + vector expected = { + "; negcache dump follows\n", + ";\n", + "www1.powerdns.com. 600 IN TYPE0 VIA powerdns.com. ; (Indeterminate) origttl=600 ss=0\n", + "powerdns.com. 600 IN SOA ns1. hostmaster. 1 2 3 4 5 ; (Indeterminate)\n", + "powerdns.com. 600 IN RRSIG SOA 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n", + "powerdns.com. 600 IN NSEC deadbeef. ; (Indeterminate)\n", + "powerdns.com. 600 IN RRSIG NSEC 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n", + "www2.powerdns.com. 600 IN TYPE0 VIA powerdns.com. ; (Indeterminate) origttl=600 ss=0\n", + "powerdns.com. 600 IN SOA ns1. hostmaster. 1 2 3 4 5 ; (Indeterminate)\n", + "powerdns.com. 600 IN RRSIG SOA 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n", + "powerdns.com. 600 IN NSEC deadbeef. ; (Indeterminate)\n", + "powerdns.com. 600 IN RRSIG NSEC 5 3 600 20370101000000 20370101000000 24567 dummy. data ;\n", + "; negcache size: 2/0 shards: 1 min/max shard size: 2/2\n" + }; struct timeval now; Utility::gettimeofday(&now, 0); @@ -462,7 +466,7 @@ BOOST_AUTO_TEST_CASE(test_dumpToFile) if (!fp) BOOST_FAIL("Temporary file could not be opened"); - cache.dumpToFile(fp.get(), now); + cache.doDump(fileno(fp.get()), 0); rewind(fp.get()); char* line = nullptr;