dnsdist: Adjust for sampling rate in rings' consumers

Signed-off-by: Remi Gacogne <remi.gacogne@powerdns.com>

diff --git a/pdns/dnsdistdist/dnsdist-dynblocks.cc b/pdns/dnsdistdist/dnsdist-dynblocks.cc
index a9d86d9a2b4c96031669269c5cfc4b59a68f663f..b47f321025d47a42eb724b7f5b00d14eb3389688 100644 (file)
--- a/pdns/dnsdistdist/dnsdist-dynblocks.cc
+++ b/pdns/dnsdistdist/dnsdist-dynblocks.cc
@@ -35,20 +35,20 @@ void DynBlockRulesGroup::apply(const timespec& now)
     const auto& requestor = entry.first;
     const auto& counters = entry.second;
 
-    if (d_queryRateRule.warningRateExceeded(counters.queries, now)) {
+    if (d_queryRateRule.warningRateExceeded(g_rings.adjustForSamplingRate(counters.queries), now)) {
       handleWarning(blocks, now, requestor, d_queryRateRule, updated);
     }
 
-    if (d_queryRateRule.rateExceeded(counters.queries, now)) {
+    if (d_queryRateRule.rateExceeded(g_rings.adjustForSamplingRate(counters.queries), now)) {
       addBlock(blocks, now, requestor, d_queryRateRule, updated);
       continue;
     }
 
-    if (d_respRateRule.warningRateExceeded(counters.respBytes, now)) {
+    if (d_respRateRule.warningRateExceeded(g_rings.adjustForSamplingRate(counters.respBytes), now)) {
       handleWarning(blocks, now, requestor, d_respRateRule, updated);
     }
 
-    if (d_respRateRule.rateExceeded(counters.respBytes, now)) {
+    if (d_respRateRule.rateExceeded(g_rings.adjustForSamplingRate(counters.respBytes), now)) {
       addBlock(blocks, now, requestor, d_respRateRule, updated);
       continue;
     }
@@ -69,11 +69,11 @@ void DynBlockRulesGroup::apply(const timespec& now)
       const auto& typeIt = counters.d_qtypeCounts.find(qtype);
       if (typeIt != counters.d_qtypeCounts.cend()) {
 
-        if (pair.second.warningRateExceeded(typeIt->second, now)) {
+        if (pair.second.warningRateExceeded(g_rings.adjustForSamplingRate(typeIt->second), now)) {
           handleWarning(blocks, now, requestor, pair.second, updated);
         }
 
-        if (pair.second.rateExceeded(typeIt->second, now)) {
+        if (pair.second.rateExceeded(g_rings.adjustForSamplingRate(typeIt->second), now)) {
           addBlock(blocks, now, requestor, pair.second, updated);
           break;
         }
@@ -85,11 +85,11 @@ void DynBlockRulesGroup::apply(const timespec& now)
 
       const auto& rcodeIt = counters.d_rcodeCounts.find(rcode);
       if (rcodeIt != counters.d_rcodeCounts.cend()) {
-        if (pair.second.warningRateExceeded(rcodeIt->second, now)) {
+        if (pair.second.warningRateExceeded(g_rings.adjustForSamplingRate(rcodeIt->second), now)) {
           handleWarning(blocks, now, requestor, pair.second, updated);
         }
 
-        if (pair.second.rateExceeded(rcodeIt->second, now)) {
+        if (pair.second.rateExceeded(g_rings.adjustForSamplingRate(rcodeIt->second), now)) {
           addBlock(blocks, now, requestor, pair.second, updated);
           break;
         }
@@ -486,7 +486,7 @@ void DynBlockRulesGroup::processResponseRules(counts_t& counts, StatNode& root,
 
       if (suffixMatchRuleMatches) {
         const bool hit = ringEntry.isACacheHit();
-        root.submit(ringEntry.name, ((ringEntry.dh.rcode == 0 && ringEntry.usec == std::numeric_limits<unsigned int>::max()) ? -1 : ringEntry.dh.rcode), ringEntry.size, hit, std::nullopt);
+        root.submit(ringEntry.name, ((ringEntry.dh.rcode == 0 && ringEntry.usec == std::numeric_limits<uint32_t>::max()) ? -1 : ringEntry.dh.rcode), ringEntry.size, hit, std::nullopt, g_rings.getSamplingRate());
       }
     }
   }

diff --git a/pdns/dnsdistdist/dnsdist-lua-inspection-ffi.cc b/pdns/dnsdistdist/dnsdist-lua-inspection-ffi.cc
index 232d5f991923e2a47b37117d42be923e8595b560..8f58ae6b8e9c17e13725668b096a843bad38ce90 100644 (file)
--- a/pdns/dnsdistdist/dnsdist-lua-inspection-ffi.cc
+++ b/pdns/dnsdistdist/dnsdist-lua-inspection-ffi.cc
@@ -73,7 +73,7 @@ void dnsdist_ffi_stat_node_get_full_name_raw(const dnsdist_ffi_stat_node_t* node
 
 unsigned int dnsdist_ffi_stat_node_get_children_count(const dnsdist_ffi_stat_node_t* node)
 {
-  return node->node.children.size();
+  return node->node.size();
 }
 
 uint64_t dnsdist_ffi_stat_node_get_children_queries_count(const dnsdist_ffi_stat_node_t* node)

diff --git a/pdns/dnsdistdist/dnsdist-lua-inspection.cc b/pdns/dnsdistdist/dnsdist-lua-inspection.cc
index 22bf3c8c70bab0f0ac9ea6723287cad4e5644930..50f5722439d4e023fd54def4e182a3cc277afccf 100644 (file)
--- a/pdns/dnsdistdist/dnsdist-lua-inspection.cc
+++ b/pdns/dnsdistdist/dnsdist-lua-inspection.cc
@@ -145,7 +145,7 @@ static void statNodeRespRing(statvisitor_t visitor, uint64_t seconds)
       }
 
       const bool hit = entry.isACacheHit();
-      root.submit(entry.name, ((entry.dh.rcode == 0 && entry.usec == std::numeric_limits<unsigned int>::max()) ? -1 : entry.dh.rcode), entry.size, hit, std::nullopt);
+      root.submit(entry.name, ((entry.dh.rcode == 0 && entry.usec == std::numeric_limits<uint32_t>::max()) ? -1 : entry.dh.rcode), entry.size, hit, std::nullopt, g_rings.getSamplingRate());
     }
   }
 
@@ -245,7 +245,7 @@ static counts_t exceedRCode(unsigned int rate, int seconds, int rcode)
 {
   return exceedRespGen(rate, seconds, [rcode](counts_t& counts, const Rings::Response& resp) {
     if (resp.dh.rcode == rcode) {
-      counts[resp.requestor]++;
+      counts[resp.requestor] += g_rings.adjustForSamplingRate(1U);
     }
   });
 }
@@ -253,7 +253,7 @@ static counts_t exceedRCode(unsigned int rate, int seconds, int rcode)
 static counts_t exceedRespByterate(unsigned int rate, int seconds)
 {
   return exceedRespGen(rate, seconds, [](counts_t& counts, const Rings::Response& resp) {
-    counts[resp.requestor] += resp.size;
+    counts[resp.requestor] += g_rings.adjustForSamplingRate(resp.size);
   });
 }
 
@@ -883,7 +883,7 @@ void setupLuaInspection(LuaContext& luaCtx)
     setLuaNoSideEffect();
     return exceedQueryGen(rate, seconds, [type](counts_t& counts, const Rings::Query& query) {
       if (query.qtype == type) {
-        counts[query.requestor]++;
+        counts[query.requestor] += g_rings.adjustForSamplingRate(1U);
       }
     });
   });
@@ -891,7 +891,7 @@ void setupLuaInspection(LuaContext& luaCtx)
   luaCtx.writeFunction("exceedQRate", [](unsigned int rate, int seconds) {
     setLuaNoSideEffect();
     return exceedQueryGen(rate, seconds, [](counts_t& counts, const Rings::Query& query) {
-      counts[query.requestor]++;
+      counts[query.requestor] += g_rings.adjustForSamplingRate(1U);
     });
   });
 
@@ -900,7 +900,7 @@ void setupLuaInspection(LuaContext& luaCtx)
   /* StatNode */
   luaCtx.registerFunction<unsigned int (StatNode::*)() const>("numChildren",
                                                               [](const StatNode& node) -> unsigned int {
-                                                                return node.children.size();
+                                                                return node.size();
                                                               });
   luaCtx.registerMember("fullname", &StatNode::fullname);
   luaCtx.registerMember("labelsCount", &StatNode::labelsCount);

diff --git a/pdns/dnsdistdist/dnsdist-rings.cc b/pdns/dnsdistdist/dnsdist-rings.cc
index 5dc98d7a9e846b781b5d9b41c98ef874a9f08965..e1d9b607e17017dc9faee8abf29a732ad9227895 100644 (file)
--- a/pdns/dnsdistdist/dnsdist-rings.cc
+++ b/pdns/dnsdistdist/dnsdist-rings.cc
@@ -210,3 +210,12 @@ bool Rings::shouldSkipDueToSampling()
   auto counter = d_samplingCounter++;
   return (counter % d_samplingRate) == 0;
 }
+
+uint32_t Rings::adjustForSamplingRate(uint32_t count) const
+{
+  const auto samplingRate = getSamplingRate();
+  if (samplingRate > 0) {
+    return count * samplingRate;
+  }
+  return count;
+}

diff --git a/pdns/dnsdistdist/dnsdist-rings.hh b/pdns/dnsdistdist/dnsdist-rings.hh
index 7cdee74f599f95bba75957cba7d65e02a8178734..03378941fdffeb28a1efb9ee3e4b64d926d225f2 100644 (file)
--- a/pdns/dnsdistdist/dnsdist-rings.hh
+++ b/pdns/dnsdistdist/dnsdist-rings.hh
@@ -242,6 +242,8 @@ struct Rings
     return d_samplingRate;
   }
 
+  uint32_t adjustForSamplingRate(uint32_t count) const;
+
   std::vector<std::unique_ptr<Shard>> d_shards;
   pdns::stat_t d_blockingQueryInserts{0};
   pdns::stat_t d_blockingResponseInserts{0};

diff --git a/pdns/dnsscope.cc b/pdns/dnsscope.cc
index 3884f81ad02baca44661a06283fb50543f5075b2..74876fe26905a2a37f24842e6bf353e18ba644e7 100644 (file)
--- a/pdns/dnsscope.cc
+++ b/pdns/dnsscope.cc
@@ -114,11 +114,11 @@ static void visitor(const StatNode* node, const StatNode::Stat& /* selfstat */,
 {
   // 20% servfails, >100 children, on average less than 2 copies of a query
   // >100 different subqueries
-  double dups=1.0*childstat.queries/node->children.size();
+  double dups=1.0*childstat.queries/node->size();
   if(dups > 2.0)
     return;
-  if(1.0*childstat.servfails / childstat.queries > 0.2 && node->children.size()>100) {
-    cout<<node->fullname<<", servfails: "<<childstat.servfails<<", nxdomains: "<<childstat.nxdomains<<", remotes: "<<childstat.remotes.size()<<", children: "<<node->children.size()<<", childstat.queries: "<<childstat.queries;
+  if(1.0*childstat.servfails / childstat.queries > 0.2 && node->size()>100) {
+    cout<<node->fullname<<", servfails: "<<childstat.servfails<<", nxdomains: "<<childstat.nxdomains<<", remotes: "<<childstat.remotes.size()<<", children: "<<node->size()<<", childstat.queries: "<<childstat.queries;
     cout<<", dups2: "<<dups<<endl;
     for(const StatNode::Stat::remotes_t::value_type& rem :  childstat.remotes) {
       cout<<"source: "<<node->fullname<<"\t"<<rem.first.toString()<<"\t"<<rem.second<<endl;
@@ -389,8 +389,9 @@ try
             ComboAddress rem = pr.getDest();
             rem.sin4.sin_port=0;
 
-            if(doServFailTree)
-              root.submit(qname, header.rcode, pr.d_len, false, rem);
+            if (doServFailTree) {
+              root.submit(qname, header.rcode, pr.d_len, false, rem, 0U);
+            }
           }
 
           if(!qd.d_qcount || qd.d_qcount == qd.d_answercount) {

diff --git a/pdns/statnode.cc b/pdns/statnode.cc
index 253d064d34f6e73406a85297513f1e2997723d8f..801419a6cae1050181c6b45441c1ef1d723ddcf4 100644 (file)
--- a/pdns/statnode.cc
+++ b/pdns/statnode.cc
@@ -48,7 +48,7 @@ void StatNode::visit(const visitor_t& visitor, Stat& newstat, unsigned int depth
   newstat += childstat;
 }
 
-void StatNode::submit(const DNSName& domain, int rcode, unsigned int bytes, bool hit, const std::optional<ComboAddress>& remote)
+void StatNode::submit(const DNSName& domain, int rcode, unsigned int bytes, bool hit, const std::optional<ComboAddress>& remote, size_t samplingRate)
 {
   //  cerr<<"FIRST submit called on '"<<domain<<"'"<<endl;
   std::vector<string> tmp = domain.getRawLabels();
@@ -57,17 +57,26 @@ void StatNode::submit(const DNSName& domain, int rcode, unsigned int bytes, bool
   }
 
   auto last = tmp.end() - 1;
-  children[*last].submit(last, tmp.begin(), "", rcode, bytes, remote, 1, hit);
+  children[*last].submit(last, tmp.begin(), "", rcode, bytes, remote, 1, hit, samplingRate);
 }
 
-/* www.powerdns.com. -> 
+static uint64_t adjustForSampling(uint32_t count, size_t samplingRate)
+{
+  if (samplingRate > 0) {
+    return count * samplingRate;
+  }
+  return count;
+}
+
+
+/* www.powerdns.com. ->
    .                 <- fullnames
    com.
    powerdns.com
    www.powerdns.com. 
 */
 
-void StatNode::submit(std::vector<string>::const_iterator end, std::vector<string>::const_iterator begin, const std::string& domain, int rcode, unsigned int bytes, const std::optional<ComboAddress>& remote, unsigned int count, bool hit)
+void StatNode::submit(std::vector<string>::const_iterator end, std::vector<string>::const_iterator begin, const std::string& domain, int rcode, unsigned int bytes, const std::optional<ComboAddress>& remote, unsigned int count, bool hit, size_t samplingRate)
 {
   //  cerr<<"Submit called for domain='"<<domain<<"': ";
   //  for(const std::string& n :  labels) 
@@ -94,19 +103,19 @@ void StatNode::submit(std::vector<string>::const_iterator end, std::vector<strin
       labelsCount = count;
     }
     //    cerr<<"Hit the end, set our fullname to '"<<fullname<<"'"<<endl<<endl;
-    s.queries++;
-    s.bytes += bytes;
+    s.queries += adjustForSampling(1U, samplingRate);
+    s.bytes += adjustForSampling(bytes, samplingRate);
     if (rcode < 0) {
-      s.drops++;
+      s.drops += adjustForSampling(1U, samplingRate);
     }
     else if (rcode == RCode::NoError) {
-      s.noerrors++;
+      s.noerrors += adjustForSampling(1U, samplingRate);
     }
     else if (rcode == RCode::ServFail) {
-      s.servfails++;
+      s.servfails += adjustForSampling(1U, samplingRate);
     }
     else if (rcode == RCode::NXDomain) {
-      s.nxdomains++;
+      s.nxdomains += adjustForSampling(1U, samplingRate);
     }
 
     if (remote) {
@@ -114,7 +123,7 @@ void StatNode::submit(std::vector<string>::const_iterator end, std::vector<strin
     }
 
     if (hit) {
-      ++s.hits;
+      s.hits += adjustForSampling(1U, samplingRate);
     }
   }
   else {
@@ -130,6 +139,6 @@ void StatNode::submit(std::vector<string>::const_iterator end, std::vector<strin
     }
     //    cerr<<"Not yet end, set our fullname to '"<<fullname<<"', recursing"<<endl;
     --end;
-    children[*end].submit(end, begin, fullname, rcode, bytes, remote, count+1, hit);
+    children[*end].submit(end, begin, fullname, rcode, bytes, remote, count+1, hit, samplingRate);
   }
 }

diff --git a/pdns/statnode.hh b/pdns/statnode.hh
index 8b663f5b98459f824a3bc107d8af347aaf449547..6635c4953197596d121f5ebb547c4d68238523f7 100644 (file)
--- a/pdns/statnode.hh
+++ b/pdns/statnode.hh
@@ -58,22 +58,27 @@ public:
   };
 
   using visitor_t = std::function<void(const StatNode*, const Stat& selfstat, const Stat& childstat)>;
-  using children_t = std::map<std::string, StatNode, CIStringCompare>;
 
   Stat s;
   std::string name;
   std::string fullname;
   uint8_t labelsCount{0};
 
-  void submit(const DNSName& domain, int rcode, unsigned int bytes, bool hit, const std::optional<ComboAddress>& remote);
+  void submit(const DNSName& domain, int rcode, uint32_t bytes, bool hit, const std::optional<ComboAddress>& remote, size_t samplingRate);
   Stat print(unsigned int depth=0, Stat newstat=Stat(), bool silent=false) const;
   void visit(const visitor_t& visitor, Stat& newstat, unsigned int depth = 0) const;
   bool empty() const
   {
     return children.empty() && s.remotes.empty();
   }
-  children_t children;
+  size_t size() const
+  {
+    return children.size();
+  }
 
 private:
-  void submit(std::vector<string>::const_iterator end, std::vector<string>::const_iterator begin, const std::string& domain, int rcode, unsigned int bytes, const std::optional<ComboAddress>& remote, unsigned int count, bool hit);
+  void submit(std::vector<string>::const_iterator end, std::vector<string>::const_iterator begin, const std::string& domain, int rcode, uint32_t bytes, const std::optional<ComboAddress>& remote, unsigned int count, bool hit, size_t samplingRate);
+
+  using children_t = std::map<std::string, StatNode, CIStringCompare>;
+  children_t children;
 };