Keep track of what destinations are causing the most timeouts

diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc
index c7e1bb3db80ef4e57329daaf07d38304b0784d9a..63f5c40c93ae13522308ba0590ad2e117b706708 100644 (file)
--- a/pdns/pdns_recursor.cc
+++ b/pdns/pdns_recursor.cc
@@ -116,7 +116,7 @@ thread_local std::unique_ptr<MT_t> MT; // the big MTasker
 thread_local std::unique_ptr<MemRecursorCache> t_RC;
 thread_local std::unique_ptr<RecursorPacketCache> t_packetCache;
 thread_local FDMultiplexer* t_fdm{nullptr};
-thread_local std::unique_ptr<addrringbuf_t> t_remotes, t_servfailremotes, t_largeanswerremotes, t_bogusremotes;
+thread_local std::unique_ptr<addrringbuf_t> t_remotes, t_servfailremotes, t_largeanswerremotes, t_bogusremotes, t_timeouts;
 thread_local std::unique_ptr<boost::circular_buffer<pair<DNSName, uint16_t> > > t_queryring, t_servfailqueryring, t_bogusqueryring;
 thread_local std::shared_ptr<NetmaskGroup> t_allowFrom;
 #ifdef HAVE_PROTOBUF
@@ -3755,6 +3755,8 @@ try
     t_bogusremotes->set_capacity(ringsize);
     t_largeanswerremotes = std::unique_ptr<addrringbuf_t>(new addrringbuf_t());
     t_largeanswerremotes->set_capacity(ringsize);
+    t_timeouts = std::unique_ptr<addrringbuf_t>(new addrringbuf_t());
+    t_timeouts->set_capacity(ringsize);
 
     t_queryring = std::unique_ptr<boost::circular_buffer<pair<DNSName, uint16_t> > >(new boost::circular_buffer<pair<DNSName, uint16_t> >());
     t_queryring->set_capacity(ringsize);

diff --git a/pdns/rec_channel.hh b/pdns/rec_channel.hh
index 526faa6a3f93169a33fc361a542a25953d24b9d9..1b8e4d8f308a0a57411887bb4bbd96a6076a0dac 100644 (file)
--- a/pdns/rec_channel.hh
+++ b/pdns/rec_channel.hh
@@ -73,6 +73,7 @@ std::vector<ComboAddress>* pleaseGetRemotes();
 std::vector<ComboAddress>* pleaseGetServfailRemotes();
 std::vector<ComboAddress>* pleaseGetBogusRemotes();
 std::vector<ComboAddress>* pleaseGetLargeAnswerRemotes();
+std::vector<ComboAddress>* pleaseGetTimeouts();
 DNSName getRegisteredName(const DNSName& dom);
 std::atomic<unsigned long>* getDynMetric(const std::string& str);
 optional<uint64_t> getStatByName(const std::string& name);

diff --git a/pdns/rec_channel_rec.cc b/pdns/rec_channel_rec.cc
index ad3cfdfdf0d99d1242b5c7ea3da5f4b6462a7ac0..ef69e736c7ce69570dcdc4dbc489966a557f471b 100644 (file)
--- a/pdns/rec_channel_rec.cc
+++ b/pdns/rec_channel_rec.cc
@@ -1100,6 +1100,18 @@ vector<ComboAddress>* pleaseGetLargeAnswerRemotes()
   return ret;
 }
 
+vector<ComboAddress>* pleaseGetTimeouts()
+{
+  vector<ComboAddress>* ret = new vector<ComboAddress>();
+  if(!t_timeouts)
+    return ret;
+  ret->reserve(t_timeouts->size());
+  for(const ComboAddress& ca :  *t_timeouts) {
+    ret->push_back(ca);
+  }
+  return ret;
+}
+
 string doGenericTopRemotes(pleaseremotefunc_t func)
 {
   typedef map<ComboAddress, int, ComboAddress::addressOnlyLessThan> counts_t;
@@ -1275,6 +1287,7 @@ string RecursorControlParser::getAnswer(const string& question, RecursorControlP
 "top-queries                      show top queries\n"
 "top-pub-queries                  show top queries grouped by public suffix list\n"
 "top-remotes                      show top remotes\n"
+"top-timeouts                     show top downstream timeouts"
 "top-servfail-queries             show top queries receiving servfail answers\n"
 "top-bogus-queries                show top queries validating as bogus\n"
 "top-pub-servfail-queries         show top queries receiving servfail answers grouped by public suffix list\n"
@@ -1412,6 +1425,9 @@ string RecursorControlParser::getAnswer(const string& question, RecursorControlP
   if(cmd=="top-largeanswer-remotes")
     return doGenericTopRemotes(pleaseGetLargeAnswerRemotes);
 
+  if(cmd=="top-timeouts")
+    return doGenericTopRemotes(pleaseGetTimeouts);
+
 
   if(cmd=="current-queries")
     return doCurrentQueries();

diff --git a/pdns/recursordist/docs/manpages/rec_control.1.rst b/pdns/recursordist/docs/manpages/rec_control.1.rst
index cc4b1cd7ae5605e93a18b81d9c44525c2c779344..8739a9b1ca846a99cb34fef076f42c76b478093b 100644 (file)
--- a/pdns/recursordist/docs/manpages/rec_control.1.rst
+++ b/pdns/recursordist/docs/manpages/rec_control.1.rst
@@ -237,6 +237,10 @@ top-bogus-remotes
     Shows the top-20 most active remote hosts causing bogus responses.
     Statistics are over the last 'stats-ringbuffer-entries' queries.
 
+top-timeouts
+    Shows the top-20 most active downstream timeout destinations.
+    Statistics are over the last 'stats-ringbuffer-entries' queries.
+
 trace-regex *REGEX*
     Emit resolution trace for matching queries. Empty regex to disable trace.
 

diff --git a/pdns/recursordist/html/index.html b/pdns/recursordist/html/index.html
index 9ae0f622bd6f9f49ebf1fd2a151cd64c964b5a77..c913094e46729ce129a3f189aff6c694bef06437 100644 (file)
--- a/pdns/recursordist/html/index.html
+++ b/pdns/recursordist/html/index.html
@@ -66,6 +66,7 @@
             <div class="stats-table" id="remotering"></div>
             <div class="stats-table" id="servfailremotering"></div>
             <div class="stats-table" id="bogusremotering"></div>
+            <div class="stats-table" id="timeouts"></div>
         </div>
     </div>
 
@@ -178,5 +179,20 @@
     </table>
 </script>
 
+<script id="timeouts-template" type="text/x-handlebars-template">
+    <table class="two-col">
+        <tr>
+            <th>Number</th>
+            <th>Downstream timeouts</th>
+        </tr>
+        {{#each rows}}
+        <tr>
+            <td>{{0}}</td>
+            <td>{{1}}</td>
+        </tr>
+        {{/each}}
+    </table>
+</script>
+
 </body>
 </html>

diff --git a/pdns/recursordist/html/local.js b/pdns/recursordist/html/local.js
index c11b0b23efd1d730ade4ac96c4a228486fa83625..9e6fc03190c29b6e6cc8712f30e59f174a34cc0d 100644 (file)
--- a/pdns/recursordist/html/local.js
+++ b/pdns/recursordist/html/local.js
@@ -137,6 +137,11 @@ $(document).ready(function () {
                 var rows = makeRingRows(data);
                 render('bogusremotering', {rows: rows});
             });
+        $.getJSON('jsonstat', jsonstatParams('get-remote-ring', 'timeouts', false),
+            function (data) {
+                var rows = makeRingRows(data);
+                render('timeouts', {rows: rows});
+            });
     }
 
     var connectionOK = function (ok, o) {

diff --git a/pdns/syncres.cc b/pdns/syncres.cc
index fd2d0dfab2c0ad7380e1e99f1d1c47a1e6f76ae3..dbe84a1b9025368f199498a76ab16a2551314f65 100644 (file)
--- a/pdns/syncres.cc
+++ b/pdns/syncres.cc
@@ -2484,6 +2484,8 @@ bool SyncRes::doResolveAtThisIP(const std::string& prefix, const DNSName& qname,
       else {
         // timeout
         t_sstorage.throttle.throttle(d_now.tv_sec, boost::make_tuple(remoteIP, qname, qtype.getCode()), 10, 5);
+        if(t_timeouts)
+          t_timeouts->push_back(remoteIP);
       }
     }
 

diff --git a/pdns/syncres.hh b/pdns/syncres.hh
index 3fc515d79c66a809fd60698f3111ecec7cd5583b..51f3bede6aeebdd61323262cfe65ddddbdb62ec7 100644 (file)
--- a/pdns/syncres.hh
+++ b/pdns/syncres.hh
@@ -963,7 +963,7 @@ public:
 };
 
 typedef boost::circular_buffer<ComboAddress> addrringbuf_t;
-extern thread_local std::unique_ptr<addrringbuf_t> t_servfailremotes, t_largeanswerremotes, t_remotes, t_bogusremotes;
+extern thread_local std::unique_ptr<addrringbuf_t> t_servfailremotes, t_largeanswerremotes, t_remotes, t_bogusremotes, t_timeouts;
 
 extern thread_local std::unique_ptr<boost::circular_buffer<pair<DNSName,uint16_t> > > t_queryring, t_servfailqueryring, t_bogusqueryring;
 extern thread_local std::shared_ptr<NetmaskGroup> t_allowFrom;

diff --git a/pdns/ws-recursor.cc b/pdns/ws-recursor.cc
index e3a5e8024309c9cd51f526512f2a8b8d574ea22e..1b3a8c990840b1befdc9ba1f6dcb6b25868618cd 100644 (file)
--- a/pdns/ws-recursor.cc
+++ b/pdns/ws-recursor.cc
@@ -540,6 +540,8 @@ void RecursorWebServer::jsonstat(HttpRequest* req, HttpResponse *resp)
       queries=broadcastAccFunction<vector<ComboAddress> >(pleaseGetBogusRemotes);
     else if(req->getvars["name"]=="large-answer-remotes")
       queries=broadcastAccFunction<vector<ComboAddress> >(pleaseGetLargeAnswerRemotes);
+    else if(req->getvars["name"]=="timeouts")
+      queries=broadcastAccFunction<vector<ComboAddress> >(pleaseGetTimeouts);
 
     typedef map<ComboAddress,unsigned int,ComboAddress::addressOnlyLessThan> counts_t;
     counts_t counts;