pdns/dnsscope.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #define __FAVOR_BSD
  23 #ifdef HAVE_CONFIG_H
  24 #include "config.h"
  25 #endif
  26 #if HAVE_BOOST_GE_148
  27 #include "histog.hh"
  28 #endif
  29
  30 #include "statbag.hh"
  31 #include "dnspcap.hh"
  32 #include "dnsparser.hh"
  33 #include "dnsname.hh"
  34 #include <boost/tuple/tuple.hpp>
  35 #include <boost/tuple/tuple_comparison.hpp>
  36 #include <map>
  37 #include <set>
  38 #include <fstream>
  39 #include <algorithm>
  40 #include "anadns.hh"
  41 #include <boost/program_options.hpp>
  42 #include <unordered_set>
  43 #include <boost/logic/tribool.hpp>
  44 #include "arguments.hh"
  45 #include "namespaces.hh"
  46 #include "dnsrecords.hh"
  47 #include "statnode.hh"
  48
  49 namespace po = boost::program_options;
  50 po::variables_map g_vm;
  51
  52 ArgvMap& arg()
  53 {
  54   static ArgvMap theArg;
  55   return theArg;
  56 }
  57 StatBag S;
  58
  59
  60
  61 struct QuestionData
  62 {
  63   QuestionData() : d_qcount(0), d_answercount(0)
  64   {
  65     d_firstquestiontime.tv_sec=0;
  66   }
  67
  68   int d_qcount;
  69   int d_answercount;
  70
  71   struct pdns_timeval d_firstquestiontime;
  72 };
  73
  74 typedef map<QuestionIdentifier, QuestionData> statmap_t;
  75 statmap_t statmap;
  76
  77 unsigned int liveQuestions()
  78 {
  79   unsigned int ret=0;
  80   for(statmap_t::value_type& val :  statmap) {
  81     if(!val.second.d_answercount)
  82       ret++;
  83     //    if(val.second.d_qcount > val.second.d_answercount)
  84     //      ret+= val.second.d_qcount - val.second.d_answercount;
  85   }
  86   return ret;
  87 }
  88
  89 struct LiveCounts
  90 {
  91   unsigned int questions;
  92   unsigned int answers;
  93   unsigned int outstanding;
  94
  95   LiveCounts()
  96   {
  97     questions=answers=outstanding=0;
  98   }
  99
 100   LiveCounts operator-(const LiveCounts& rhs)
 101   {
 102     LiveCounts ret;
 103     ret.questions = questions - rhs.questions;
 104     ret.answers = answers - rhs.answers;
 105     ret.outstanding = outstanding;
 106     return ret;
 107   }
 108 };
 109
 110 void visitor(const StatNode* node, const StatNode::Stat& selfstat, const StatNode::Stat& childstat)
 111 {
 112   // 20% servfails, >100 children, on average less than 2 copies of a query
 113   // >100 different subqueries
 114   double dups=1.0*childstat.queries/node->children.size();
 115   if(dups > 2.0)
 116     return;
 117   if(1.0*childstat.servfails / childstat.queries > 0.2 && node->children.size()>100) {
 118     cout<<node->fullname<<", servfails: "<<childstat.servfails<<", nxdomains: "<<childstat.nxdomains<<", remotes: "<<childstat.remotes.size()<<", children: "<<node->children.size()<<", childstat.queries: "<<childstat.queries;
 119     cout<<", dups2: "<<dups<<endl;
 120     for(const StatNode::Stat::remotes_t::value_type& rem :  childstat.remotes) {
 121       cout<<"source: "<<node->fullname<<"\t"<<rem.first.toString()<<"\t"<<rem.second<<endl;
 122     }
 123   }
 124 }
 125
 126 const struct timeval operator-(const struct pdns_timeval& lhs, const struct pdns_timeval& rhs)
 127 {
 128   struct timeval a{lhs.tv_sec, static_cast<suseconds_t>(lhs.tv_usec)}, b{rhs.tv_sec, static_cast<suseconds_t>(rhs.tv_usec)};
 129   return operator-(a,b);
 130 }
 131
 132
 133 int main(int argc, char** argv)
 134 try
 135 {
 136   po::options_description desc("Allowed options"), hidden, alloptions;
 137   desc.add_options()
 138     ("help,h", "produce help message")
 139     ("version", "print version number")
 140     ("rd", po::value<bool>(), "If set to true, only process RD packets, to false only non-RD, unset: both")
 141     ("ipv4", po::value<bool>()->default_value(true), "Process IPv4 packets")
 142     ("ipv6", po::value<bool>()->default_value(true), "Process IPv6 packets")
 143 #if HAVE_BOOST_GE_148
 144     ("log-histogram", "Write a log-histogram to file 'log-histogram'")
 145     ("full-histogram", po::value<double>(), "Write a log-histogram to file 'full-histogram' with this millisecond bin size")
 146 #endif
 147     ("filter-name,f", po::value<string>(), "Do statistics only for queries within this domain")
 148     ("load-stats,l", po::value<string>()->default_value(""), "if set, emit per-second load statistics (questions, answers, outstanding)")
 149     ("no-servfail-stats", "Don't include servfails in response time stats")
 150     ("servfail-tree", "Figure out subtrees that generate servfails")
 151     ("stats-dir", po::value<string>()->default_value("."), "Directory where statistics will be saved")
 152     ("write-failures,w", po::value<string>()->default_value(""), "if set, write weird packets to this PCAP file")
 153     ("verbose,v", "be verbose");
 154
 155   hidden.add_options()
 156     ("files", po::value<vector<string> >(), "files");
 157
 158   alloptions.add(desc).add(hidden);
 159
 160   po::positional_options_description p;
 161   p.add("files", -1);
 162
 163   po::store(po::command_line_parser(argc, argv).options(alloptions).positional(p).run(), g_vm);
 164   po::notify(g_vm);
 165
 166   vector<string> files;
 167   if(g_vm.count("files"))
 168     files = g_vm["files"].as<vector<string> >();
 169
 170   if(g_vm.count("version")) {
 171     cerr<<"dnsscope "<<VERSION<<endl;
 172     exit(0);
 173   }
 174
 175   if(files.empty() || g_vm.count("help")) {
 176     cerr<<"Syntax: dnsscope filename.pcap [filenam2.pcap...]"<<endl;
 177     cout << desc << endl;
 178     exit(0);
 179   }
 180
 181   DNSName filtername;
 182   if(g_vm.count("filter-name"))
 183     filtername = DNSName(g_vm["filter-name"].as<string>());
 184   uint32_t nameMismatch = 0;
 185
 186   StatNode root;
 187
 188   bool verbose = g_vm.count("verbose");
 189
 190   bool haveRDFilter=0, rdFilter=0;
 191   if(g_vm.count("rd")) {
 192     rdFilter = g_vm["rd"].as<bool>();
 193     haveRDFilter=1;
 194     cout<<"Filtering on recursion desired="<<rdFilter<<endl;
 195   }
 196   else
 197     cout<<"Warning, looking at both RD and non-RD traffic!"<<endl;
 198
 199   bool doIPv4 = g_vm["ipv4"].as<bool>();
 200   bool doIPv6 = g_vm["ipv6"].as<bool>();
 201   bool doServFailTree = g_vm.count("servfail-tree");
 202   bool noservfailstats = g_vm.count("no-servfail-stats");
 203   int dnserrors=0, parsefail=0;
 204   typedef map<uint32_t,uint32_t> cumul_t;
 205   cumul_t cumul;
 206   unsigned int untracked=0, errorresult=0, nonRDQueries=0, queries=0;
 207   unsigned int ipv4DNSPackets=0, ipv6DNSPackets=0, fragmented=0, rdNonRAAnswers=0;
 208   unsigned int answers=0, nonDNSIP=0, rdFilterMismatch=0;
 209   unsigned int dnssecOK=0, edns=0;
 210   unsigned int dnssecCD=0, dnssecAD=0;
 211   unsigned int reuses=0;
 212   typedef map<uint16_t,uint32_t> rcodes_t;
 213   rcodes_t rcodes;
 214
 215   time_t lowestTime=2000000000, highestTime=0;
 216   time_t lastsec=0;
 217   LiveCounts lastcounts;
 218   std::unordered_set<ComboAddress, ComboAddress::addressOnlyHash> requestors, recipients, rdnonra;
 219   typedef vector<pair<time_t, LiveCounts> > pcounts_t;
 220   pcounts_t pcounts;
 221   OPTRecordContent::report();
 222
 223   for(unsigned int fno=0; fno < files.size(); ++fno) {
 224     PcapPacketReader pr(files[fno]);
 225     PcapPacketWriter* pw=0;
 226     if(!g_vm["write-failures"].as<string>().empty())
 227       pw=new PcapPacketWriter(g_vm["write-failures"].as<string>(), pr);
 228
 229     EDNSOpts edo;
 230     while(pr.getUDPPacket()) {
 231
 232       if((ntohs(pr.d_udp->uh_dport)==5300 || ntohs(pr.d_udp->uh_sport)==5300 ||
 233           ntohs(pr.d_udp->uh_dport)==53   || ntohs(pr.d_udp->uh_sport)==53) &&
 234          pr.d_len > 12) {
 235         try {
 236           if((pr.d_ip->ip_v == 4 && !doIPv4) || (pr.d_ip->ip_v == 6 && !doIPv6))
 237             continue;
 238           if(pr.d_ip->ip_v == 4) {
 239             uint16_t frag = ntohs(pr.d_ip->ip_off);
 240             if((frag & IP_MF) || (frag & IP_OFFMASK)) { // more fragments or IS a fragment
 241               fragmented++;
 242               continue;
 243             }
 244           }
 245           uint16_t qtype;
 246           DNSName qname((const char*)pr.d_payload, pr.d_len, 12, false, &qtype);
 247           struct dnsheader header;
 248           memcpy(&header, (struct dnsheader*)pr.d_payload, 12);
 249
 250           if(haveRDFilter && header.rd != rdFilter) {
 251             rdFilterMismatch++;
 252             continue;
 253           }
 254
 255           if(!filtername.empty() && !qname.isPartOf(filtername)) {
 256             nameMismatch++;
 257             continue;
 258           }
 259
 260           if(!header.qr) {
 261             uint16_t udpsize, z;
 262             if(getEDNSUDPPayloadSizeAndZ((const char*)pr.d_payload, pr.d_len, &udpsize, &z)) {
 263               edns++;
 264               if(z & EDNSOpts::DNSSECOK)
 265                 dnssecOK++;
 266               if(header.cd)
 267                 dnssecCD++;
 268               if(header.ad)
 269                 dnssecAD++;
 270             }
 271           }
 272
 273           if(pr.d_ip->ip_v == 4)
 274             ++ipv4DNSPackets;
 275           else
 276             ++ipv6DNSPackets;
 277
 278           if(pr.d_pheader.ts.tv_sec != lastsec) {
 279             LiveCounts lc;
 280             if(lastsec) {
 281               lc.questions = queries;
 282               lc.answers = answers;
 283               lc.outstanding = liveQuestions();
 284
 285               LiveCounts diff = lc - lastcounts;
 286               pcounts.push_back(make_pair(pr.d_pheader.ts.tv_sec, diff));
 287
 288             }
 289             lastsec = pr.d_pheader.ts.tv_sec;
 290             lastcounts = lc;
 291           }
 292
 293           lowestTime=min((time_t)lowestTime,  (time_t)pr.d_pheader.ts.tv_sec);
 294           highestTime=max((time_t)highestTime, (time_t)pr.d_pheader.ts.tv_sec);
 295
 296           QuestionIdentifier qi=QuestionIdentifier::create(pr.getSource(), pr.getDest(), header, qname, qtype);
 297
 298           if(!header.qr) { // question
 299             //      cout<<"Query "<<qi<<endl;
 300             if(!header.rd)
 301               nonRDQueries++;
 302             queries++;
 303
 304             ComboAddress rem = pr.getSource();
 305             rem.sin4.sin_port=0;
 306             requestors.insert(rem);
 307
 308             QuestionData& qd=statmap[qi];
 309
 310             if(!qd.d_firstquestiontime.tv_sec)
 311               qd.d_firstquestiontime=pr.d_pheader.ts;
 312             else {
 313               auto delta=makeFloat(pr.d_pheader.ts - qd.d_firstquestiontime);
 314               //              cout<<"Reuse of "<<qi<<", delta t="<<delta<<", count="<<qd.d_qcount<<endl;
 315               if(delta > 2.0) {
 316                 //              cout<<"Resetting old entry for "<<qi<<", too old"<<endl;
 317                 qd.d_qcount=0;
 318                 qd.d_answercount=0;
 319                 qd.d_firstquestiontime=pr.d_pheader.ts;
 320               }
 321             }
 322             if(qd.d_qcount++)
 323               reuses++;
 324           }
 325           else  {  // answer
 326             //      cout<<"Response "<<qi<<endl;
 327             rcodes[header.rcode]++;
 328             answers++;
 329             if(header.rd && !header.ra) {
 330               rdNonRAAnswers++;
 331               rdnonra.insert(pr.getDest());
 332             }
 333
 334             if(header.ra) {
 335               ComboAddress rem = pr.getDest();
 336               rem.sin4.sin_port=0;
 337               recipients.insert(rem);
 338             }
 339
 340             QuestionData& qd=statmap[qi];
 341             if(!qd.d_qcount) {
 342               //              cout<<"Untracked answer: "<<qi<<endl;
 343               untracked++;
 344             }
 345
 346             qd.d_answercount++;
 347
 348             if(qd.d_qcount) {
 349               uint32_t usecs= (pr.d_pheader.ts.tv_sec - qd.d_firstquestiontime.tv_sec) * 1000000 +
 350                 (pr.d_pheader.ts.tv_usec - qd.d_firstquestiontime.tv_usec) ;
 351
 352               //              cout<<"Usecs for "<<qi<<": "<<usecs<<endl;
 353               if(!noservfailstats || header.rcode != 2)
 354                 cumul[usecs]++;
 355
 356               if(header.rcode != 0 && header.rcode!=3)
 357                 errorresult++;
 358               ComboAddress rem = pr.getDest();
 359               rem.sin4.sin_port=0;
 360
 361               if(doServFailTree)
 362                 root.submit(qname, header.rcode, rem);
 363             }
 364
 365             if(!qd.d_qcount || qd.d_qcount == qd.d_answercount) {
 366               //              cout<<"Clearing state for "<<qi<<endl<<endl;
 367               statmap.erase(qi);
 368             }
 369             else {
 370               //              cout<<"State for qi remains open, qcount="<<qd.d_qcount<<", answercount="<<qd.d_answercount<<endl;
 371             }
 372           }
 373         }
 374         catch(std::exception& e) {
 375           if(verbose)
 376             cout<<"error parsing packet: "<<e.what()<<endl;
 377
 378           if(pw)
 379             pw->write();
 380           parsefail++;
 381           continue;
 382         }
 383       }
 384       else { // non-DNS ip
 385         nonDNSIP++;
 386       }
 387     }
 388     cout<<"PCAP contained "<<pr.d_correctpackets<<" correct packets, "<<pr.d_runts<<" runts, "<< pr.d_oversized<<" oversize, "<<pr.d_nonetheripudp<<" non-UDP.\n";
 389
 390   }
 391
 392   /*
 393   cout<<"Open when done: "<<endl;
 394   for(const auto& a : statmap) {
 395     cout<<a.first<<": qcount="<<a.second.d_qcount<<", answercount="<<a.second.d_answercount<<endl;
 396   }
 397   */
 398
 399   cout<<"Timespan: "<<(highestTime-lowestTime)/3600.0<<" hours"<<endl;
 400
 401   cout<<nonDNSIP<<" non-DNS UDP, "<<dnserrors<<" dns decoding errors, "<<parsefail<<" packets failed to parse"<<endl;
 402   cout<<"Ignored fragment packets: "<<fragmented<<endl;
 403   cout<<"Dropped DNS packets based on recursion-desired filter: "<<rdFilterMismatch<<endl;
 404   if(!filtername.empty())
 405     cout <<"Dropped DNS packets because not part of '"<<filtername<<"': "<<nameMismatch << endl;
 406   cout<<"DNS IPv4: "<<ipv4DNSPackets<<" packets, IPv6: "<<ipv6DNSPackets<<" packets"<<endl;
 407   cout<<"Questions: "<<queries<<", answers: "<<answers<<endl;
 408   cout<<"Reuses of same state entry: "<<reuses<<endl;
 409   unsigned int unanswered=0;
 410
 411
 412   //  ofstream openf("openf");
 413   for(statmap_t::const_iterator i=statmap.begin(); i!=statmap.end(); ++i) {
 414     if(!i->second.d_answercount) {
 415       unanswered++;
 416     }
 417     //openf<< i->first.d_source.toStringWithPort()<<' ' <<i->first.d_dest.toStringWithPort()<<' '<<i->first.d_id<<' '<<i->first.d_qname <<" " <<i->first.d_qtype<< " "<<i->second.d_qcount <<" " <<i->second.d_answercount<<endl;
 418   }
 419
 420   cout<< boost::format("%d (%.02f%% of all) queries did not request recursion") % nonRDQueries % ((nonRDQueries*100.0)/queries) << endl;
 421   cout<< rdNonRAAnswers << " answers had recursion desired bit set, but recursion available=0 (for "<<rdnonra.size()<<" remotes)"<<endl;
 422   cout<<statmap.size()<<" queries went unanswered, of which "<< statmap.size()-unanswered<<" were answered on exact retransmit"<<endl;
 423   cout<<untracked<<" responses could not be matched to questions"<<endl;
 424   cout<<edns <<" questions requested EDNS processing, do=1: "<<dnssecOK<<", ad=1: "<<dnssecAD<<", cd=1: "<<dnssecCD<<endl;
 425
 426   if(answers) {
 427     cout<<(boost::format("%1% %|25t|%2%") % "Rcode" % "Count\n");
 428     for(rcodes_t::const_iterator i=rcodes.begin(); i!=rcodes.end(); ++i)
 429       cout<<(boost::format("%s %|25t|%d %|35t|(%.1f%%)") % RCode::to_s(i->first) % i->second % (i->second*100.0/answers))<<endl;
 430   }
 431
 432   uint32_t sum=0;
 433   //  ofstream stats("stats");
 434   uint32_t totpairs=0;
 435   double tottime=0;
 436   for(cumul_t::const_iterator i=cumul.begin(); i!=cumul.end(); ++i) {
 437     //    stats<<i->first<<"\t"<<(sum+=i->second)<<"\n";
 438     totpairs+=i->second;
 439     tottime+=i->first*i->second;
 440   }
 441
 442   typedef map<uint32_t, bool> done_t;
 443   done_t done;
 444   for(auto a : {50, 100, 200, 300, 400, 800, 1000, 2000, 4000, 8000, 32000, 64000, 256000, 1024000, 2048000})
 445     done[a]=false;
 446
 447   cout.setf(std::ios::fixed);
 448   cout.precision(4);
 449   sum=0;
 450
 451 #if HAVE_BOOST_GE_148
 452   if(g_vm.count("log-histogram")) {
 453     string fname = g_vm["stats-dir"].as<string>()+"/log-histogram";
 454     ofstream loglog(fname);
 455     if(!loglog)
 456       throw runtime_error("Unable to write statistics to "+fname);
 457
 458     writeLogHistogramFile(cumul, loglog);
 459   }
 460
 461   if(g_vm.count("full-histogram")) {
 462     string fname=g_vm["stats-dir"].as<string>()+"/full-histogram";
 463     ofstream loglog(fname);
 464     if(!loglog)
 465       throw runtime_error("Unable to write statistics to "+fname);
 466     writeFullHistogramFile(cumul, g_vm["full-histogram"].as<double>(), loglog);
 467   }
 468 #endif
 469
 470
 471   sum=0;
 472   double lastperc=0, perc=0;
 473   uint64_t lastsum=0;
 474
 475   for(cumul_t::const_iterator i=cumul.begin(); i!=cumul.end(); ++i) {
 476     for(done_t::iterator j=done.begin(); j!=done.end(); ++j) {
 477       if(!j->second && i->first > j->first) {
 478         j->second=true;
 479
 480         perc=sum*100.0/totpairs;
 481         if(j->first < 1024)
 482           cout<< perc <<"% of questions answered within " << j->first << " usec (";
 483         else
 484           cout<< perc <<"% of questions answered within " << j->first/1000.0 << " msec (";
 485
 486         cout<<perc-lastperc<<"%)\n";
 487         lastperc=sum*100.0/totpairs;
 488         lastsum=sum;
 489       }
 490     }
 491     sum+=i->second;
 492   }
 493
 494   for(auto j = done.begin(); j != done.end(); ++j) {
 495     if(!j->second) {
 496       perc=sum*100.0/totpairs;
 497       if(j->first < 1024)
 498         cout<< perc <<"% of questions answered within " << j->first << " usec (";
 499       else
 500         cout<< perc <<"% of questions answered within " << j->first/1000.0 << " msec (";
 501
 502       cout<<perc-lastperc<<"%)\n";
 503       lastperc=sum*100.0/totpairs;
 504       lastsum=sum;
 505       break;
 506     }
 507   }
 508
 509   cout<< (totpairs-lastsum)<<" responses ("<<((totpairs-lastsum)*100.0/answers) <<"%) older than "<< (done.rbegin()->first/1000000.0) <<" seconds"<<endl;
 510   if(totpairs)
 511     cout<<"Average non-late response time: "<<tottime/totpairs<<" usec"<<endl;
 512
 513   if(!g_vm["load-stats"].as<string>().empty()) {
 514     ofstream load(g_vm["load-stats"].as<string>().c_str());
 515     if(!load)
 516       throw runtime_error("Error writing load statistics to "+g_vm["load-stats"].as<string>());
 517     for(pcounts_t::value_type& val :  pcounts) {
 518       load<<val.first<<'\t'<<val.second.questions<<'\t'<<val.second.answers<<'\t'<<val.second.outstanding<<'\n';
 519     }
 520   }
 521
 522
 523   cout<<"Saw questions from "<<requestors.size()<<" distinct remotes, answers to "<<recipients.size()<<endl;
 524   ofstream remotes("remotes");
 525   for(const ComboAddress& rem :  requestors) {
 526     remotes<<rem.toString()<<'\n';
 527   }
 528
 529   vector<ComboAddress> diff;
 530   set_difference(requestors.begin(), requestors.end(), recipients.begin(), recipients.end(), back_inserter(diff), ComboAddress::addressOnlyLessThan());
 531   cout<<"Saw "<<diff.size()<<" unique remotes asking questions, but not getting RA answers"<<endl;
 532
 533   ofstream ignored("ignored");
 534   for(const ComboAddress& rem :  diff) {
 535     ignored<<rem.toString()<<'\n';
 536   }
 537   ofstream rdnonrafs("rdnonra");
 538   for(const ComboAddress& rem :  rdnonra) {
 539     rdnonrafs<<rem.toString()<<'\n';
 540   }
 541
 542   if(doServFailTree) {
 543     StatNode::Stat node;
 544     root.visit(visitor, node);
 545   }
 546
 547 }
 548 catch(std::exception& e)
 549 {
 550   cerr<<"Fatal: "<<e.what()<<endl;
 551 }