pdns/comfun.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #include "statbag.hh"
  23 #include "zoneparser-tng.hh"
  24 #include "namespaces.hh"
  25 #include "dnsrecords.hh"
  26 #include <fstream>
  27 #include <atomic>
  28 #include <thread>
  29 #include <unordered_set>
  30 #include "inflighter.cc"
  31 //#include "malloctrace.hh"
  32 StatBag S;
  33 bool g_quiet;
  34 std::unique_ptr<ofstream> g_powerdns;
  35 std::atomic<unsigned int> g_count;
  36 std::atomic<bool> g_stop;
  37
  38
  39 struct namecount {
  40   set<DNSName> names;
  41   unsigned int count;
  42   bool isPowerDNS{false};
  43 };
  44
  45 struct DNSResult
  46 {
  47   string content;
  48   int ttl{0};
  49   uint16_t qclass;
  50 };
  51
  52 struct NSQuery
  53 {
  54   ComboAddress a;
  55   set<DNSName> nsnames;
  56   DNSName qname;
  57   unsigned int count;
  58 };
  59
  60 struct SendReceive
  61 {
  62   typedef int Identifier;
  63   typedef DNSResult Answer; // ip
  64   int d_socket;
  65   deque<uint16_t> d_idqueue;
  66
  67   SendReceive(map<ComboAddress, namecount, ComboAddress::addressOnlyLessThan>& res) : d_res(res)
  68   {
  69     d_socket = socket(AF_INET, SOCK_DGRAM, 0);
  70     int val=1;
  71     setsockopt(d_socket, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val));
  72
  73     for(unsigned int id =0 ; id < std::numeric_limits<uint16_t>::max(); ++id)
  74       d_idqueue.push_back(id);
  75   }
  76
  77   ~SendReceive()
  78   {
  79     close(d_socket);
  80   }
  81
  82   Identifier send(NSQuery& domain)
  83   {
  84     //cerr<<"Sending query for '"<<domain<<"'"<<endl;
  85
  86     // send it, copy code from 'sdig'
  87     vector<uint8_t> packet;
  88
  89     DNSPacketWriter pw(packet, domain.qname, QType::TXT, 3);
  90
  91     if(d_idqueue.empty()) {
  92       cerr<<"Exhausted ids!"<<endl;
  93       exit(1);
  94     }
  95     pw.getHeader()->id = d_idqueue.front();
  96     d_idqueue.pop_front();
  97     pw.getHeader()->rd = 0;
  98     pw.getHeader()->qr = 0;
  99
 100     if(::sendto(d_socket, &*packet.begin(), packet.size(), 0, (struct sockaddr*)&domain.a, domain.a.getSocklen()) < 0)
 101       d_senderrors++;
 102
 103     return pw.getHeader()->id;
 104   }
 105
 106   bool receive(Identifier& id, DNSResult& dr)
 107   {
 108     if(waitForData(d_socket, 0, 500000) > 0) {
 109       char buf[512];
 110       ComboAddress from;
 111       from.sin4.sin_family = AF_INET;
 112       socklen_t socklen=from.getSocklen();
 113       int len = recvfrom(d_socket, buf, sizeof(buf),0, (struct sockaddr*)&from, &socklen);
 114       if(len < 0) {
 115         d_receiveerrors++;
 116         return 0;
 117       }
 118       else {
 119         d_receiveds++;
 120       }
 121       // parse packet, set 'id', fill out 'ip'
 122
 123       MOADNSParser mdp(false, string(buf, len));
 124       if(!g_quiet) {
 125         cout<<"Reply to question for qname='"<<mdp.d_qname<<"', qtype="<<DNSRecordContent::NumberToType(mdp.d_qtype)<<endl;
 126         cout<<"Rcode: "<<mdp.d_header.rcode<<", RD: "<<mdp.d_header.rd<<", QR: "<<mdp.d_header.qr;
 127         cout<<", TC: "<<mdp.d_header.tc<<", AA: "<<mdp.d_header.aa<<", opcode: "<<mdp.d_header.opcode<<endl;
 128       }
 129       id = mdp.d_header.id;
 130       d_idqueue.push_back(id);
 131       dr.qclass = mdp.d_qclass;
 132       dr.content.clear();
 133       dr.ttl=0;
 134       for(const auto& a : mdp.d_answers) {
 135         if(a.first.d_type == QType::TXT) {
 136           dr.content=a.first.d_content->getZoneRepresentation();
 137           dr.ttl=a.first.d_ttl;
 138         }
 139       }
 140       if(dr.content.empty())
 141         dr.content="RCode: "+RCode::to_s(mdp.d_header.rcode);
 142       return 1;
 143     }
 144     return 0;
 145   }
 146
 147   void deliverTimeout(const Identifier& id)
 148   {
 149     if(!g_quiet) {
 150       cout<<"Timeout for id "<<id<<endl;
 151     }
 152     d_idqueue.push_back(id);
 153   }
 154
 155   void deliverAnswer(NSQuery& domain, const DNSResult& dr, unsigned int usec)
 156   {
 157     cout<<domain.a.toString()<<"\t"<<domain.qname<<"\t";
 158     for(const auto& n : domain.nsnames)
 159       cout<<n<<",";
 160     cout<<"\t"<<domain.count<<"\t"<<dr.qclass<<'\t'<<dr.ttl<<": "<<dr.content<<endl;
 161
 162     if(dr.qclass==1 || toLower(dr.content).find("powerdns") != string::npos || dr.ttl==5) {
 163       auto f = d_res.find(domain.a);
 164       if(!f->second.isPowerDNS) {
 165         (*g_powerdns)<<domain.a.toString()<<'\t'<<domain.count<<'\t';
 166         for(const auto& n : domain.nsnames)
 167           (*g_powerdns)<<n<<'\t';
 168         (*g_powerdns)<<"\n";
 169         f->second.isPowerDNS=true;
 170       }
 171
 172     }
 173
 174   }
 175   unsigned int d_errors, d_nxdomains, d_nodatas, d_oks, d_unknowns;
 176   unsigned int d_receiveds, d_receiveerrors, d_senderrors;
 177   map<ComboAddress, namecount, ComboAddress::addressOnlyLessThan>& d_res;
 178 };
 179
 180
 181 struct RESResult
 182 {
 183   vector<ComboAddress> addrs;
 184   uint16_t rcode;
 185 };
 186
 187 typedef DNSName RESQuery;
 188
 189 struct SendReceiveRes
 190 {
 191   typedef int Identifier;
 192   typedef RESResult Answer; // ip
 193   int d_socket;
 194   deque<uint16_t> d_idqueue;
 195   map<DNSName, vector<ComboAddress>>& d_out;
 196   SendReceiveRes(const ComboAddress& remote, map<DNSName,vector<ComboAddress>>& out) : d_out(out)
 197   {
 198     d_socket = socket(AF_INET, SOCK_DGRAM, 0);
 199     int val=1;
 200     setsockopt(d_socket, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val));
 201     connect(d_socket, (struct sockaddr*)&remote, remote.getSocklen());
 202     for(unsigned int id =0 ; id < std::numeric_limits<uint16_t>::max(); ++id)
 203       d_idqueue.push_back(id);
 204   }
 205
 206   ~SendReceiveRes()
 207   {
 208     close(d_socket);
 209   }
 210
 211   Identifier send(RESQuery& domain)
 212   {
 213     //cerr<<"Sending query for '"<<domain<<"'"<<endl;
 214
 215     // send it, copy code from 'sdig'
 216     vector<uint8_t> packet;
 217
 218     DNSPacketWriter pw(packet, domain, QType::A);
 219
 220     if(d_idqueue.empty()) {
 221       cerr<<"Exhausted ids!"<<endl;
 222       exit(1);
 223     }
 224     pw.getHeader()->id = d_idqueue.front();
 225     d_idqueue.pop_front();
 226     pw.getHeader()->rd = 1;
 227     pw.getHeader()->qr = 0;
 228
 229     if(::send(d_socket, &*packet.begin(), packet.size(), 0) < 0) {
 230       cout<<"Error sending: "<<strerror(errno)<<endl;
 231       d_senderrors++;
 232     }
 233
 234     return pw.getHeader()->id;
 235   }
 236
 237   bool receive(Identifier& id, RESResult& dr)
 238   {
 239     if(waitForData(d_socket, 0, 500000) > 0) {
 240       char buf[512];
 241       ComboAddress from;
 242       from.sin4.sin_family = AF_INET;
 243       socklen_t socklen=from.getSocklen();
 244       int len = recvfrom(d_socket, buf, sizeof(buf),0, (struct sockaddr*)&from, &socklen);
 245       if(len < 0) {
 246         d_receiveerrors++;
 247         return 0;
 248       }
 249       else {
 250         d_receiveds++;
 251       }
 252       // parse packet, set 'id', fill out 'ip'
 253
 254       MOADNSParser mdp(false, string(buf, len));
 255       if(!g_quiet) {
 256         cout<<"Reply to question for qname='"<<mdp.d_qname<<"', qtype="<<DNSRecordContent::NumberToType(mdp.d_qtype)<<endl;
 257         cout<<"Rcode: "<<mdp.d_header.rcode<<", RD: "<<mdp.d_header.rd<<", QR: "<<mdp.d_header.qr<<", answers: "<<mdp.d_answers.size();
 258         cout<<", TC: "<<mdp.d_header.tc<<", AA: "<<mdp.d_header.aa<<", opcode: "<<mdp.d_header.opcode<<endl;
 259       }
 260       id = mdp.d_header.id;
 261       d_idqueue.push_back(id);
 262       dr.rcode = mdp.d_header.rcode;
 263       dr.addrs.clear();
 264       for(const auto& a : mdp.d_answers) {
 265         if(a.first.d_name != mdp.d_qname)
 266           continue;
 267         if(a.first.d_type == QType::A || a.first.d_type == QType::AAAA) {
 268           if(!g_quiet)
 269             cout<<a.first.d_content->getZoneRepresentation()<<endl;
 270           dr.addrs.push_back(getAddr(a.first));
 271         }
 272       }
 273       ++g_count;
 274       return 1;
 275     }
 276     return 0;
 277   }
 278
 279   void deliverTimeout(const Identifier& id)
 280   {
 281     if(!g_quiet) {
 282       cout<<"Timeout for id "<<id<<endl;
 283     }
 284     ++g_count;
 285     d_idqueue.push_back(id);
 286   }
 287
 288   void deliverAnswer(DNSName& domain, const RESResult& dr, unsigned int usec)
 289   {
 290     d_out[domain]=dr.addrs;
 291     cout<<domain<<"\t"<<dr.rcode<<'\t';
 292     for(const auto& a : dr.addrs)
 293       cout<<a.toString()<<'\t';
 294     cout<<'\n';
 295   }
 296   unsigned int d_errors, d_nxdomains, d_nodatas, d_oks, d_unknowns;
 297   unsigned int d_receiveds, d_receiveerrors, d_senderrors;
 298 };
 299
 300
 301 void printStats()
 302 {
 303   while(!g_stop) {
 304     sleep(1);
 305     cerr<<"\r"<<g_count;
 306     cerr.flush();
 307   }
 308   cerr<<"\n";
 309 }
 310
 311 int parseZone(const std::string& str, unsigned int limit)
 312 {
 313   ZoneParserTNG zpt(str);
 314   DNSResourceRecord rr;
 315
 316   std::thread stats(printStats);
 317
 318   map<DNSName,unsigned int> nsnames;
 319   map<DNSName,set<ComboAddress,ComboAddress::addressOnlyLessThan> > addresses;
 320
 321
 322   while(zpt.get(rr)) {
 323     if(rr.qtype.getCode() == QType::NS)
 324       nsnames[DNSName(rr.content)]++;
 325     else if(rr.qtype.getCode() == QType::A || rr.qtype.getCode() == QType::AAAA) {
 326       DNSRecord dr(rr);
 327       addresses[rr.qname].insert(getAddr(dr, 53));
 328     }
 329     ++g_count;
 330     if(g_count == limit)
 331       break;
 332   }
 333   g_stop=true;
 334   stats.join();
 335
 336   cout<<"Got "<<nsnames.size()<<" different nameserver names"<<endl;
 337   cout<<"Got at least one address for "<<addresses.size()<<" names"<<endl;
 338
 339   ofstream ns(str+".nameservers");
 340   ofstream needres(str+".needres");
 341   for(const auto& a: nsnames) {
 342     ns<<a.first<<"\t"<<a.second<<"\t";
 343     if(auto hit=rplookup(addresses, a.first)) {
 344       for(const auto& b : *hit)
 345         ns<<b.toString()<<"\t";
 346     }
 347     else
 348       needres<<a.first<<"\n";
 349     ns<<"\n";
 350   }
 351   return 0;
 352 }
 353
 354 int resolveNS(const std::string& fname)
 355 {
 356   string line;
 357   ifstream needres(fname);
 358   if(!needres)
 359     unixDie("Unable to open file "+fname);
 360   vector<DNSName> tores;
 361   while(getline(needres,line)) {
 362     tores.push_back(DNSName(line));
 363   }
 364   cerr<<"Going to resolve "<<tores.size()<<" names"<<endl;
 365   std::thread stats(printStats);
 366   map<DNSName, vector<ComboAddress>> output;
 367   SendReceiveRes sr(ComboAddress("192.168.1.2", 53), output);
 368   Inflighter<vector<DNSName>, SendReceiveRes> inflighter(tores, sr);
 369   inflighter.d_maxInFlight = 1000;
 370   inflighter.d_timeoutSeconds = 3;
 371   inflighter.d_burst = 100;
 372   for(;;) {
 373     try {
 374       inflighter.run();
 375       break;
 376     }
 377     catch(std::exception& e) {
 378       cerr<<"Caught exception: "<<e.what()<<endl;
 379     }
 380   }
 381   g_stop=true;
 382   stats.join();
 383   return EXIT_SUCCESS;
 384 }
 385
 386 void readRESNames(const std::string& fname, map<DNSName, vector<ComboAddress>>& addrs)
 387 {
 388   ifstream ifs(fname);
 389   if(!ifs)
 390     unixDie("Reading resolved names from "+fname+": "+string(strerror(errno)));
 391   vector<string> parts;
 392   string line;
 393   addrs.clear();
 394   while(getline(ifs, line)) {
 395     parts.clear();
 396     stringtok(parts, line,"\t");
 397     for(unsigned int n=2; n < parts.size(); ++n)
 398       addrs[DNSName(parts[0])].push_back(ComboAddress(parts[n], 53));
 399   }
 400   //EARTH.DOMAINS.SHELTEK.CA.       0       67.15.253.219   67.15.47.188    67.15.253.252   67.15.253.251   67.15.47.189    67.15.253.220
 401   cerr<<"Got "<<addrs.size()<<" resolved nameserver names from file"<<endl;
 402
 403 }
 404
 405 int main(int argc, char**argv)
 406 try
 407 {
 408   g_quiet=true;
 409   reportAllTypes();
 410   string mode=argv[1];
 411   if(mode == "parse-zone") {
 412     unsigned int limit = 0;
 413     if(argc > 3)
 414       limit = atoi(argv[3]);
 415
 416     return parseZone(argv[2], limit);
 417   }
 418   else if(mode=="resolve-ns") {
 419     return resolveNS(string(argv[2])+".needres");
 420   }
 421   else if(mode=="scan-ns") {
 422     ifstream ns(string(argv[2])+".nameservers");
 423     g_powerdns = make_unique<ofstream>(string(argv[2])+".powerdns");
 424     string line;
 425     int count=0;
 426     vector<string> parts;
 427
 428     struct NSCount
 429     {
 430       unsigned int count{0};
 431       vector<ComboAddress> addrs;
 432     };
 433     map<DNSName, NSCount> stats;
 434     NSCount nscount;
 435     // NS1.IHOST2000.COM.      9       162.251.82.122  162.251.82.123  162.251.82.250  162.251.82.251
 436     while(getline(ns, line)) {
 437       ++count;
 438       parts.clear();
 439       stringtok(parts, line,"\t");
 440       nscount.count=atoi(parts[1].c_str());
 441       nscount.addrs.clear();
 442       for(unsigned int n = 2; n < parts.size(); ++n)
 443         nscount.addrs.push_back(ComboAddress(parts[n], 53));
 444       stats.insert({DNSName(parts[0]), nscount});
 445     }
 446     cerr<<"Had "<<count<<" lines from summary"<<endl;
 447
 448     map<DNSName, vector<ComboAddress>> lookedup;
 449     readRESNames(argv[2]+string(".resolved"), lookedup);
 450
 451     map<ComboAddress, namecount, ComboAddress::addressOnlyLessThan> pure;
 452
 453     unsigned int noaddrs=0;
 454     for(const auto& s : stats) {
 455       auto ptr = &s.second.addrs;
 456       if(ptr->empty()) {
 457         if(lookedup.count(s.first)) {
 458           ptr = &lookedup[s.first];
 459         }
 460         else {
 461           //cout<<"Have no address for "<<s.first.toString()<<endl;
 462           noaddrs++;
 463         }
 464       }
 465
 466       for(const auto& a : *ptr) {
 467         pure[a].count += s.second.count;
 468         pure[a].names.insert(s.first);
 469       }
 470     }
 471
 472     cerr<<"Have "<<pure.size()<<" IP addresses to query, "<<noaddrs<<" names w/o address"<<endl;
 473     SendReceive sr(pure);
 474     vector<NSQuery> domains;
 475
 476     Inflighter<vector<NSQuery>, SendReceive> inflighter(domains, sr);
 477     inflighter.d_maxInFlight = 1000;
 478     inflighter.d_timeoutSeconds = 3;
 479     inflighter.d_burst = 100;
 480
 481     for(const auto& p : pure) {
 482       NSQuery nsq;
 483       nsq.a=p.first;
 484       nsq.nsnames = p.second.names;
 485       nsq.count = p.second.count;
 486
 487       nsq.qname=DNSName("version.bind");
 488       domains.push_back(nsq);
 489       nsq.qname=DNSName("id.server");
 490       domains.push_back(nsq);
 491       nsq.qname=DNSName("bind.version");
 492       domains.push_back(nsq);
 493     }
 494
 495     sort(domains.begin(), domains.end(), [](const NSQuery& a, const NSQuery& b) { return b.count < a.count; });
 496     for(;;) {
 497       try {
 498         inflighter.run();
 499         break;
 500       }
 501       catch(std::exception& e) {
 502         cerr<<"Caught exception: "<<e.what()<<endl;
 503       }
 504     }
 505   }
 506   else if(mode=="score-ns") {
 507     std::unordered_set<DNSName> powerdns;
 508     ifstream ifs(string(argv[2])+".powerdns");
 509     string line;
 510     vector<string> parts;
 511     while(getline(ifs,line)) {
 512       // 64.96.240.53    1234     NS1.UNIREGISTRYMARKET.LINK.     NS1.INTERNETTRAFFIC.COM.        BUY.INTERNETTRAFFIC.COM.        NS3.SECUREDOFFERS.COM.  NS3.GI.NET.     NS3.IT.GI.NET.  NS3.EASILY.NET.
 513       parts.clear();
 514       stringtok(parts, line);
 515       for(unsigned int n=2; n < parts.size(); ++n)
 516         powerdns.insert(DNSName(parts[n]));
 517     }
 518     cerr<<"Have "<<powerdns.size()<<" known NS names that are PowerDNS"<<endl;
 519     ZoneParserTNG zpt(argv[2]);
 520     DNSResourceRecord rr;
 521
 522     set<DNSName> seen, pdnsdomains;
 523     int count=0;
 524     while(zpt.get(rr)) {
 525       if(!seen.count(rr.qname)) {
 526         seen.insert(rr.qname);
 527       }
 528       if(rr.qtype.getCode() == QType::NS && powerdns.count(DNSName(rr.content)) && !pdnsdomains.count(DNSName(rr.qname))) {
 529         pdnsdomains.insert(DNSName(rr.qname));
 530       }
 531       if(!(count%100000)) {
 532         cerr<<"\rUnique domains: "<<seen.size()<<", PowerDNS domains: "<<pdnsdomains.size()<<" ("<<(pdnsdomains.size()*100.0/seen.size())<<"%)";
 533       }
 534       count++;
 535     }
 536     cerr<<"\n";
 537   }
 538   else {
 539     cerr<<"Unknown mode "<<argv[1]<<endl;
 540   }
 541   //  cout<<g_mtracer->topAllocatorsString(20)<<endl;
 542 }
 543 catch(PDNSException& pe) {
 544   cerr<<"Fatal error: "<<pe.reason<<endl;
 545 }