pdns/comfun.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #include "statbag.hh"
  23 #include "zoneparser-tng.hh"
  24 #include "namespaces.hh"
  25 #include "dnsrecords.hh"
  26 #include <fstream>
  27 #include <atomic>
  28 #include <thread>
  29 #include <unordered_set>
  30 #include "inflighter.cc"
  31 //#include "malloctrace.hh"
  32 StatBag S;
  33 bool g_quiet;
  34 std::unique_ptr<ofstream> g_powerdns;
  35 std::atomic<unsigned int> g_count;
  36 std::atomic<bool> g_stop;
  37
  38
  39 struct namecount {
  40   set<DNSName> names;
  41   unsigned int count;
  42   bool isPowerDNS{false};
  43 };
  44
  45 struct DNSResult
  46 {
  47   string content;
  48   int ttl{0};
  49   uint16_t qclass;
  50 };
  51
  52 struct NSQuery
  53 {
  54   ComboAddress a;
  55   set<DNSName> nsnames;
  56   DNSName qname;
  57   unsigned int count;
  58 };
  59
  60 struct SendReceive
  61 {
  62   typedef int Identifier;
  63   typedef DNSResult Answer; // ip
  64   int d_socket;
  65   deque<uint16_t> d_idqueue;
  66
  67   SendReceive(map<ComboAddress, namecount, ComboAddress::addressOnlyLessThan>& res) : d_res(res)
  68   {
  69     d_socket = socket(AF_INET, SOCK_DGRAM, 0);
  70     int val=1;
  71     setsockopt(d_socket, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val));
  72
  73     for(unsigned int id =0 ; id < std::numeric_limits<uint16_t>::max(); ++id)
  74       d_idqueue.push_back(id);
  75   }
  76
  77   ~SendReceive()
  78   {
  79     close(d_socket);
  80   }
  81
  82   Identifier send(NSQuery& domain)
  83   {
  84     //cerr<<"Sending query for '"<<domain<<"'"<<endl;
  85
  86     // send it, copy code from 'sdig'
  87     vector<uint8_t> packet;
  88
  89     DNSPacketWriter pw(packet, domain.qname, QType::TXT, 3);
  90
  91     if(d_idqueue.empty()) {
  92       cerr<<"Exhausted ids!"<<endl;
  93       exit(1);
  94     }
  95     pw.getHeader()->id = d_idqueue.front();
  96     d_idqueue.pop_front();
  97     pw.getHeader()->rd = 0;
  98     pw.getHeader()->qr = 0;
  99
 100     if(::sendto(d_socket, &*packet.begin(), packet.size(), 0, (struct sockaddr*)&domain.a, domain.a.getSocklen()) < 0)
 101       d_senderrors++;
 102
 103     return pw.getHeader()->id;
 104   }
 105
 106   bool receive(Identifier& id, DNSResult& dr)
 107   {
 108     if(waitForData(d_socket, 0, 500000) > 0) {
 109       char buf[512];
 110       ComboAddress from;
 111       from.sin4.sin_family = AF_INET;
 112       socklen_t socklen=from.getSocklen();
 113       int len = recvfrom(d_socket, buf, sizeof(buf),0, (struct sockaddr*)&from, &socklen);
 114       if(len < 0) {
 115         d_receiveerrors++;
 116         return 0;
 117       }
 118       else {
 119         d_receiveds++;
 120       }
 121       // parse packet, set 'id', fill out 'ip'
 122
 123       MOADNSParser mdp(false, string(buf, len));
 124       if(!g_quiet) {
 125         cout<<"Reply to question for qname='"<<mdp.d_qname<<"', qtype="<<DNSRecordContent::NumberToType(mdp.d_qtype)<<endl;
 126         cout<<"Rcode: "<<mdp.d_header.rcode<<", RD: "<<mdp.d_header.rd<<", QR: "<<mdp.d_header.qr;
 127         cout<<", TC: "<<mdp.d_header.tc<<", AA: "<<mdp.d_header.aa<<", opcode: "<<mdp.d_header.opcode<<endl;
 128       }
 129       id = mdp.d_header.id;
 130       d_idqueue.push_back(id);
 131       dr.qclass = mdp.d_qclass;
 132       dr.content.clear();
 133       dr.ttl=0;
 134       for(const auto& a : mdp.d_answers) {
 135         if(a.first.d_type == QType::TXT) {
 136           dr.content=a.first.d_content->getZoneRepresentation();
 137           dr.ttl=a.first.d_ttl;
 138         }
 139       }
 140       if(dr.content.empty())
 141         dr.content="RCode: "+RCode::to_s(mdp.d_header.rcode);
 142       return 1;
 143     }
 144     return 0;
 145   }
 146
 147   void deliverTimeout(const Identifier& id)
 148   {
 149     if(!g_quiet) {
 150       cout<<"Timeout for id "<<id<<endl;
 151     }
 152     d_idqueue.push_back(id);
 153   }
 154
 155   void deliverAnswer(NSQuery& domain, const DNSResult& dr, unsigned int usec)
 156   {
 157     cout<<domain.a.toString()<<"\t"<<domain.qname<<"\t";
 158     for(const auto& n : domain.nsnames)
 159       cout<<n<<",";
 160     cout<<"\t"<<domain.count<<"\t"<<dr.qclass<<'\t'<<dr.ttl<<": "<<dr.content<<endl;
 161
 162     if(dr.qclass==1 || toLower(dr.content).find("powerdns") != string::npos || dr.ttl==5) {
 163       auto f = d_res.find(domain.a);
 164       if(!f->second.isPowerDNS) {
 165         (*g_powerdns)<<domain.a.toString()<<'\t'<<domain.count<<'\t';
 166         for(const auto& n : domain.nsnames)
 167           (*g_powerdns)<<n<<'\t';
 168         (*g_powerdns)<<"\n";
 169         f->second.isPowerDNS=true;
 170       }
 171
 172     }
 173
 174   }
 175   unsigned int d_errors, d_nxdomains, d_nodatas, d_oks, d_unknowns;
 176   unsigned int d_receiveds, d_receiveerrors, d_senderrors;
 177   map<ComboAddress, namecount, ComboAddress::addressOnlyLessThan>& d_res;
 178 };
 179
 180
 181 struct RESResult
 182 {
 183   vector<ComboAddress> addrs;
 184   uint16_t rcode;
 185 };
 186
 187 typedef DNSName RESQuery;
 188
 189 struct SendReceiveRes
 190 {
 191   typedef int Identifier;
 192   typedef RESResult Answer; // ip
 193   int d_socket;
 194   deque<uint16_t> d_idqueue;
 195   map<DNSName, vector<ComboAddress>>& d_out;
 196   SendReceiveRes(const ComboAddress& remote, map<DNSName,vector<ComboAddress>>& out) : d_out(out)
 197   {
 198     d_socket = socket(AF_INET, SOCK_DGRAM, 0);
 199     int val=1;
 200     setsockopt(d_socket, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val));
 201     connect(d_socket, (struct sockaddr*)&remote, remote.getSocklen());
 202     for(unsigned int id =0 ; id < std::numeric_limits<uint16_t>::max(); ++id)
 203       d_idqueue.push_back(id);
 204   }
 205
 206   ~SendReceiveRes()
 207   {
 208     close(d_socket);
 209   }
 210
 211   Identifier send(RESQuery& domain)
 212   {
 213     //cerr<<"Sending query for '"<<domain<<"'"<<endl;
 214
 215     // send it, copy code from 'sdig'
 216     vector<uint8_t> packet;
 217
 218     DNSPacketWriter pw(packet, domain, QType::A);
 219
 220     if(d_idqueue.empty()) {
 221       cerr<<"Exhausted ids!"<<endl;
 222       exit(1);
 223     }
 224     pw.getHeader()->id = d_idqueue.front();
 225     d_idqueue.pop_front();
 226     pw.getHeader()->rd = 1;
 227     pw.getHeader()->qr = 0;
 228
 229     if(::send(d_socket, &*packet.begin(), packet.size(), 0) < 0) {
 230       cout<<"Error sending: "<<stringerror()<<endl;
 231       d_senderrors++;
 232     }
 233
 234     return pw.getHeader()->id;
 235   }
 236
 237   bool receive(Identifier& id, RESResult& dr)
 238   {
 239     if(waitForData(d_socket, 0, 500000) > 0) {
 240       char buf[512];
 241       ComboAddress from;
 242       from.sin4.sin_family = AF_INET;
 243       socklen_t socklen=from.getSocklen();
 244       int len = recvfrom(d_socket, buf, sizeof(buf),0, (struct sockaddr*)&from, &socklen);
 245       if(len < 0) {
 246         d_receiveerrors++;
 247         return 0;
 248       }
 249       else {
 250         d_receiveds++;
 251       }
 252       // parse packet, set 'id', fill out 'ip'
 253
 254       MOADNSParser mdp(false, string(buf, len));
 255       if(!g_quiet) {
 256         cout<<"Reply to question for qname='"<<mdp.d_qname<<"', qtype="<<DNSRecordContent::NumberToType(mdp.d_qtype)<<endl;
 257         cout<<"Rcode: "<<mdp.d_header.rcode<<", RD: "<<mdp.d_header.rd<<", QR: "<<mdp.d_header.qr<<", answers: "<<mdp.d_answers.size();
 258         cout<<", TC: "<<mdp.d_header.tc<<", AA: "<<mdp.d_header.aa<<", opcode: "<<mdp.d_header.opcode<<endl;
 259       }
 260       id = mdp.d_header.id;
 261       d_idqueue.push_back(id);
 262       dr.rcode = mdp.d_header.rcode;
 263       dr.addrs.clear();
 264       for(const auto& a : mdp.d_answers) {
 265         if(a.first.d_name != mdp.d_qname)
 266           continue;
 267         if(a.first.d_type == QType::A || a.first.d_type == QType::AAAA) {
 268           if(!g_quiet)
 269             cout<<a.first.d_content->getZoneRepresentation()<<endl;
 270           dr.addrs.push_back(getAddr(a.first));
 271         }
 272       }
 273       ++g_count;
 274       return 1;
 275     }
 276     return 0;
 277   }
 278
 279   void deliverTimeout(const Identifier& id)
 280   {
 281     if(!g_quiet) {
 282       cout<<"Timeout for id "<<id<<endl;
 283     }
 284     ++g_count;
 285     d_idqueue.push_back(id);
 286   }
 287
 288   void deliverAnswer(DNSName& domain, const RESResult& dr, unsigned int usec)
 289   {
 290     d_out[domain]=dr.addrs;
 291     cout<<domain<<"\t"<<dr.rcode<<'\t';
 292     for(const auto& a : dr.addrs)
 293       cout<<a.toString()<<'\t';
 294     cout<<'\n';
 295   }
 296   unsigned int d_errors, d_nxdomains, d_nodatas, d_oks, d_unknowns;
 297   unsigned int d_receiveds, d_receiveerrors, d_senderrors;
 298 };
 299
 300
 301 void printStats()
 302 {
 303   while(!g_stop) {
 304     sleep(1);
 305     cerr<<"\r"<<g_count;
 306     cerr.flush();
 307   }
 308   cerr<<"\n";
 309 }
 310
 311 int parseZone(const std::string& str, unsigned int limit)
 312 {
 313   ZoneParserTNG zpt(str);
 314   zpt.disableGenerate();
 315   DNSResourceRecord rr;
 316
 317   std::thread stats(printStats);
 318
 319   map<DNSName,unsigned int> nsnames;
 320   map<DNSName,set<ComboAddress,ComboAddress::addressOnlyLessThan> > addresses;
 321
 322
 323   while(zpt.get(rr)) {
 324     if(rr.qtype.getCode() == QType::NS)
 325       nsnames[DNSName(rr.content)]++;
 326     else if(rr.qtype.getCode() == QType::A || rr.qtype.getCode() == QType::AAAA) {
 327       DNSRecord dr(rr);
 328       addresses[rr.qname].insert(getAddr(dr, 53));
 329     }
 330     ++g_count;
 331     if(g_count == limit)
 332       break;
 333   }
 334   g_stop=true;
 335   stats.join();
 336
 337   cout<<"Got "<<nsnames.size()<<" different nameserver names"<<endl;
 338   cout<<"Got at least one address for "<<addresses.size()<<" names"<<endl;
 339
 340   ofstream ns(str+".nameservers");
 341   ofstream needres(str+".needres");
 342   for(const auto& a: nsnames) {
 343     ns<<a.first<<"\t"<<a.second<<"\t";
 344     if(auto hit=rplookup(addresses, a.first)) {
 345       for(const auto& b : *hit)
 346         ns<<b.toString()<<"\t";
 347     }
 348     else
 349       needres<<a.first<<"\n";
 350     ns<<"\n";
 351   }
 352   return 0;
 353 }
 354
 355 int resolveNS(const std::string& fname)
 356 {
 357   string line;
 358   ifstream needres(fname);
 359   if(!needres)
 360     unixDie("Unable to open file "+fname);
 361   vector<DNSName> tores;
 362   while(getline(needres,line)) {
 363     tores.push_back(DNSName(line));
 364   }
 365   cerr<<"Going to resolve "<<tores.size()<<" names"<<endl;
 366   std::thread stats(printStats);
 367   map<DNSName, vector<ComboAddress>> output;
 368   SendReceiveRes sr(ComboAddress("192.168.1.2", 53), output);
 369   Inflighter<vector<DNSName>, SendReceiveRes> inflighter(tores, sr);
 370   inflighter.d_maxInFlight = 1000;
 371   inflighter.d_timeoutSeconds = 3;
 372   inflighter.d_burst = 100;
 373   for(;;) {
 374     try {
 375       inflighter.run();
 376       break;
 377     }
 378     catch(std::exception& e) {
 379       cerr<<"Caught exception: "<<e.what()<<endl;
 380     }
 381   }
 382   g_stop=true;
 383   stats.join();
 384   return EXIT_SUCCESS;
 385 }
 386
 387 void readRESNames(const std::string& fname, map<DNSName, vector<ComboAddress>>& addrs)
 388 {
 389   ifstream ifs(fname);
 390   if(!ifs)
 391     unixDie("Reading resolved names from "+fname+": "+stringerror());
 392   vector<string> parts;
 393   string line;
 394   addrs.clear();
 395   while(getline(ifs, line)) {
 396     parts.clear();
 397     stringtok(parts, line,"\t");
 398     for(unsigned int n=2; n < parts.size(); ++n)
 399       addrs[DNSName(parts[0])].push_back(ComboAddress(parts[n], 53));
 400   }
 401   //EARTH.DOMAINS.SHELTEK.CA.       0       67.15.253.219   67.15.47.188    67.15.253.252   67.15.253.251   67.15.47.189    67.15.253.220
 402   cerr<<"Got "<<addrs.size()<<" resolved nameserver names from file"<<endl;
 403
 404 }
 405
 406 int main(int argc, char**argv)
 407 try
 408 {
 409   g_quiet=true;
 410   reportAllTypes();
 411   string mode=argv[1];
 412   if(mode == "parse-zone") {
 413     unsigned int limit = 0;
 414     if(argc > 3)
 415       limit = atoi(argv[3]);
 416
 417     return parseZone(argv[2], limit);
 418   }
 419   else if(mode=="resolve-ns") {
 420     return resolveNS(string(argv[2])+".needres");
 421   }
 422   else if(mode=="scan-ns") {
 423     ifstream ns(string(argv[2])+".nameservers");
 424     g_powerdns = std::unique_ptr<ofstream>(new ofstream(string(argv[2])+".powerdns"));
 425     string line;
 426     int count=0;
 427     vector<string> parts;
 428
 429     struct NSCount
 430     {
 431       unsigned int count{0};
 432       vector<ComboAddress> addrs;
 433     };
 434     map<DNSName, NSCount> stats;
 435     NSCount nscount;
 436     // NS1.IHOST2000.COM.      9       162.251.82.122  162.251.82.123  162.251.82.250  162.251.82.251
 437     while(getline(ns, line)) {
 438       ++count;
 439       parts.clear();
 440       stringtok(parts, line,"\t");
 441       nscount.count=atoi(parts[1].c_str());
 442       nscount.addrs.clear();
 443       for(unsigned int n = 2; n < parts.size(); ++n)
 444         nscount.addrs.push_back(ComboAddress(parts[n], 53));
 445       stats.insert({DNSName(parts[0]), nscount});
 446     }
 447     cerr<<"Had "<<count<<" lines from summary"<<endl;
 448
 449     map<DNSName, vector<ComboAddress>> lookedup;
 450     readRESNames(argv[2]+string(".resolved"), lookedup);
 451
 452     map<ComboAddress, namecount, ComboAddress::addressOnlyLessThan> pure;
 453
 454     unsigned int noaddrs=0;
 455     for(const auto& s : stats) {
 456       auto ptr = &s.second.addrs;
 457       if(ptr->empty()) {
 458         if(lookedup.count(s.first)) {
 459           ptr = &lookedup[s.first];
 460         }
 461         else {
 462           //cout<<"Have no address for "<<s.first.toString()<<endl;
 463           noaddrs++;
 464         }
 465       }
 466
 467       for(const auto& a : *ptr) {
 468         pure[a].count += s.second.count;
 469         pure[a].names.insert(s.first);
 470       }
 471     }
 472
 473     cerr<<"Have "<<pure.size()<<" IP addresses to query, "<<noaddrs<<" names w/o address"<<endl;
 474     SendReceive sr(pure);
 475     vector<NSQuery> domains;
 476
 477     Inflighter<vector<NSQuery>, SendReceive> inflighter(domains, sr);
 478     inflighter.d_maxInFlight = 1000;
 479     inflighter.d_timeoutSeconds = 3;
 480     inflighter.d_burst = 100;
 481
 482     for(const auto& p : pure) {
 483       NSQuery nsq;
 484       nsq.a=p.first;
 485       nsq.nsnames = p.second.names;
 486       nsq.count = p.second.count;
 487
 488       nsq.qname=DNSName("version.bind");
 489       domains.push_back(nsq);
 490       nsq.qname=DNSName("id.server");
 491       domains.push_back(nsq);
 492       nsq.qname=DNSName("bind.version");
 493       domains.push_back(nsq);
 494     }
 495
 496     sort(domains.begin(), domains.end(), [](const NSQuery& a, const NSQuery& b) { return b.count < a.count; });
 497     for(;;) {
 498       try {
 499         inflighter.run();
 500         break;
 501       }
 502       catch(std::exception& e) {
 503         cerr<<"Caught exception: "<<e.what()<<endl;
 504       }
 505     }
 506   }
 507   else if(mode=="score-ns") {
 508     std::unordered_set<DNSName> powerdns;
 509     ifstream ifs(string(argv[2])+".powerdns");
 510     string line;
 511     vector<string> parts;
 512     while(getline(ifs,line)) {
 513       // 64.96.240.53    1234     NS1.UNIREGISTRYMARKET.LINK.     NS1.INTERNETTRAFFIC.COM.        BUY.INTERNETTRAFFIC.COM.        NS3.SECUREDOFFERS.COM.  NS3.GI.NET.     NS3.IT.GI.NET.  NS3.EASILY.NET.
 514       parts.clear();
 515       stringtok(parts, line);
 516       for(unsigned int n=2; n < parts.size(); ++n)
 517         powerdns.insert(DNSName(parts[n]));
 518     }
 519     cerr<<"Have "<<powerdns.size()<<" known NS names that are PowerDNS"<<endl;
 520     ZoneParserTNG zpt(argv[2]);
 521     zpt.disableGenerate();
 522     DNSResourceRecord rr;
 523
 524     set<DNSName> seen, pdnsdomains;
 525     int count=0;
 526     while(zpt.get(rr)) {
 527       if(!seen.count(rr.qname)) {
 528         seen.insert(rr.qname);
 529       }
 530       if(rr.qtype.getCode() == QType::NS && powerdns.count(DNSName(rr.content)) && !pdnsdomains.count(DNSName(rr.qname))) {
 531         pdnsdomains.insert(DNSName(rr.qname));
 532       }
 533       if(!(count%100000)) {
 534         cerr<<"\rUnique domains: "<<seen.size()<<", PowerDNS domains: "<<pdnsdomains.size()<<" ("<<(pdnsdomains.size()*100.0/seen.size())<<"%)";
 535       }
 536       count++;
 537     }
 538     cerr<<"\n";
 539   }
 540   else {
 541     cerr<<"Unknown mode "<<argv[1]<<endl;
 542   }
 543   //  cout<<g_mtracer->topAllocatorsString(20)<<endl;
 544 }
 545 catch(PDNSException& pe) {
 546   cerr<<"Fatal error: "<<pe.reason<<endl;
 547 }