pdns/syncres.hh

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #pragma once
  23 #include <string>
  24 #include <atomic>
  25 #include "utility.hh"
  26 #include "dns.hh"
  27 #include "qtype.hh"
  28 #include <vector>
  29 #include <set>
  30 #include <unordered_set>
  31 #include <map>
  32 #include <cmath>
  33 #include <iostream>
  34 #include <utility>
  35 #include "misc.hh"
  36 #include "lwres.hh"
  37 #include <boost/optional.hpp>
  38 #include <boost/circular_buffer.hpp>
  39 #include <boost/utility.hpp>
  40 #include "sstuff.hh"
  41 #include "recursor_cache.hh"
  42 #include "recpacketcache.hh"
  43 #include <boost/tuple/tuple.hpp>
  44 #include <boost/optional.hpp>
  45 #include <boost/tuple/tuple_comparison.hpp>
  46 #include "mtasker.hh"
  47 #include "iputils.hh"
  48 #include "validate.hh"
  49 #include "ednssubnet.hh"
  50 #include "filterpo.hh"
  51 #include "negcache.hh"
  52
  53 #ifdef HAVE_CONFIG_H
  54 #include "config.h"
  55 #endif
  56
  57 #ifdef HAVE_PROTOBUF
  58 #include <boost/uuid/uuid.hpp>
  59 #include <boost/uuid/uuid_generators.hpp>
  60 #endif
  61
  62 class RecursorLua4;
  63
  64 typedef map<
  65   DNSName,
  66   pair<
  67     vector<ComboAddress>,
  68     bool
  69   >
  70 > NsSet;
  71
  72 template<class Thing> class Throttle : public boost::noncopyable
  73 {
  74 public:
  75   Throttle()
  76   {
  77     d_limit=3;
  78     d_ttl=60;
  79     d_last_clean=time(0);
  80   }
  81   bool shouldThrottle(time_t now, const Thing& t)
  82   {
  83     if(now > d_last_clean + 300 ) {
  84
  85       d_last_clean=now;
  86       for(typename cont_t::iterator i=d_cont.begin();i!=d_cont.end();) {
  87         if( i->second.ttd < now) {
  88           d_cont.erase(i++);
  89         }
  90         else
  91           ++i;
  92       }
  93     }
  94
  95     typename cont_t::iterator i=d_cont.find(t);
  96     if(i==d_cont.end())
  97       return false;
  98     if(now > i->second.ttd || i->second.count == 0) {
  99       d_cont.erase(i);
 100       return false;
 101     }
 102     i->second.count--;
 103
 104     return true; // still listed, still blocked
 105   }
 106   void throttle(time_t now, const Thing& t, time_t ttl=0, unsigned int tries=0)
 107   {
 108     typename cont_t::iterator i=d_cont.find(t);
 109     entry e={ now+(ttl ? ttl : d_ttl), tries ? tries : d_limit};
 110
 111     if(i==d_cont.end()) {
 112       d_cont[t]=e;
 113     }
 114     else if(i->second.ttd > e.ttd || (i->second.count) < e.count)
 115       d_cont[t]=e;
 116   }
 117
 118   unsigned int size() const
 119   {
 120     return (unsigned int)d_cont.size();
 121   }
 122
 123   void clear()
 124   {
 125     d_cont.clear();
 126   }
 127 private:
 128   unsigned int d_limit;
 129   time_t d_ttl;
 130   time_t d_last_clean;
 131   struct entry
 132   {
 133     time_t ttd;
 134     unsigned int count;
 135   };
 136   typedef map<Thing,entry> cont_t;
 137   cont_t d_cont;
 138 };
 139
 140
 141 /** Class that implements a decaying EWMA.
 142     This class keeps an exponentially weighted moving average which, additionally, decays over time.
 143     The decaying is only done on get.
 144 */
 145 class DecayingEwma
 146 {
 147 public:
 148   DecayingEwma() :  d_val(0.0)
 149   {
 150     d_needinit=true;
 151     d_last.tv_sec = d_last.tv_usec = 0;
 152     d_lastget=d_last;
 153   }
 154
 155   DecayingEwma(const DecayingEwma& orig) : d_last(orig.d_last),  d_lastget(orig.d_lastget), d_val(orig.d_val), d_needinit(orig.d_needinit)
 156   {
 157   }
 158
 159   void submit(int val, const struct timeval* tv)
 160   {
 161     struct timeval now=*tv;
 162
 163     if(d_needinit) {
 164       d_last=now;
 165       d_lastget=now;
 166       d_needinit=false;
 167       d_val = val;
 168     }
 169     else {
 170       float diff= makeFloat(d_last - now);
 171
 172       d_last=now;
 173       double factor=exp(diff)/2.0; // might be '0.5', or 0.0001
 174       d_val=(float)((1-factor)*val+ (float)factor*d_val);
 175     }
 176   }
 177
 178   double get(const struct timeval* tv)
 179   {
 180     struct timeval now=*tv;
 181     float diff=makeFloat(d_lastget-now);
 182     d_lastget=now;
 183     float factor=exp(diff/60.0f); // is 1.0 or less
 184     return d_val*=factor;
 185   }
 186
 187   double peek(void) const
 188   {
 189     return d_val;
 190   }
 191
 192   bool stale(time_t limit) const
 193   {
 194     return limit > d_lastget.tv_sec;
 195   }
 196
 197 private:
 198   struct timeval d_last;          // stores time
 199   struct timeval d_lastget;       // stores time
 200   float d_val;
 201   bool d_needinit;
 202 };
 203
 204 template<class Thing> class Counters : public boost::noncopyable
 205 {
 206 public:
 207   Counters()
 208   {
 209   }
 210   unsigned long value(const Thing& t) const
 211   {
 212     typename cont_t::const_iterator i=d_cont.find(t);
 213
 214     if(i==d_cont.end()) {
 215       return 0;
 216     }
 217     return (unsigned long)i->second;
 218   }
 219   unsigned long incr(const Thing& t)
 220   {
 221     typename cont_t::iterator i=d_cont.find(t);
 222
 223     if(i==d_cont.end()) {
 224       d_cont[t]=1;
 225       return 1;
 226     }
 227     else {
 228       if (i->second < std::numeric_limits<unsigned long>::max())
 229         i->second++;
 230       return (unsigned long)i->second;
 231    }
 232   }
 233   unsigned long decr(const Thing& t)
 234   {
 235     typename cont_t::iterator i=d_cont.find(t);
 236
 237     if(i!=d_cont.end() && --i->second == 0) {
 238       d_cont.erase(i);
 239       return 0;
 240     } else
 241       return (unsigned long)i->second;
 242   }
 243   void clear(const Thing& t)
 244   {
 245     typename cont_t::iterator i=d_cont.find(t);
 246
 247     if(i!=d_cont.end()) {
 248       d_cont.erase(i);
 249     }
 250   }
 251   void clear()
 252   {
 253     d_cont.clear();
 254   }
 255   size_t size() const
 256   {
 257     return d_cont.size();
 258   }
 259 private:
 260   typedef map<Thing,unsigned long> cont_t;
 261   cont_t d_cont;
 262 };
 263
 264
 265 class SyncRes : public boost::noncopyable
 266 {
 267 public:
 268   enum LogMode { LogNone, Log, Store};
 269   typedef std::function<int(const ComboAddress& ip, const DNSName& qdomain, int qtype, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional<Netmask>& srcmask, boost::optional<const ResolveContext&> context, std::shared_ptr<RemoteLogger> outgoingLogger, LWResult *lwr)> asyncresolve_t;
 270
 271   struct EDNSStatus
 272   {
 273     EDNSStatus() : mode(UNKNOWN), modeSetAt(0) {}
 274     enum EDNSMode { UNKNOWN=0, EDNSOK=1, EDNSIGNORANT=2, NOEDNS=3 } mode;
 275     time_t modeSetAt;
 276   };
 277
 278     //! This represents a number of decaying Ewmas, used to store performance per nameserver-name.
 279   /** Modelled to work mostly like the underlying DecayingEwma. After you've called get,
 280       d_best is filled out with the best address for this collection */
 281   struct DecayingEwmaCollection
 282   {
 283     void submit(const ComboAddress& remote, int usecs, const struct timeval* now)
 284     {
 285       collection_t::iterator pos;
 286       for(pos=d_collection.begin(); pos != d_collection.end(); ++pos)
 287         if(pos->first==remote)
 288           break;
 289       if(pos!=d_collection.end()) {
 290         pos->second.submit(usecs, now);
 291       }
 292       else {
 293         DecayingEwma de;
 294         de.submit(usecs, now);
 295         d_collection.push_back(make_pair(remote, de));
 296       }
 297     }
 298
 299     double get(const struct timeval* now)
 300     {
 301       if(d_collection.empty())
 302         return 0;
 303       double ret=std::numeric_limits<double>::max();
 304       double tmp;
 305       for(collection_t::iterator pos=d_collection.begin(); pos != d_collection.end(); ++pos) {
 306         if((tmp=pos->second.get(now)) < ret) {
 307           ret=tmp;
 308           d_best=pos->first;
 309         }
 310       }
 311
 312       return ret;
 313     }
 314
 315     bool stale(time_t limit) const
 316     {
 317       for(collection_t::const_iterator pos=d_collection.begin(); pos != d_collection.end(); ++pos)
 318         if(!pos->second.stale(limit))
 319           return false;
 320       return true;
 321     }
 322
 323     typedef vector<pair<ComboAddress, DecayingEwma> > collection_t;
 324     collection_t d_collection;
 325     ComboAddress d_best;
 326   };
 327
 328   typedef map<DNSName, DecayingEwmaCollection> nsspeeds_t;
 329   typedef map<ComboAddress, EDNSStatus> ednsstatus_t;
 330
 331   class AuthDomain
 332   {
 333   public:
 334     typedef multi_index_container <
 335       DNSRecord,
 336       indexed_by <
 337         ordered_non_unique<
 338           composite_key< DNSRecord,
 339                          member<DNSRecord, DNSName, &DNSRecord::d_name>,
 340                          member<DNSRecord, uint16_t, &DNSRecord::d_type>
 341                        >,
 342           composite_key_compare<std::less<DNSName>, std::less<uint16_t> >
 343         >
 344       >
 345     > records_t;
 346
 347     records_t d_records;
 348     vector<ComboAddress> d_servers;
 349     DNSName d_name;
 350     bool d_rdForward{false};
 351
 352     int getRecords(const DNSName& qname, uint16_t qtype, std::vector<DNSRecord>& records) const;
 353     bool isAuth() const
 354     {
 355       return d_servers.empty();
 356     }
 357     bool isForward() const
 358     {
 359       return !isAuth();
 360     }
 361     bool shouldRecurse() const
 362     {
 363       return d_rdForward;
 364     }
 365     const DNSName& getName() const
 366     {
 367       return d_name;
 368     }
 369
 370   private:
 371     void addSOA(std::vector<DNSRecord>& records) const;
 372   };
 373
 374   typedef map<DNSName, AuthDomain> domainmap_t;
 375   typedef Throttle<boost::tuple<ComboAddress,DNSName,uint16_t> > throttle_t;
 376   typedef Counters<ComboAddress> fails_t;
 377
 378   struct ThreadLocalStorage {
 379     NegCache negcache;
 380     nsspeeds_t nsSpeeds;
 381     throttle_t throttle;
 382     ednsstatus_t ednsstatus;
 383     fails_t fails;
 384     std::shared_ptr<domainmap_t> domainmap;
 385   };
 386
 387   static void setDefaultLogMode(LogMode lm)
 388   {
 389     s_lm = lm;
 390   }
 391   static void doEDNSDumpAndClose(int fd);
 392   static uint64_t doDumpNSSpeeds(int fd);
 393   static int getRootNS(struct timeval now, asyncresolve_t asyncCallback);
 394   static void clearDelegationOnly()
 395   {
 396     s_delegationOnly.clear();
 397   }
 398   static void addDelegationOnly(const DNSName& name)
 399   {
 400     s_delegationOnly.insert(name);
 401   }
 402   static void addDontQuery(const std::string& mask)
 403   {
 404     if (!s_dontQuery)
 405       s_dontQuery = std::unique_ptr<NetmaskGroup>(new NetmaskGroup());
 406
 407     s_dontQuery->addMask(mask);
 408   }
 409   static void addDontQuery(const Netmask& mask)
 410   {
 411     if (!s_dontQuery)
 412       s_dontQuery = std::unique_ptr<NetmaskGroup>(new NetmaskGroup());
 413
 414     s_dontQuery->addMask(mask);
 415   }
 416   static void clearDontQuery()
 417   {
 418     s_dontQuery = nullptr;
 419   }
 420   static void parseEDNSSubnetWhitelist(const std::string& wlist);
 421   static void addEDNSSubnet(const Netmask& subnet)
 422   {
 423     s_ednssubnets.addMask(subnet);
 424   }
 425   static void addEDNSDomain(const DNSName& domain)
 426   {
 427     s_ednsdomains.add(domain);
 428   }
 429   static void clearEDNSSubnets()
 430   {
 431     s_ednssubnets.clear();
 432   }
 433   static void clearEDNSDomains()
 434   {
 435     s_ednsdomains = SuffixMatchNode();
 436   }
 437   static void pruneNSSpeeds(time_t limit)
 438   {
 439     for(auto i = t_sstorage.nsSpeeds.begin(), end = t_sstorage.nsSpeeds.end(); i != end; ) {
 440       if(i->second.stale(limit)) {
 441         i = t_sstorage.nsSpeeds.erase(i);
 442       }
 443       else {
 444         ++i;
 445       }
 446     }
 447   }
 448   static uint64_t getNSSpeedsSize()
 449   {
 450     return t_sstorage.nsSpeeds.size();
 451   }
 452   static void submitNSSpeed(const DNSName& server, const ComboAddress& ca, uint32_t usec, const struct timeval* now)
 453   {
 454     t_sstorage.nsSpeeds[server].submit(ca, usec, now);
 455   }
 456   static void clearNSSpeeds()
 457   {
 458     t_sstorage.nsSpeeds.clear();
 459   }
 460   static EDNSStatus::EDNSMode getEDNSStatus(const ComboAddress& server)
 461   {
 462     const auto& it = t_sstorage.ednsstatus.find(server);
 463     if (it == t_sstorage.ednsstatus.end())
 464       return EDNSStatus::UNKNOWN;
 465
 466     return it->second.mode;
 467   }
 468   static uint64_t getEDNSStatusesSize()
 469   {
 470     return t_sstorage.ednsstatus.size();
 471   }
 472   static void clearEDNSStatuses()
 473   {
 474     t_sstorage.ednsstatus.clear();
 475   }
 476   static uint64_t getThrottledServersSize()
 477   {
 478     return t_sstorage.throttle.size();
 479   }
 480   static void clearThrottle()
 481   {
 482     t_sstorage.throttle.clear();
 483   }
 484   static bool isThrottled(time_t now, const ComboAddress& server, const DNSName& target, uint16_t qtype)
 485   {
 486     return t_sstorage.throttle.shouldThrottle(now, boost::make_tuple(server, target, qtype));
 487   }
 488   static bool isThrottled(time_t now, const ComboAddress& server)
 489   {
 490     return t_sstorage.throttle.shouldThrottle(now, boost::make_tuple(server, "", 0));
 491   }
 492   static void doThrottle(time_t now, const ComboAddress& server, time_t duration, unsigned int tries)
 493   {
 494     t_sstorage.throttle.throttle(now, boost::make_tuple(server, "", 0), duration, tries);
 495   }
 496   static uint64_t getFailedServersSize()
 497   {
 498     return t_sstorage.fails.size();
 499   }
 500   static void clearFailedServers()
 501   {
 502     t_sstorage.fails.clear();
 503   }
 504   static unsigned long getServerFailsCount(const ComboAddress& server)
 505   {
 506     return t_sstorage.fails.value(server);
 507   }
 508
 509   static void clearNegCache()
 510   {
 511     t_sstorage.negcache.clear();
 512   }
 513
 514   static uint64_t getNegCacheSize()
 515   {
 516     return t_sstorage.negcache.size();
 517   }
 518
 519   static void pruneNegCache(unsigned int maxEntries)
 520   {
 521     t_sstorage.negcache.prune(maxEntries);
 522   }
 523
 524   static uint64_t wipeNegCache(const DNSName& name, bool subtree = false)
 525   {
 526     return t_sstorage.negcache.wipe(name, subtree);
 527   }
 528
 529   static void setDomainMap(std::shared_ptr<domainmap_t> newMap)
 530   {
 531     t_sstorage.domainmap = newMap;
 532   }
 533
 534   static const std::shared_ptr<domainmap_t> getDomainMap()
 535   {
 536     return t_sstorage.domainmap;
 537   }
 538
 539   explicit SyncRes(const struct timeval& now);
 540
 541   int beginResolve(const DNSName &qname, const QType &qtype, uint16_t qclass, vector<DNSRecord>&ret);
 542   void setId(int id)
 543   {
 544     if(doLog())
 545       d_prefix="["+itoa(id)+"] ";
 546   }
 547
 548   void setLogMode(LogMode lm)
 549   {
 550     d_lm = lm;
 551   }
 552
 553   bool doLog() const
 554   {
 555     return d_lm != LogNone;
 556   }
 557
 558   void setCacheOnly(bool state=true)
 559   {
 560     d_cacheonly=state;
 561   }
 562
 563   void setDoEDNS0(bool state=true)
 564   {
 565     d_doEDNS0=state;
 566   }
 567
 568   void setDoDNSSEC(bool state=true)
 569   {
 570     d_doDNSSEC=state;
 571   }
 572
 573   void setWantsRPZ(bool state=true)
 574   {
 575     d_wantsRPZ=state;
 576   }
 577
 578   bool getWantsRPZ() const
 579   {
 580     return d_wantsRPZ;
 581   }
 582
 583   void setIncomingECSFound(bool state=true)
 584   {
 585     d_incomingECSFound=state;
 586   }
 587
 588   string getTrace() const
 589   {
 590     return d_trace.str();
 591   }
 592
 593   void setLuaEngine(shared_ptr<RecursorLua4> pdl)
 594   {
 595     d_pdl = pdl;
 596   }
 597
 598   bool wasVariable() const
 599   {
 600     return d_wasVariable;
 601   }
 602
 603   bool wasOutOfBand() const
 604   {
 605     return d_wasOutOfBand;
 606   }
 607
 608   struct timeval getNow() const
 609   {
 610     return d_now;
 611   }
 612
 613   void setSkipCNAMECheck(bool skip = false)
 614   {
 615     d_skipCNAMECheck = skip;
 616   }
 617
 618   void setIncomingECS(boost::optional<const EDNSSubnetOpts&> incomingECS)
 619   {
 620     d_incomingECS = incomingECS;
 621     if (incomingECS) {
 622       d_incomingECSNetwork = incomingECS->source.getMaskedNetwork();
 623     }
 624     else {
 625       d_incomingECSNetwork = ComboAddress();
 626     }
 627   }
 628
 629 #ifdef HAVE_PROTOBUF
 630   void setInitialRequestId(boost::optional<const boost::uuids::uuid&> initialRequestId)
 631   {
 632     d_initialRequestId = initialRequestId;
 633   }
 634 #endif
 635
 636   void setAsyncCallback(asyncresolve_t func)
 637   {
 638     d_asyncResolve = func;
 639   }
 640
 641   vState getValidationState() const
 642   {
 643     return d_queryValidationState;
 644   }
 645
 646   static thread_local ThreadLocalStorage t_sstorage;
 647
 648   static std::atomic<uint64_t> s_queries;
 649   static std::atomic<uint64_t> s_outgoingtimeouts;
 650   static std::atomic<uint64_t> s_outgoing4timeouts;
 651   static std::atomic<uint64_t> s_outgoing6timeouts;
 652   static std::atomic<uint64_t> s_throttledqueries;
 653   static std::atomic<uint64_t> s_dontqueries;
 654   static std::atomic<uint64_t> s_outqueries;
 655   static std::atomic<uint64_t> s_tcpoutqueries;
 656   static std::atomic<uint64_t> s_nodelegated;
 657   static std::atomic<uint64_t> s_unreachables;
 658   static std::atomic<uint64_t> s_ecsqueries;
 659   static std::atomic<uint64_t> s_ecsresponses;
 660
 661   static string s_serverID;
 662   static unsigned int s_minimumTTL;
 663   static unsigned int s_maxqperq;
 664   static unsigned int s_maxtotusec;
 665   static unsigned int s_maxdepth;
 666   static unsigned int s_maxnegttl;
 667   static unsigned int s_maxcachettl;
 668   static unsigned int s_packetcachettl;
 669   static unsigned int s_packetcacheservfailttl;
 670   static unsigned int s_serverdownmaxfails;
 671   static unsigned int s_serverdownthrottletime;
 672   static uint8_t s_ecsipv4limit;
 673   static uint8_t s_ecsipv6limit;
 674   static bool s_doIPv6;
 675   static bool s_noEDNSPing;
 676   static bool s_noEDNS;
 677   static bool s_rootNXTrust;
 678   static bool s_nopacketcache;
 679
 680   std::unordered_map<std::string,bool> d_discardedPolicies;
 681   DNSFilterEngine::Policy d_appliedPolicy;
 682   unsigned int d_outqueries;
 683   unsigned int d_tcpoutqueries;
 684   unsigned int d_throttledqueries;
 685   unsigned int d_timeouts;
 686   unsigned int d_unreachables;
 687   unsigned int d_totUsec;
 688   ComboAddress d_requestor;
 689
 690 private:
 691
 692   static std::unordered_set<DNSName> s_delegationOnly;
 693   static NetmaskGroup s_ednssubnets;
 694   static SuffixMatchNode s_ednsdomains;
 695   static LogMode s_lm;
 696   static std::unique_ptr<NetmaskGroup> s_dontQuery;
 697
 698   struct GetBestNSAnswer
 699   {
 700     DNSName qname;
 701     set<pair<DNSName,DNSName> > bestns;
 702     uint8_t qtype; // only A and AAAA anyhow
 703     bool operator<(const GetBestNSAnswer &b) const
 704     {
 705       return boost::tie(qname, qtype, bestns) <
 706         boost::tie(b.qname, b.qtype, b.bestns);
 707     }
 708   };
 709
 710   typedef std::map<DNSName,vState> zonesStates_t;
 711
 712   int doResolveAt(NsSet &nameservers, DNSName auth, bool flawedNSSet, const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret,
 713                   unsigned int depth, set<GetBestNSAnswer>&beenthere, vState& state);
 714   bool doResolveAtThisIP(const std::string& prefix, const DNSName& qname, const QType& qtype, LWResult& lwr, boost::optional<Netmask>& ednsmask, const DNSName& auth, bool const sendRDQuery, const DNSName& nsName, const ComboAddress& remoteIP, bool doTCP, bool* truncated);
 715   bool processAnswer(unsigned int depth, LWResult& lwr, const DNSName& qname, const QType& qtype, DNSName& auth, bool wasForwarded, const boost::optional<Netmask> ednsmask, bool sendRDQuery, NsSet &nameservers, std::vector<DNSRecord>& ret, const DNSFilterEngine& dfe, bool* gotNewServers, int* rcode, vState& state);
 716
 717   int doResolve(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, set<GetBestNSAnswer>& beenthere, vState& state);
 718   bool doOOBResolve(const AuthDomain& domain, const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, int& res) const;
 719   bool doOOBResolve(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, int &res);
 720   domainmap_t::const_iterator getBestAuthZone(DNSName* qname) const;
 721   bool doCNAMECacheCheck(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, int &res, vState& state);
 722   bool doCacheCheck(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, int &res, vState& state);
 723   void getBestNSFromCache(const DNSName &qname, const QType &qtype, vector<DNSRecord>&bestns, bool* flawedNSSet, unsigned int depth, set<GetBestNSAnswer>& beenthere);
 724   DNSName getBestNSNamesFromCache(const DNSName &qname, const QType &qtype, NsSet& nsset, bool* flawedNSSet, unsigned int depth, set<GetBestNSAnswer>&beenthere);
 725
 726   inline vector<DNSName> shuffleInSpeedOrder(NsSet &nameservers, const string &prefix);
 727   bool moreSpecificThan(const DNSName& a, const DNSName &b) const;
 728   vector<ComboAddress> getAddrs(const DNSName &qname, unsigned int depth, set<GetBestNSAnswer>& beenthere);
 729
 730   bool nameserversBlockedByRPZ(const DNSFilterEngine& dfe, const NsSet& nameservers);
 731   bool nameserverIPBlockedByRPZ(const DNSFilterEngine& dfe, const ComboAddress&);
 732   bool throttledOrBlocked(const std::string& prefix, const ComboAddress& remoteIP, const DNSName& qname, const QType& qtype, bool pierceDontQuery);
 733
 734   vector<ComboAddress> retrieveAddressesForNS(const std::string& prefix, const DNSName& qname, vector<DNSName >::const_iterator& tns, const unsigned int depth, set<GetBestNSAnswer>& beenthere, const vector<DNSName >& rnameservers, NsSet& nameservers, bool& sendRDQuery, bool& pierceDontQuery, bool& flawedNSSet);
 735   RCode::rcodes_ updateCacheFromRecords(unsigned int depth, LWResult& lwr, const DNSName& qname, const QType& qtype, const DNSName& auth, bool wasForwarded, const boost::optional<Netmask>, vState& state, bool& needWildcardProof);
 736   bool processRecords(const std::string& prefix, const DNSName& qname, const QType& qtype, const DNSName& auth, LWResult& lwr, const bool sendRDQuery, vector<DNSRecord>& ret, set<DNSName>& nsset, DNSName& newtarget, DNSName& newauth, bool& realreferral, bool& negindic, vState& state, bool needWildcardProof);
 737
 738   bool doSpecialNamesResolve(const DNSName &qname, const QType &qtype, const uint16_t qclass, vector<DNSRecord> &ret);
 739
 740   int asyncresolveWrapper(const ComboAddress& ip, bool ednsMANDATORY, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, struct timeval* now, boost::optional<Netmask>& srcmask, LWResult* res) const;
 741
 742   boost::optional<Netmask> getEDNSSubnetMask(const ComboAddress& local, const DNSName&dn, const ComboAddress& rem);
 743
 744   bool validationEnabled() const;
 745   uint32_t computeLowestTTD(const std::vector<DNSRecord>& records, const std::vector<std::shared_ptr<RRSIGRecordContent> >& signatures, uint32_t signaturesTTL) const;
 746   void updateValidationState(vState& state, const vState stateUpdate);
 747   vState validateRecordsWithSigs(unsigned int depth, const DNSName& qname, const QType& qtype, const DNSName& name, const std::vector<DNSRecord>& records, const std::vector<std::shared_ptr<RRSIGRecordContent> >& signatures);
 748   vState validateDNSKeys(const DNSName& zone, const std::vector<DNSRecord>& dnskeys, const std::vector<std::shared_ptr<RRSIGRecordContent> >& signatures, unsigned int depth);
 749   vState getDSRecords(const DNSName& zone, dsmap_t& ds, bool onlyTA, unsigned int depth);
 750   vState getDNSKeys(const DNSName& signer, skeyset_t& keys, unsigned int depth);
 751   void getDenialValidationState(NegCache::NegCacheEntry& ne, vState& state, const dState expectedState, bool allowOptOut);
 752   vState getTA(const DNSName& zone, dsmap_t& ds);
 753   bool haveExactValidationStatus(const DNSName& domain);
 754   vState getValidationStatus(const DNSName& subdomain);
 755
 756   void computeZoneCuts(const DNSName& begin, const DNSName& end, unsigned int depth);
 757
 758   void setUpdatingRootNS()
 759   {
 760     d_updatingRootNS = true;
 761   }
 762
 763   zonesStates_t d_cutStates;
 764   ostringstream d_trace;
 765   shared_ptr<RecursorLua4> d_pdl;
 766   boost::optional<const EDNSSubnetOpts&> d_incomingECS;
 767   ComboAddress d_incomingECSNetwork;
 768 #ifdef HAVE_PROTOBUF
 769   boost::optional<const boost::uuids::uuid&> d_initialRequestId;
 770 #endif
 771   asyncresolve_t d_asyncResolve{nullptr};
 772   struct timeval d_now;
 773   string d_prefix;
 774   vState d_queryValidationState{Indeterminate};
 775
 776   /* When d_cacheonly is set to true, we will only check the cache.
 777    * This is set when the RD bit is unset in the incoming query
 778    */
 779   bool d_cacheonly;
 780   bool d_doDNSSEC;
 781   bool d_doEDNS0{true};
 782   bool d_incomingECSFound{false};
 783   bool d_requireAuthData{true};
 784   bool d_skipCNAMECheck{false};
 785   bool d_updatingRootNS{false};
 786   bool d_wantsRPZ{true};
 787   bool d_wasOutOfBand{false};
 788   bool d_wasVariable{false};
 789
 790   LogMode d_lm;
 791 };
 792
 793 class Socket;
 794 /* external functions, opaque to us */
 795 int asendtcp(const string& data, Socket* sock);
 796 int arecvtcp(string& data, size_t len, Socket* sock, bool incompleteOkay);
 797
 798
 799 struct PacketID
 800 {
 801   PacketID() : id(0), type(0), sock(0), inNeeded(0), inIncompleteOkay(false), outPos(0), nearMisses(0), fd(-1)
 802   {
 803     memset(&remote, 0, sizeof(remote));
 804   }
 805
 806   uint16_t id;  // wait for a specific id/remote pair
 807   uint16_t type;             // and this is its type
 808   ComboAddress remote;  // this is the remote
 809   DNSName domain;             // this is the question
 810
 811   Socket* sock;  // or wait for an event on a TCP fd
 812   string inMSG; // they'll go here
 813   size_t inNeeded; // if this is set, we'll read until inNeeded bytes are read
 814   bool inIncompleteOkay;
 815
 816   string outMSG; // the outgoing message that needs to be sent
 817   string::size_type outPos;    // how far we are along in the outMSG
 818
 819   typedef set<uint16_t > chain_t;
 820   mutable chain_t chain;
 821   mutable uint32_t nearMisses; // number of near misses - host correct, id wrong
 822   int fd;
 823
 824   bool operator<(const PacketID& b) const
 825   {
 826     int ourSock= sock ? sock->getHandle() : 0;
 827     int bSock = b.sock ? b.sock->getHandle() : 0;
 828     if( tie(remote, ourSock, type) < tie(b.remote, bSock, b.type))
 829       return true;
 830     if( tie(remote, ourSock, type) > tie(b.remote, bSock, b.type))
 831       return false;
 832
 833     return tie(domain, fd, id) < tie(b.domain, b.fd, b.id);
 834   }
 835 };
 836
 837 struct PacketIDBirthdayCompare: public std::binary_function<PacketID, PacketID, bool>
 838 {
 839   bool operator()(const PacketID& a, const PacketID& b) const
 840   {
 841     int ourSock= a.sock ? a.sock->getHandle() : 0;
 842     int bSock = b.sock ? b.sock->getHandle() : 0;
 843     if( tie(a.remote, ourSock, a.type) < tie(b.remote, bSock, b.type))
 844       return true;
 845     if( tie(a.remote, ourSock, a.type) > tie(b.remote, bSock, b.type))
 846       return false;
 847
 848     return a.domain < b.domain;
 849   }
 850 };
 851 extern thread_local std::unique_ptr<MemRecursorCache> t_RC;
 852 extern thread_local std::unique_ptr<RecursorPacketCache> t_packetCache;
 853 typedef MTasker<PacketID,string> MT_t;
 854 MT_t* getMT();
 855
 856 struct RecursorStats
 857 {
 858   std::atomic<uint64_t> servFails;
 859   std::atomic<uint64_t> nxDomains;
 860   std::atomic<uint64_t> noErrors;
 861   std::atomic<uint64_t> answers0_1, answers1_10, answers10_100, answers100_1000, answersSlow;
 862   std::atomic<uint64_t> auth4Answers0_1, auth4Answers1_10, auth4Answers10_100, auth4Answers100_1000, auth4AnswersSlow;
 863   std::atomic<uint64_t> auth6Answers0_1, auth6Answers1_10, auth6Answers10_100, auth6Answers100_1000, auth6AnswersSlow;
 864   double avgLatencyUsec;
 865   std::atomic<uint64_t> qcounter;     // not increased for unauth packets
 866   std::atomic<uint64_t> ipv6qcounter;
 867   std::atomic<uint64_t> tcpqcounter;
 868   std::atomic<uint64_t> unauthorizedUDP;  // when this is increased, qcounter isn't
 869   std::atomic<uint64_t> unauthorizedTCP;  // when this is increased, qcounter isn't
 870   std::atomic<uint64_t> policyDrops;
 871   std::atomic<uint64_t> tcpClientOverflow;
 872   std::atomic<uint64_t> clientParseError;
 873   std::atomic<uint64_t> serverParseError;
 874   std::atomic<uint64_t> tooOldDrops;
 875   std::atomic<uint64_t> unexpectedCount;
 876   std::atomic<uint64_t> caseMismatchCount;
 877   std::atomic<uint64_t> spoofCount;
 878   std::atomic<uint64_t> resourceLimits;
 879   std::atomic<uint64_t> overCapacityDrops;
 880   std::atomic<uint64_t> ipv6queries;
 881   std::atomic<uint64_t> chainResends;
 882   std::atomic<uint64_t> nsSetInvalidations;
 883   std::atomic<uint64_t> ednsPingMatches;
 884   std::atomic<uint64_t> ednsPingMismatches;
 885   std::atomic<uint64_t> noPingOutQueries, noEdnsOutQueries;
 886   std::atomic<uint64_t> packetCacheHits;
 887   std::atomic<uint64_t> noPacketError;
 888   std::atomic<uint64_t> ignoredCount;
 889   time_t startupTime;
 890   std::atomic<uint64_t> dnssecQueries;
 891   unsigned int maxMThreadStackUsage;
 892   std::atomic<uint64_t> dnssecValidations; // should be the sum of all dnssecResult* stats
 893   std::map<vState, std::atomic<uint64_t> > dnssecResults;
 894   std::map<DNSFilterEngine::PolicyKind, std::atomic<uint64_t> > policyResults;
 895 };
 896
 897 //! represents a running TCP/IP client session
 898 class TCPConnection : public boost::noncopyable
 899 {
 900 public:
 901   TCPConnection(int fd, const ComboAddress& addr);
 902   ~TCPConnection();
 903
 904   int getFD() const
 905   {
 906     return d_fd;
 907   }
 908   enum stateenum {BYTE0, BYTE1, GETQUESTION, DONE} state{BYTE0};
 909   uint16_t qlen{0};
 910   uint16_t bytesread{0};
 911   const ComboAddress d_remote;
 912   char data[65535]; // damn
 913   size_t queriesCount{0};
 914
 915   static unsigned int getCurrentConnections() { return s_currentConnections; }
 916 private:
 917   const int d_fd;
 918   static AtomicCounter s_currentConnections; //!< total number of current TCP connections
 919 };
 920
 921 class ImmediateServFailException
 922 {
 923 public:
 924   ImmediateServFailException(string r){reason=r;};
 925
 926   string reason; //! Print this to tell the user what went wrong
 927 };
 928
 929 #if (__GNUC__ == 4 && __GNUC_MINOR__ == 2)
 930 typedef boost::circular_buffer<SComboAddress> addrringbuf_t;
 931 #else
 932 typedef boost::circular_buffer<ComboAddress> addrringbuf_t;
 933 #endif
 934 extern thread_local std::unique_ptr<addrringbuf_t> t_servfailremotes, t_largeanswerremotes, t_remotes;
 935
 936 extern thread_local std::unique_ptr<boost::circular_buffer<pair<DNSName,uint16_t> > > t_queryring, t_servfailqueryring;
 937 extern thread_local std::shared_ptr<NetmaskGroup> t_allowFrom;
 938 string doQueueReloadLuaScript(vector<string>::const_iterator begin, vector<string>::const_iterator end);
 939 string doTraceRegex(vector<string>::const_iterator begin, vector<string>::const_iterator end);
 940 void parseACLs();
 941 extern RecursorStats g_stats;
 942 extern unsigned int g_numThreads;
 943 extern uint16_t g_outgoingEDNSBufsize;
 944
 945
 946 std::string reloadAuthAndForwards();
 947 ComboAddress parseIPAndPort(const std::string& input, uint16_t port);
 948 ComboAddress getQueryLocalAddress(int family, uint16_t port);
 949 typedef boost::function<void*(void)> pipefunc_t;
 950 void broadcastFunction(const pipefunc_t& func, bool skipSelf = false);
 951 void distributeAsyncFunction(const std::string& question, const pipefunc_t& func);
 952
 953 int directResolve(const DNSName& qname, const QType& qtype, int qclass, vector<DNSRecord>& ret);
 954
 955 template<class T> T broadcastAccFunction(const boost::function<T*()>& func, bool skipSelf=false);
 956
 957 std::shared_ptr<SyncRes::domainmap_t> parseAuthAndForwards();
 958 uint64_t* pleaseGetNsSpeedsSize();
 959 uint64_t* pleaseGetCacheSize();
 960 uint64_t* pleaseGetNegCacheSize();
 961 uint64_t* pleaseGetCacheHits();
 962 uint64_t* pleaseGetCacheMisses();
 963 uint64_t* pleaseGetConcurrentQueries();
 964 uint64_t* pleaseGetThrottleSize();
 965 uint64_t* pleaseGetPacketCacheHits();
 966 uint64_t* pleaseGetPacketCacheSize();
 967 uint64_t* pleaseWipeCache(const DNSName& canon, bool subtree=false);
 968 uint64_t* pleaseWipePacketCache(const DNSName& canon, bool subtree);
 969 uint64_t* pleaseWipeAndCountNegCache(const DNSName& canon, bool subtree=false);
 970 void doCarbonDump(void*);
 971 void primeHints(void);
 972
 973 extern __thread struct timeval g_now;
 974
 975 #ifdef HAVE_PROTOBUF
 976 extern thread_local std::unique_ptr<boost::uuids::random_generator> t_uuidGenerator;
 977 #endif