pdns/syncres.hh

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #pragma once
  23 #include <string>
  24 #include <atomic>
  25 #include "utility.hh"
  26 #include "dns.hh"
  27 #include "qtype.hh"
  28 #include <vector>
  29 #include <set>
  30 #include <unordered_set>
  31 #include <map>
  32 #include <cmath>
  33 #include <iostream>
  34 #include <utility>
  35 #include "misc.hh"
  36 #include "lwres.hh"
  37 #include <boost/optional.hpp>
  38 #include <boost/circular_buffer.hpp>
  39 #include <boost/utility.hpp>
  40 #include "sstuff.hh"
  41 #include "recursor_cache.hh"
  42 #include "recpacketcache.hh"
  43 #include <boost/tuple/tuple.hpp>
  44 #include <boost/optional.hpp>
  45 #include <boost/tuple/tuple_comparison.hpp>
  46 #include "mtasker.hh"
  47 #include "iputils.hh"
  48 #include "validate.hh"
  49 #include "ednssubnet.hh"
  50 #include "filterpo.hh"
  51 #include "negcache.hh"
  52
  53 #ifdef HAVE_CONFIG_H
  54 #include "config.h"
  55 #endif
  56
  57 #ifdef HAVE_PROTOBUF
  58 #include <boost/uuid/uuid.hpp>
  59 #endif
  60
  61 class RecursorLua4;
  62
  63 typedef map<
  64   DNSName,
  65   pair<
  66     vector<ComboAddress>,
  67     bool
  68   >
  69 > NsSet;
  70
  71 template<class Thing> class Throttle : public boost::noncopyable
  72 {
  73 public:
  74   Throttle() : d_limit(3), d_ttl(60), d_last_clean(time(nullptr))
  75   {
  76   }
  77
  78   struct entry
  79   {
  80     time_t ttd;
  81     unsigned int count;
  82   };
  83   typedef map<Thing,entry> cont_t;
  84
  85   bool shouldThrottle(time_t now, const Thing& t)
  86   {
  87     if(now > d_last_clean + 300 ) {
  88
  89       d_last_clean=now;
  90       for(typename cont_t::iterator i=d_cont.begin();i!=d_cont.end();) {
  91         if( i->second.ttd < now) {
  92           d_cont.erase(i++);
  93         }
  94         else
  95           ++i;
  96       }
  97     }
  98
  99     typename cont_t::iterator i=d_cont.find(t);
 100     if(i==d_cont.end())
 101       return false;
 102     if(now > i->second.ttd || i->second.count == 0) {
 103       d_cont.erase(i);
 104       return false;
 105     }
 106     i->second.count--;
 107
 108     return true; // still listed, still blocked
 109   }
 110   void throttle(time_t now, const Thing& t, time_t ttl=0, unsigned int tries=0)
 111   {
 112     typename cont_t::iterator i=d_cont.find(t);
 113     entry e={ now+(ttl ? ttl : d_ttl), tries ? tries : d_limit};
 114
 115     if(i==d_cont.end()) {
 116       d_cont[t]=e;
 117     }
 118     else if(i->second.ttd > e.ttd || (i->second.count) < e.count)
 119       d_cont[t]=e;
 120   }
 121
 122   unsigned int size() const
 123   {
 124     return (unsigned int)d_cont.size();
 125   }
 126
 127   const cont_t& getThrottleMap() const
 128   {
 129     return d_cont;
 130   }
 131
 132   void clear()
 133   {
 134     d_cont.clear();
 135   }
 136
 137 private:
 138   unsigned int d_limit;
 139   time_t d_ttl;
 140   time_t d_last_clean;
 141   cont_t d_cont;
 142 };
 143
 144
 145 /** Class that implements a decaying EWMA.
 146     This class keeps an exponentially weighted moving average which, additionally, decays over time.
 147     The decaying is only done on get.
 148 */
 149 class DecayingEwma
 150 {
 151 public:
 152   DecayingEwma() :  d_val(0.0)
 153   {
 154     d_needinit=true;
 155     d_last.tv_sec = d_last.tv_usec = 0;
 156     d_lastget=d_last;
 157   }
 158
 159   DecayingEwma(const DecayingEwma& orig) : d_last(orig.d_last),  d_lastget(orig.d_lastget), d_val(orig.d_val), d_needinit(orig.d_needinit)
 160   {
 161   }
 162
 163   void submit(int val, const struct timeval* tv)
 164   {
 165     struct timeval now=*tv;
 166
 167     if(d_needinit) {
 168       d_last=now;
 169       d_lastget=now;
 170       d_needinit=false;
 171       d_val = val;
 172     }
 173     else {
 174       float diff= makeFloat(d_last - now);
 175
 176       d_last=now;
 177       double factor=exp(diff)/2.0; // might be '0.5', or 0.0001
 178       d_val=(float)((1-factor)*val+ (float)factor*d_val);
 179     }
 180   }
 181
 182   double get(const struct timeval* tv)
 183   {
 184     struct timeval now=*tv;
 185     float diff=makeFloat(d_lastget-now);
 186     d_lastget=now;
 187     float factor=exp(diff/60.0f); // is 1.0 or less
 188     return d_val*=factor;
 189   }
 190
 191   double peek(void) const
 192   {
 193     return d_val;
 194   }
 195
 196   bool stale(time_t limit) const
 197   {
 198     return limit > d_lastget.tv_sec;
 199   }
 200
 201 private:
 202   struct timeval d_last;          // stores time
 203   struct timeval d_lastget;       // stores time
 204   float d_val;
 205   bool d_needinit;
 206 };
 207
 208 template<class Thing> class Counters : public boost::noncopyable
 209 {
 210 public:
 211   Counters()
 212   {
 213   }
 214   unsigned long value(const Thing& t) const
 215   {
 216     typename cont_t::const_iterator i=d_cont.find(t);
 217
 218     if(i==d_cont.end()) {
 219       return 0;
 220     }
 221     return (unsigned long)i->second;
 222   }
 223   unsigned long incr(const Thing& t)
 224   {
 225     typename cont_t::iterator i=d_cont.find(t);
 226
 227     if(i==d_cont.end()) {
 228       d_cont[t]=1;
 229       return 1;
 230     }
 231     else {
 232       if (i->second < std::numeric_limits<unsigned long>::max())
 233         i->second++;
 234       return (unsigned long)i->second;
 235    }
 236   }
 237   unsigned long decr(const Thing& t)
 238   {
 239     typename cont_t::iterator i=d_cont.find(t);
 240
 241     if(i!=d_cont.end() && --i->second == 0) {
 242       d_cont.erase(i);
 243       return 0;
 244     } else
 245       return (unsigned long)i->second;
 246   }
 247   void clear(const Thing& t)
 248   {
 249     typename cont_t::iterator i=d_cont.find(t);
 250
 251     if(i!=d_cont.end()) {
 252       d_cont.erase(i);
 253     }
 254   }
 255   void clear()
 256   {
 257     d_cont.clear();
 258   }
 259   size_t size() const
 260   {
 261     return d_cont.size();
 262   }
 263 private:
 264   typedef map<Thing,unsigned long> cont_t;
 265   cont_t d_cont;
 266 };
 267
 268
 269 class SyncRes : public boost::noncopyable
 270 {
 271 public:
 272   enum LogMode { LogNone, Log, Store};
 273   typedef std::function<int(const ComboAddress& ip, const DNSName& qdomain, int qtype, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional<Netmask>& srcmask, boost::optional<const ResolveContext&> context, LWResult *lwr, bool* chained)> asyncresolve_t;
 274
 275   struct EDNSStatus
 276   {
 277     EDNSStatus() : mode(UNKNOWN), modeSetAt(0) {}
 278     enum EDNSMode { UNKNOWN=0, EDNSOK=1, EDNSIGNORANT=2, NOEDNS=3 } mode;
 279     time_t modeSetAt;
 280   };
 281
 282     //! This represents a number of decaying Ewmas, used to store performance per nameserver-name.
 283   /** Modelled to work mostly like the underlying DecayingEwma. After you've called get,
 284       d_best is filled out with the best address for this collection */
 285   struct DecayingEwmaCollection
 286   {
 287     void submit(const ComboAddress& remote, int usecs, const struct timeval* now)
 288     {
 289       d_collection[remote].submit(usecs, now);
 290     }
 291
 292     double get(const struct timeval* now)
 293     {
 294       if(d_collection.empty())
 295         return 0;
 296       double ret=std::numeric_limits<double>::max();
 297       double tmp;
 298       for (auto& entry : d_collection) {
 299         if((tmp = entry.second.get(now)) < ret) {
 300           ret=tmp;
 301           d_best=entry.first;
 302         }
 303       }
 304
 305       return ret;
 306     }
 307
 308     bool stale(time_t limit) const
 309     {
 310       for(const auto& entry : d_collection)
 311         if(!entry.second.stale(limit))
 312           return false;
 313       return true;
 314     }
 315
 316     void purge(const std::map<ComboAddress, double>& keep)
 317     {
 318       for (auto iter = d_collection.begin(); iter != d_collection.end(); ) {
 319         if (keep.find(iter->first) != keep.end()) {
 320           ++iter;
 321         }
 322         else {
 323           iter = d_collection.erase(iter);
 324         }
 325       }
 326     }
 327
 328     typedef std::map<ComboAddress, DecayingEwma> collection_t;
 329     collection_t d_collection;
 330     ComboAddress d_best;
 331   };
 332
 333   typedef map<DNSName, DecayingEwmaCollection> nsspeeds_t;
 334   typedef map<ComboAddress, EDNSStatus> ednsstatus_t;
 335
 336   vState getDSRecords(const DNSName& zone, dsmap_t& ds, bool onlyTA, unsigned int depth, bool bogusOnNXD=true, bool* foundCut=nullptr);
 337
 338   class AuthDomain
 339   {
 340   public:
 341     typedef multi_index_container <
 342       DNSRecord,
 343       indexed_by <
 344         ordered_non_unique<
 345           composite_key< DNSRecord,
 346                          member<DNSRecord, DNSName, &DNSRecord::d_name>,
 347                          member<DNSRecord, uint16_t, &DNSRecord::d_type>
 348                        >,
 349           composite_key_compare<std::less<DNSName>, std::less<uint16_t> >
 350         >
 351       >
 352     > records_t;
 353
 354     records_t d_records;
 355     vector<ComboAddress> d_servers;
 356     DNSName d_name;
 357     bool d_rdForward{false};
 358
 359     int getRecords(const DNSName& qname, uint16_t qtype, std::vector<DNSRecord>& records) const;
 360     bool isAuth() const
 361     {
 362       return d_servers.empty();
 363     }
 364     bool isForward() const
 365     {
 366       return !isAuth();
 367     }
 368     bool shouldRecurse() const
 369     {
 370       return d_rdForward;
 371     }
 372     const DNSName& getName() const
 373     {
 374       return d_name;
 375     }
 376
 377   private:
 378     void addSOA(std::vector<DNSRecord>& records) const;
 379   };
 380
 381   typedef map<DNSName, AuthDomain> domainmap_t;
 382   typedef Throttle<boost::tuple<ComboAddress,DNSName,uint16_t> > throttle_t;
 383   typedef Counters<ComboAddress> fails_t;
 384
 385   struct ThreadLocalStorage {
 386     NegCache negcache;
 387     nsspeeds_t nsSpeeds;
 388     throttle_t throttle;
 389     ednsstatus_t ednsstatus;
 390     fails_t fails;
 391     std::shared_ptr<domainmap_t> domainmap;
 392   };
 393
 394   static void setDefaultLogMode(LogMode lm)
 395   {
 396     s_lm = lm;
 397   }
 398   static uint64_t doEDNSDump(int fd);
 399   static uint64_t doDumpNSSpeeds(int fd);
 400   static uint64_t doDumpThrottleMap(int fd);
 401   static int getRootNS(struct timeval now, asyncresolve_t asyncCallback);
 402   static void clearDelegationOnly()
 403   {
 404     s_delegationOnly.clear();
 405   }
 406   static void addDelegationOnly(const DNSName& name)
 407   {
 408     s_delegationOnly.insert(name);
 409   }
 410   static void addDontQuery(const std::string& mask)
 411   {
 412     if (!s_dontQuery)
 413       s_dontQuery = std::unique_ptr<NetmaskGroup>(new NetmaskGroup());
 414
 415     s_dontQuery->addMask(mask);
 416   }
 417   static void addDontQuery(const Netmask& mask)
 418   {
 419     if (!s_dontQuery)
 420       s_dontQuery = std::unique_ptr<NetmaskGroup>(new NetmaskGroup());
 421
 422     s_dontQuery->addMask(mask);
 423   }
 424   static void clearDontQuery()
 425   {
 426     s_dontQuery = nullptr;
 427   }
 428   static void parseEDNSSubnetWhitelist(const std::string& wlist);
 429   static void parseEDNSSubnetAddFor(const std::string& subnetlist);
 430   static void addEDNSLocalSubnet(const std::string& subnet)
 431   {
 432     s_ednslocalsubnets.addMask(subnet);
 433   }
 434   static void addEDNSRemoteSubnet(const std::string& subnet)
 435   {
 436     s_ednsremotesubnets.addMask(subnet);
 437   }
 438   static void addEDNSDomain(const DNSName& domain)
 439   {
 440     s_ednsdomains.add(domain);
 441   }
 442   static void clearEDNSLocalSubnets()
 443   {
 444     s_ednslocalsubnets.clear();
 445   }
 446   static void clearEDNSRemoteSubnets()
 447   {
 448     s_ednsremotesubnets.clear();
 449   }
 450   static void clearEDNSDomains()
 451   {
 452     s_ednsdomains = SuffixMatchNode();
 453   }
 454   static void pruneNSSpeeds(time_t limit)
 455   {
 456     for(auto i = t_sstorage.nsSpeeds.begin(), end = t_sstorage.nsSpeeds.end(); i != end; ) {
 457       if(i->second.stale(limit)) {
 458         i = t_sstorage.nsSpeeds.erase(i);
 459       }
 460       else {
 461         ++i;
 462       }
 463     }
 464   }
 465   static uint64_t getNSSpeedsSize()
 466   {
 467     return t_sstorage.nsSpeeds.size();
 468   }
 469   static void submitNSSpeed(const DNSName& server, const ComboAddress& ca, uint32_t usec, const struct timeval* now)
 470   {
 471     t_sstorage.nsSpeeds[server].submit(ca, usec, now);
 472   }
 473   static void clearNSSpeeds()
 474   {
 475     t_sstorage.nsSpeeds.clear();
 476   }
 477   static EDNSStatus::EDNSMode getEDNSStatus(const ComboAddress& server)
 478   {
 479     const auto& it = t_sstorage.ednsstatus.find(server);
 480     if (it == t_sstorage.ednsstatus.end())
 481       return EDNSStatus::UNKNOWN;
 482
 483     return it->second.mode;
 484   }
 485   static uint64_t getEDNSStatusesSize()
 486   {
 487     return t_sstorage.ednsstatus.size();
 488   }
 489   static void clearEDNSStatuses()
 490   {
 491     t_sstorage.ednsstatus.clear();
 492   }
 493   static uint64_t getThrottledServersSize()
 494   {
 495     return t_sstorage.throttle.size();
 496   }
 497   static void clearThrottle()
 498   {
 499     t_sstorage.throttle.clear();
 500   }
 501   static bool isThrottled(time_t now, const ComboAddress& server, const DNSName& target, uint16_t qtype)
 502   {
 503     return t_sstorage.throttle.shouldThrottle(now, boost::make_tuple(server, target, qtype));
 504   }
 505   static bool isThrottled(time_t now, const ComboAddress& server)
 506   {
 507     return t_sstorage.throttle.shouldThrottle(now, boost::make_tuple(server, "", 0));
 508   }
 509   static void doThrottle(time_t now, const ComboAddress& server, time_t duration, unsigned int tries)
 510   {
 511     t_sstorage.throttle.throttle(now, boost::make_tuple(server, "", 0), duration, tries);
 512   }
 513   static uint64_t getFailedServersSize()
 514   {
 515     return t_sstorage.fails.size();
 516   }
 517   static void clearFailedServers()
 518   {
 519     t_sstorage.fails.clear();
 520   }
 521   static unsigned long getServerFailsCount(const ComboAddress& server)
 522   {
 523     return t_sstorage.fails.value(server);
 524   }
 525
 526   static void clearNegCache()
 527   {
 528     t_sstorage.negcache.clear();
 529   }
 530
 531   static uint64_t getNegCacheSize()
 532   {
 533     return t_sstorage.negcache.size();
 534   }
 535
 536   static void pruneNegCache(unsigned int maxEntries)
 537   {
 538     t_sstorage.negcache.prune(maxEntries);
 539   }
 540
 541   static uint64_t wipeNegCache(const DNSName& name, bool subtree = false)
 542   {
 543     return t_sstorage.negcache.wipe(name, subtree);
 544   }
 545
 546   static void setDomainMap(std::shared_ptr<domainmap_t> newMap)
 547   {
 548     t_sstorage.domainmap = newMap;
 549   }
 550
 551   static const std::shared_ptr<domainmap_t> getDomainMap()
 552   {
 553     return t_sstorage.domainmap;
 554   }
 555
 556   static void setECSScopeZeroAddress(const Netmask& scopeZeroMask)
 557   {
 558     s_ecsScopeZero.source = scopeZeroMask;
 559   }
 560
 561   explicit SyncRes(const struct timeval& now);
 562
 563   int beginResolve(const DNSName &qname, const QType &qtype, uint16_t qclass, vector<DNSRecord>&ret);
 564   void setId(int id)
 565   {
 566     if(doLog())
 567       d_prefix="["+itoa(id)+"] ";
 568   }
 569
 570   void setLogMode(LogMode lm)
 571   {
 572     d_lm = lm;
 573   }
 574
 575   bool doLog() const
 576   {
 577     return d_lm != LogNone;
 578   }
 579
 580   void setCacheOnly(bool state=true)
 581   {
 582     d_cacheonly=state;
 583   }
 584
 585   void setDoEDNS0(bool state=true)
 586   {
 587     d_doEDNS0=state;
 588   }
 589
 590   void setDoDNSSEC(bool state=true)
 591   {
 592     d_doDNSSEC=state;
 593   }
 594
 595   void setDNSSECValidationRequested(bool requested=true)
 596   {
 597     d_DNSSECValidationRequested = requested;
 598   }
 599
 600   bool isDNSSECValidationRequested() const
 601   {
 602     return d_DNSSECValidationRequested;
 603   }
 604
 605   bool shouldValidate() const
 606   {
 607     return d_DNSSECValidationRequested && !d_wasOutOfBand;
 608   }
 609
 610   void setWantsRPZ(bool state=true)
 611   {
 612     d_wantsRPZ=state;
 613   }
 614
 615   bool getWantsRPZ() const
 616   {
 617     return d_wantsRPZ;
 618   }
 619
 620   string getTrace() const
 621   {
 622     return d_trace.str();
 623   }
 624
 625   void setLuaEngine(shared_ptr<RecursorLua4> pdl)
 626   {
 627     d_pdl = pdl;
 628   }
 629
 630   bool wasVariable() const
 631   {
 632     return d_wasVariable;
 633   }
 634
 635   bool wasOutOfBand() const
 636   {
 637     return d_wasOutOfBand;
 638   }
 639
 640   struct timeval getNow() const
 641   {
 642     return d_now;
 643   }
 644
 645   void setSkipCNAMECheck(bool skip = false)
 646   {
 647     d_skipCNAMECheck = skip;
 648   }
 649
 650   void setQuerySource(const ComboAddress& requestor, boost::optional<const EDNSSubnetOpts&> incomingECS);
 651
 652 #ifdef HAVE_PROTOBUF
 653   void setInitialRequestId(boost::optional<const boost::uuids::uuid&> initialRequestId)
 654   {
 655     d_initialRequestId = initialRequestId;
 656   }
 657
 658   void setOutgoingProtobufServers(std::shared_ptr<std::vector<std::unique_ptr<RemoteLogger>>>& servers)
 659   {
 660     d_outgoingProtobufServers = servers;
 661   }
 662 #endif
 663
 664   void setAsyncCallback(asyncresolve_t func)
 665   {
 666     d_asyncResolve = func;
 667   }
 668
 669   vState getValidationState() const
 670   {
 671     return d_queryValidationState;
 672   }
 673
 674   static thread_local ThreadLocalStorage t_sstorage;
 675
 676   static std::atomic<uint64_t> s_queries;
 677   static std::atomic<uint64_t> s_outgoingtimeouts;
 678   static std::atomic<uint64_t> s_outgoing4timeouts;
 679   static std::atomic<uint64_t> s_outgoing6timeouts;
 680   static std::atomic<uint64_t> s_throttledqueries;
 681   static std::atomic<uint64_t> s_dontqueries;
 682   static std::atomic<uint64_t> s_authzonequeries;
 683   static std::atomic<uint64_t> s_outqueries;
 684   static std::atomic<uint64_t> s_tcpoutqueries;
 685   static std::atomic<uint64_t> s_nodelegated;
 686   static std::atomic<uint64_t> s_unreachables;
 687   static std::atomic<uint64_t> s_ecsqueries;
 688   static std::atomic<uint64_t> s_ecsresponses;
 689
 690   static string s_serverID;
 691   static unsigned int s_minimumTTL;
 692   static unsigned int s_maxqperq;
 693   static unsigned int s_maxtotusec;
 694   static unsigned int s_maxdepth;
 695   static unsigned int s_maxnegttl;
 696   static unsigned int s_maxbogusttl;
 697   static unsigned int s_maxcachettl;
 698   static unsigned int s_packetcachettl;
 699   static unsigned int s_packetcacheservfailttl;
 700   static unsigned int s_serverdownmaxfails;
 701   static unsigned int s_serverdownthrottletime;
 702   static uint8_t s_ecsipv4limit;
 703   static uint8_t s_ecsipv6limit;
 704   static bool s_doIPv6;
 705   static bool s_noEDNSPing;
 706   static bool s_noEDNS;
 707   static bool s_rootNXTrust;
 708   static bool s_nopacketcache;
 709
 710   std::unordered_map<std::string,bool> d_discardedPolicies;
 711   DNSFilterEngine::Policy d_appliedPolicy;
 712   unsigned int d_authzonequeries;
 713   unsigned int d_outqueries;
 714   unsigned int d_tcpoutqueries;
 715   unsigned int d_throttledqueries;
 716   unsigned int d_timeouts;
 717   unsigned int d_unreachables;
 718   unsigned int d_totUsec;
 719
 720 private:
 721   ComboAddress d_requestor;
 722   ComboAddress d_cacheRemote;
 723
 724   static std::unordered_set<DNSName> s_delegationOnly;
 725   static NetmaskGroup s_ednslocalsubnets;
 726   static NetmaskGroup s_ednsremotesubnets;
 727   static SuffixMatchNode s_ednsdomains;
 728   static EDNSSubnetOpts s_ecsScopeZero;
 729   static LogMode s_lm;
 730   static std::unique_ptr<NetmaskGroup> s_dontQuery;
 731
 732   struct GetBestNSAnswer
 733   {
 734     DNSName qname;
 735     set<pair<DNSName,DNSName> > bestns;
 736     uint8_t qtype; // only A and AAAA anyhow
 737     bool operator<(const GetBestNSAnswer &b) const
 738     {
 739       return boost::tie(qname, qtype, bestns) <
 740         boost::tie(b.qname, b.qtype, b.bestns);
 741     }
 742   };
 743
 744   typedef std::map<DNSName,vState> zonesStates_t;
 745
 746   int doResolveAt(NsSet &nameservers, DNSName auth, bool flawedNSSet, const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret,
 747                   unsigned int depth, set<GetBestNSAnswer>&beenthere, vState& state);
 748   bool doResolveAtThisIP(const std::string& prefix, const DNSName& qname, const QType& qtype, LWResult& lwr, boost::optional<Netmask>& ednsmask, const DNSName& auth, bool const sendRDQuery, const DNSName& nsName, const ComboAddress& remoteIP, bool doTCP, bool* truncated);
 749   bool processAnswer(unsigned int depth, LWResult& lwr, const DNSName& qname, const QType& qtype, DNSName& auth, bool wasForwarded, const boost::optional<Netmask> ednsmask, bool sendRDQuery, NsSet &nameservers, std::vector<DNSRecord>& ret, const DNSFilterEngine& dfe, bool* gotNewServers, int* rcode, vState& state);
 750
 751   int doResolve(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, set<GetBestNSAnswer>& beenthere, vState& state);
 752   bool doOOBResolve(const AuthDomain& domain, const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, int& res);
 753   bool doOOBResolve(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, int &res);
 754   domainmap_t::const_iterator getBestAuthZone(DNSName* qname) const;
 755   bool doCNAMECacheCheck(const DNSName &qname, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, int &res, vState& state, bool wasAuthZone, bool wasForwardRecurse);
 756   bool doCacheCheck(const DNSName &qname, const DNSName& authname, bool wasForwardedOrAuthZone, bool wasAuthZone, bool wasForwardRecurse, const QType &qtype, vector<DNSRecord>&ret, unsigned int depth, int &res, vState& state);
 757   void getBestNSFromCache(const DNSName &qname, const QType &qtype, vector<DNSRecord>&bestns, bool* flawedNSSet, unsigned int depth, set<GetBestNSAnswer>& beenthere);
 758   DNSName getBestNSNamesFromCache(const DNSName &qname, const QType &qtype, NsSet& nsset, bool* flawedNSSet, unsigned int depth, set<GetBestNSAnswer>&beenthere);
 759
 760   inline vector<DNSName> shuffleInSpeedOrder(NsSet &nameservers, const string &prefix);
 761   inline vector<ComboAddress> shuffleForwardSpeed(const vector<ComboAddress> &rnameservers, const string &prefix, const bool wasRd);
 762   bool moreSpecificThan(const DNSName& a, const DNSName &b) const;
 763   vector<ComboAddress> getAddrs(const DNSName &qname, unsigned int depth, set<GetBestNSAnswer>& beenthere, bool cacheOnly);
 764
 765   bool nameserversBlockedByRPZ(const DNSFilterEngine& dfe, const NsSet& nameservers);
 766   bool nameserverIPBlockedByRPZ(const DNSFilterEngine& dfe, const ComboAddress&);
 767   bool throttledOrBlocked(const std::string& prefix, const ComboAddress& remoteIP, const DNSName& qname, const QType& qtype, bool pierceDontQuery);
 768
 769   vector<ComboAddress> retrieveAddressesForNS(const std::string& prefix, const DNSName& qname, vector<DNSName >::const_iterator& tns, const unsigned int depth, set<GetBestNSAnswer>& beenthere, const vector<DNSName >& rnameservers, NsSet& nameservers, bool& sendRDQuery, bool& pierceDontQuery, bool& flawedNSSet, bool cacheOnly);
 770
 771   void sanitizeRecords(const std::string& prefix, LWResult& lwr, const DNSName& qname, const QType& qtype, const DNSName& auth, bool wasForwarded, bool rdQuery);
 772   RCode::rcodes_ updateCacheFromRecords(unsigned int depth, LWResult& lwr, const DNSName& qname, const QType& qtype, const DNSName& auth, bool wasForwarded, const boost::optional<Netmask>, vState& state, bool& needWildcardProof, unsigned int& wildcardLabelsCount, bool sendRDQuery);
 773   bool processRecords(const std::string& prefix, const DNSName& qname, const QType& qtype, const DNSName& auth, LWResult& lwr, const bool sendRDQuery, vector<DNSRecord>& ret, set<DNSName>& nsset, DNSName& newtarget, DNSName& newauth, bool& realreferral, bool& negindic, vState& state, const bool needWildcardProof, const unsigned int wildcardLabelsCount);
 774
 775   bool doSpecialNamesResolve(const DNSName &qname, const QType &qtype, const uint16_t qclass, vector<DNSRecord> &ret);
 776
 777   int asyncresolveWrapper(const ComboAddress& ip, bool ednsMANDATORY, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, struct timeval* now, boost::optional<Netmask>& srcmask, LWResult* res, bool* chained) const;
 778
 779   boost::optional<Netmask> getEDNSSubnetMask(const DNSName&dn, const ComboAddress& rem);
 780
 781   bool validationEnabled() const;
 782   uint32_t computeLowestTTD(const std::vector<DNSRecord>& records, const std::vector<std::shared_ptr<RRSIGRecordContent> >& signatures, uint32_t signaturesTTL) const;
 783   void updateValidationState(vState& state, const vState stateUpdate);
 784   vState validateRecordsWithSigs(unsigned int depth, const DNSName& qname, const QType& qtype, const DNSName& name, const std::vector<DNSRecord>& records, const std::vector<std::shared_ptr<RRSIGRecordContent> >& signatures);
 785   vState validateDNSKeys(const DNSName& zone, const std::vector<DNSRecord>& dnskeys, const std::vector<std::shared_ptr<RRSIGRecordContent> >& signatures, unsigned int depth);
 786   vState getDNSKeys(const DNSName& signer, skeyset_t& keys, unsigned int depth);
 787   dState getDenialValidationState(const NegCache::NegCacheEntry& ne, const vState state, const dState expectedState, bool referralToUnsigned);
 788   void updateDenialValidationState(vState& neValidationState, const DNSName& neName, vState& state, const dState denialState, const dState expectedState, bool allowOptOut);
 789   void computeNegCacheValidationStatus(const NegCache::NegCacheEntry* ne, const DNSName& qname, const QType& qtype, const int res, vState& state, unsigned int depth);
 790   vState getTA(const DNSName& zone, dsmap_t& ds);
 791   bool haveExactValidationStatus(const DNSName& domain);
 792   vState getValidationStatus(const DNSName& subdomain, bool allowIndeterminate=true);
 793   void updateValidationStatusInCache(const DNSName &qname, const QType& qt, bool aa, vState newState) const;
 794
 795   bool lookForCut(const DNSName& qname, unsigned int depth, const vState existingState, vState& newState);
 796   void computeZoneCuts(const DNSName& begin, const DNSName& end, unsigned int depth);
 797
 798   void setUpdatingRootNS()
 799   {
 800     d_updatingRootNS = true;
 801   }
 802
 803   zonesStates_t d_cutStates;
 804   ostringstream d_trace;
 805   shared_ptr<RecursorLua4> d_pdl;
 806   boost::optional<Netmask> d_outgoingECSNetwork;
 807   std::shared_ptr<std::vector<std::unique_ptr<RemoteLogger>>> d_outgoingProtobufServers{nullptr};
 808 #ifdef HAVE_PROTOBUF
 809   boost::optional<const boost::uuids::uuid&> d_initialRequestId;
 810 #endif
 811   asyncresolve_t d_asyncResolve{nullptr};
 812   struct timeval d_now;
 813   string d_prefix;
 814   vState d_queryValidationState{Indeterminate};
 815
 816   /* When d_cacheonly is set to true, we will only check the cache.
 817    * This is set when the RD bit is unset in the incoming query
 818    */
 819   bool d_cacheonly;
 820   bool d_doDNSSEC;
 821   bool d_DNSSECValidationRequested{false};
 822   bool d_doEDNS0{true};
 823   bool d_requireAuthData{true};
 824   bool d_skipCNAMECheck{false};
 825   bool d_updatingRootNS{false};
 826   bool d_wantsRPZ{true};
 827   bool d_wasOutOfBand{false};
 828   bool d_wasVariable{false};
 829
 830   LogMode d_lm;
 831 };
 832
 833 class Socket;
 834 /* external functions, opaque to us */
 835 int asendtcp(const string& data, Socket* sock);
 836 int arecvtcp(string& data, size_t len, Socket* sock, bool incompleteOkay);
 837
 838
 839 struct PacketID
 840 {
 841   PacketID() : id(0), type(0), sock(0), inNeeded(0), inIncompleteOkay(false), outPos(0), nearMisses(0), fd(-1)
 842   {
 843     remote.reset();
 844   }
 845
 846   uint16_t id;  // wait for a specific id/remote pair
 847   uint16_t type;             // and this is its type
 848   ComboAddress remote;  // this is the remote
 849   DNSName domain;             // this is the question
 850
 851   Socket* sock;  // or wait for an event on a TCP fd
 852   string inMSG; // they'll go here
 853   size_t inNeeded; // if this is set, we'll read until inNeeded bytes are read
 854   bool inIncompleteOkay;
 855
 856   string outMSG; // the outgoing message that needs to be sent
 857   string::size_type outPos;    // how far we are along in the outMSG
 858
 859   typedef set<uint16_t > chain_t;
 860   mutable chain_t chain;
 861   mutable uint32_t nearMisses; // number of near misses - host correct, id wrong
 862   int fd;
 863
 864   bool operator<(const PacketID& b) const
 865   {
 866     int ourSock= sock ? sock->getHandle() : 0;
 867     int bSock = b.sock ? b.sock->getHandle() : 0;
 868     if( tie(remote, ourSock, type) < tie(b.remote, bSock, b.type))
 869       return true;
 870     if( tie(remote, ourSock, type) > tie(b.remote, bSock, b.type))
 871       return false;
 872
 873     return tie(domain, fd, id) < tie(b.domain, b.fd, b.id);
 874   }
 875 };
 876
 877 struct PacketIDBirthdayCompare: public std::binary_function<PacketID, PacketID, bool>
 878 {
 879   bool operator()(const PacketID& a, const PacketID& b) const
 880   {
 881     int ourSock= a.sock ? a.sock->getHandle() : 0;
 882     int bSock = b.sock ? b.sock->getHandle() : 0;
 883     if( tie(a.remote, ourSock, a.type) < tie(b.remote, bSock, b.type))
 884       return true;
 885     if( tie(a.remote, ourSock, a.type) > tie(b.remote, bSock, b.type))
 886       return false;
 887
 888     return a.domain < b.domain;
 889   }
 890 };
 891 extern thread_local std::unique_ptr<MemRecursorCache> t_RC;
 892 extern thread_local std::unique_ptr<RecursorPacketCache> t_packetCache;
 893 typedef MTasker<PacketID,string> MT_t;
 894 MT_t* getMT();
 895
 896 struct RecursorStats
 897 {
 898   std::atomic<uint64_t> servFails;
 899   std::atomic<uint64_t> nxDomains;
 900   std::atomic<uint64_t> noErrors;
 901   std::atomic<uint64_t> answers0_1, answers1_10, answers10_100, answers100_1000, answersSlow;
 902   std::atomic<uint64_t> auth4Answers0_1, auth4Answers1_10, auth4Answers10_100, auth4Answers100_1000, auth4AnswersSlow;
 903   std::atomic<uint64_t> auth6Answers0_1, auth6Answers1_10, auth6Answers10_100, auth6Answers100_1000, auth6AnswersSlow;
 904   std::atomic<uint64_t> ourtime0_1, ourtime1_2, ourtime2_4, ourtime4_8, ourtime8_16, ourtime16_32, ourtimeSlow;
 905   double avgLatencyUsec{0};
 906   double avgLatencyOursUsec{0};
 907   std::atomic<uint64_t> qcounter;     // not increased for unauth packets
 908   std::atomic<uint64_t> ipv6qcounter;
 909   std::atomic<uint64_t> tcpqcounter;
 910   std::atomic<uint64_t> unauthorizedUDP;  // when this is increased, qcounter isn't
 911   std::atomic<uint64_t> unauthorizedTCP;  // when this is increased, qcounter isn't
 912   std::atomic<uint64_t> policyDrops;
 913   std::atomic<uint64_t> tcpClientOverflow;
 914   std::atomic<uint64_t> clientParseError;
 915   std::atomic<uint64_t> serverParseError;
 916   std::atomic<uint64_t> tooOldDrops;
 917   std::atomic<uint64_t> truncatedDrops;
 918   std::atomic<uint64_t> queryPipeFullDrops;
 919   std::atomic<uint64_t> unexpectedCount;
 920   std::atomic<uint64_t> caseMismatchCount;
 921   std::atomic<uint64_t> spoofCount;
 922   std::atomic<uint64_t> resourceLimits;
 923   std::atomic<uint64_t> overCapacityDrops;
 924   std::atomic<uint64_t> ipv6queries;
 925   std::atomic<uint64_t> chainResends;
 926   std::atomic<uint64_t> nsSetInvalidations;
 927   std::atomic<uint64_t> ednsPingMatches;
 928   std::atomic<uint64_t> ednsPingMismatches;
 929   std::atomic<uint64_t> noPingOutQueries, noEdnsOutQueries;
 930   std::atomic<uint64_t> packetCacheHits;
 931   std::atomic<uint64_t> noPacketError;
 932   std::atomic<uint64_t> ignoredCount;
 933   std::atomic<uint64_t> emptyQueriesCount;
 934   time_t startupTime;
 935   std::atomic<uint64_t> dnssecQueries;
 936   std::atomic<uint64_t> dnssecAuthenticDataQueries;
 937   std::atomic<uint64_t> dnssecCheckDisabledQueries;
 938   std::atomic<uint64_t> variableResponses;
 939   unsigned int maxMThreadStackUsage;
 940   std::atomic<uint64_t> dnssecValidations; // should be the sum of all dnssecResult* stats
 941   std::map<vState, std::atomic<uint64_t> > dnssecResults;
 942   std::map<DNSFilterEngine::PolicyKind, std::atomic<uint64_t> > policyResults;
 943 };
 944
 945 //! represents a running TCP/IP client session
 946 class TCPConnection : public boost::noncopyable
 947 {
 948 public:
 949   TCPConnection(int fd, const ComboAddress& addr);
 950   ~TCPConnection();
 951
 952   int getFD() const
 953   {
 954     return d_fd;
 955   }
 956
 957   std::string data;
 958   const ComboAddress d_remote;
 959   size_t queriesCount{0};
 960   enum stateenum {BYTE0, BYTE1, GETQUESTION, DONE} state{BYTE0};
 961   uint16_t qlen{0};
 962   uint16_t bytesread{0};
 963
 964   static unsigned int getCurrentConnections() { return s_currentConnections; }
 965 private:
 966   const int d_fd;
 967   static AtomicCounter s_currentConnections; //!< total number of current TCP connections
 968 };
 969
 970 class ImmediateServFailException
 971 {
 972 public:
 973   ImmediateServFailException(string r) : reason(r) {};
 974
 975   string reason; //! Print this to tell the user what went wrong
 976 };
 977
 978 typedef boost::circular_buffer<ComboAddress> addrringbuf_t;
 979 extern thread_local std::unique_ptr<addrringbuf_t> t_servfailremotes, t_largeanswerremotes, t_remotes, t_bogusremotes, t_timeouts;
 980
 981 extern thread_local std::unique_ptr<boost::circular_buffer<pair<DNSName,uint16_t> > > t_queryring, t_servfailqueryring, t_bogusqueryring;
 982 extern thread_local std::shared_ptr<NetmaskGroup> t_allowFrom;
 983 string doQueueReloadLuaScript(vector<string>::const_iterator begin, vector<string>::const_iterator end);
 984 string doTraceRegex(vector<string>::const_iterator begin, vector<string>::const_iterator end);
 985 void parseACLs();
 986 extern RecursorStats g_stats;
 987 extern unsigned int g_networkTimeoutMsec;
 988 extern unsigned int g_numThreads;
 989 extern uint16_t g_outgoingEDNSBufsize;
 990 extern std::atomic<uint32_t> g_maxCacheEntries, g_maxPacketCacheEntries;
 991 extern bool g_lowercaseOutgoing;
 992
 993
 994 std::string reloadAuthAndForwards();
 995 ComboAddress parseIPAndPort(const std::string& input, uint16_t port);
 996 ComboAddress getQueryLocalAddress(int family, uint16_t port);
 997 typedef boost::function<void*(void)> pipefunc_t;
 998 void broadcastFunction(const pipefunc_t& func);
 999 void distributeAsyncFunction(const std::string& question, const pipefunc_t& func);
1000
1001 int directResolve(const DNSName& qname, const QType& qtype, int qclass, vector<DNSRecord>& ret);
1002
1003 template<class T> T broadcastAccFunction(const boost::function<T*()>& func);
1004
1005 std::shared_ptr<SyncRes::domainmap_t> parseAuthAndForwards();
1006 uint64_t* pleaseGetNsSpeedsSize();
1007 uint64_t* pleaseGetCacheSize();
1008 uint64_t* pleaseGetNegCacheSize();
1009 uint64_t* pleaseGetCacheHits();
1010 uint64_t* pleaseGetCacheMisses();
1011 uint64_t* pleaseGetConcurrentQueries();
1012 uint64_t* pleaseGetThrottleSize();
1013 uint64_t* pleaseGetPacketCacheHits();
1014 uint64_t* pleaseGetPacketCacheSize();
1015 uint64_t* pleaseWipeCache(const DNSName& canon, bool subtree=false);
1016 uint64_t* pleaseWipePacketCache(const DNSName& canon, bool subtree);
1017 uint64_t* pleaseWipeAndCountNegCache(const DNSName& canon, bool subtree=false);
1018 void doCarbonDump(void*);
1019 void primeHints(void);
1020
1021 extern __thread struct timeval g_now;