pdns/dnsdist.hh

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #pragma once
  23 #include "config.h"
  24 #include "ext/luawrapper/include/LuaContext.hpp"
  25
  26 #include <atomic>
  27 #include <mutex>
  28 #include <string>
  29 #include <thread>
  30 #include <time.h>
  31 #include <unistd.h>
  32 #include <unordered_map>
  33
  34 #include <boost/circular_buffer.hpp>
  35 #include <boost/variant.hpp>
  36
  37 #include "bpf-filter.hh"
  38 #include "capabilities.hh"
  39 #include "dnscrypt.hh"
  40 #include "dnsdist-cache.hh"
  41 #include "dnsdist-dynbpf.hh"
  42 #include "dnsname.hh"
  43 #include "ednsoptions.hh"
  44 #include "gettime.hh"
  45 #include "iputils.hh"
  46 #include "misc.hh"
  47 #include "mplexer.hh"
  48 #include "sholder.hh"
  49 #include "tcpiohandler.hh"
  50 #include "uuid-utils.hh"
  51
  52 void carbonDumpThread();
  53 uint64_t uptimeOfProcess(const std::string& str);
  54
  55 extern uint16_t g_ECSSourcePrefixV4;
  56 extern uint16_t g_ECSSourcePrefixV6;
  57 extern bool g_ECSOverride;
  58
  59 typedef std::unordered_map<string, string> QTag;
  60
  61 struct DNSQuestion
  62 {
  63   DNSQuestion(const DNSName* name, uint16_t type, uint16_t class_, unsigned int consumed_, const ComboAddress* lc, const ComboAddress* rem, struct dnsheader* header, size_t bufferSize, uint16_t queryLen, bool isTcp, const struct timespec* queryTime_):
  64     qname(name), local(lc), remote(rem), dh(header), queryTime(queryTime_), size(bufferSize), consumed(consumed_), tempFailureTTL(boost::none), qtype(type), qclass(class_), len(queryLen), ecsPrefixLength(rem->sin4.sin_family == AF_INET ? g_ECSSourcePrefixV4 : g_ECSSourcePrefixV6), tcp(isTcp), ecsOverride(g_ECSOverride) {
  65     const uint16_t* flags = getFlagsFromDNSHeader(dh);
  66     origFlags = *flags;
  67   }
  68
  69 #ifdef HAVE_PROTOBUF
  70   boost::optional<boost::uuids::uuid> uniqueId;
  71 #endif
  72   Netmask ecs;
  73   boost::optional<Netmask> subnet;
  74   const DNSName* qname{nullptr};
  75   const ComboAddress* local{nullptr};
  76   const ComboAddress* remote{nullptr};
  77   std::shared_ptr<QTag> qTag{nullptr};
  78   std::shared_ptr<std::map<uint16_t, EDNSOptionView> > ednsOptions;
  79   std::shared_ptr<DNSCryptQuery> dnsCryptQuery{nullptr};
  80   std::shared_ptr<DNSDistPacketCache> packetCache{nullptr};
  81   struct dnsheader* dh{nullptr};
  82   const struct timespec* queryTime{nullptr};
  83   size_t size;
  84   unsigned int consumed{0};
  85   int delayMsec{0};
  86   boost::optional<uint32_t> tempFailureTTL;
  87   uint32_t cacheKeyNoECS;
  88   uint32_t cacheKey;
  89   const uint16_t qtype;
  90   const uint16_t qclass;
  91   uint16_t len;
  92   uint16_t ecsPrefixLength;
  93   uint16_t origFlags;
  94   uint8_t ednsRCode{0};
  95   const bool tcp;
  96   bool skipCache{false};
  97   bool ecsOverride;
  98   bool useECS{true};
  99   bool addXPF{true};
 100   bool ecsSet{false};
 101   bool ecsAdded{false};
 102   bool ednsAdded{false};
 103   bool useZeroScope{false};
 104   bool dnssecOK{false};
 105 };
 106
 107 struct DNSResponse : DNSQuestion
 108 {
 109   DNSResponse(const DNSName* name, uint16_t type, uint16_t class_, unsigned int consumed, const ComboAddress* lc, const ComboAddress* rem, struct dnsheader* header, size_t bufferSize, uint16_t responseLen, bool isTcp, const struct timespec* queryTime_):
 110     DNSQuestion(name, type, class_, consumed, lc, rem, header, bufferSize, responseLen, isTcp, queryTime_) { }
 111 };
 112
 113 /* so what could you do:
 114    drop,
 115    fake up nxdomain,
 116    provide actual answer,
 117    allow & and stop processing,
 118    continue processing,
 119    modify header:    (servfail|refused|notimp), set TC=1,
 120    send to pool */
 121
 122 class DNSAction
 123 {
 124 public:
 125   enum class Action { Drop, Nxdomain, Refused, Spoof, Allow, HeaderModify, Pool, Delay, Truncate, ServFail, None, NoOp, NoRecurse };
 126   static std::string typeToString(const Action& action)
 127   {
 128     switch(action) {
 129     case Action::Drop:
 130       return "Drop";
 131     case Action::Nxdomain:
 132       return "Send NXDomain";
 133     case Action::Refused:
 134       return "Send Refused";
 135     case Action::Spoof:
 136       return "Spoof an answer";
 137     case Action::Allow:
 138       return "Allow";
 139     case Action::HeaderModify:
 140       return "Modify the header";
 141     case Action::Pool:
 142       return "Route to a pool";
 143     case Action::Delay:
 144       return "Delay";
 145     case Action::Truncate:
 146       return "Truncate over UDP";
 147     case Action::ServFail:
 148       return "Send ServFail";
 149     case Action::None:
 150     case Action::NoOp:
 151       return "Do nothing";
 152     case Action::NoRecurse:
 153       return "Set rd=0";
 154     }
 155
 156     return "Unknown";
 157   }
 158
 159   virtual Action operator()(DNSQuestion*, string* ruleresult) const =0;
 160   virtual ~DNSAction()
 161   {
 162   }
 163   virtual string toString() const = 0;
 164   virtual std::map<string, double> getStats() const
 165   {
 166     return {{}};
 167   }
 168 };
 169
 170 class DNSResponseAction
 171 {
 172 public:
 173   enum class Action { Allow, Delay, Drop, HeaderModify, ServFail, None };
 174   virtual Action operator()(DNSResponse*, string* ruleresult) const =0;
 175   virtual ~DNSResponseAction()
 176   {
 177   }
 178   virtual string toString() const = 0;
 179 };
 180
 181 struct DynBlock
 182 {
 183   DynBlock(): action(DNSAction::Action::None), warning(false)
 184   {
 185   }
 186
 187   DynBlock(const std::string& reason_, const struct timespec& until_, const DNSName& domain_, DNSAction::Action action_): reason(reason_), until(until_), domain(domain_), action(action_), warning(false)
 188   {
 189   }
 190
 191   DynBlock(const DynBlock& rhs): reason(rhs.reason), until(rhs.until), domain(rhs.domain), action(rhs.action), warning(rhs.warning)
 192   {
 193     blocks.store(rhs.blocks);
 194   }
 195
 196   DynBlock& operator=(const DynBlock& rhs)
 197   {
 198     reason=rhs.reason;
 199     until=rhs.until;
 200     domain=rhs.domain;
 201     action=rhs.action;
 202     blocks.store(rhs.blocks);
 203     warning=rhs.warning;
 204     return *this;
 205   }
 206
 207   string reason;
 208   struct timespec until;
 209   DNSName domain;
 210   DNSAction::Action action;
 211   mutable std::atomic<unsigned int> blocks;
 212   bool warning;
 213 };
 214
 215 extern GlobalStateHolder<NetmaskTree<DynBlock>> g_dynblockNMG;
 216
 217 extern vector<pair<struct timeval, std::string> > g_confDelta;
 218
 219 struct DNSDistStats
 220 {
 221   using stat_t=std::atomic<uint64_t>; // aww yiss ;-)
 222   stat_t responses{0};
 223   stat_t servfailResponses{0};
 224   stat_t queries{0};
 225   stat_t frontendNXDomain{0};
 226   stat_t frontendServFail{0};
 227   stat_t frontendNoError{0};
 228   stat_t nonCompliantQueries{0};
 229   stat_t nonCompliantResponses{0};
 230   stat_t rdQueries{0};
 231   stat_t emptyQueries{0};
 232   stat_t aclDrops{0};
 233   stat_t dynBlocked{0};
 234   stat_t ruleDrop{0};
 235   stat_t ruleNXDomain{0};
 236   stat_t ruleRefused{0};
 237   stat_t ruleServFail{0};
 238   stat_t selfAnswered{0};
 239   stat_t downstreamTimeouts{0};
 240   stat_t downstreamSendErrors{0};
 241   stat_t truncFail{0};
 242   stat_t noPolicy{0};
 243   stat_t cacheHits{0};
 244   stat_t cacheMisses{0};
 245   stat_t latency0_1{0}, latency1_10{0}, latency10_50{0}, latency50_100{0}, latency100_1000{0}, latencySlow{0};
 246   stat_t securityStatus{0};
 247
 248   double latencyAvg100{0}, latencyAvg1000{0}, latencyAvg10000{0}, latencyAvg1000000{0};
 249   typedef std::function<uint64_t(const std::string&)> statfunction_t;
 250   typedef boost::variant<stat_t*, double*, statfunction_t> entry_t;
 251   std::vector<std::pair<std::string, entry_t>> entries{
 252     {"responses", &responses},
 253     {"servfail-responses", &servfailResponses},
 254     {"queries", &queries},
 255     {"frontend-nxdomain", &frontendNXDomain},
 256     {"frontend-servfail", &frontendServFail},
 257     {"frontend-noerror", &frontendNoError},
 258     {"acl-drops", &aclDrops},
 259     {"rule-drop", &ruleDrop},
 260     {"rule-nxdomain", &ruleNXDomain},
 261     {"rule-refused", &ruleRefused},
 262     {"rule-servfail", &ruleServFail},
 263     {"self-answered", &selfAnswered},
 264     {"downstream-timeouts", &downstreamTimeouts},
 265     {"downstream-send-errors", &downstreamSendErrors},
 266     {"trunc-failures", &truncFail},
 267     {"no-policy", &noPolicy},
 268     {"latency0-1", &latency0_1},
 269     {"latency1-10", &latency1_10},
 270     {"latency10-50", &latency10_50},
 271     {"latency50-100", &latency50_100},
 272     {"latency100-1000", &latency100_1000},
 273     {"latency-slow", &latencySlow},
 274     {"latency-avg100", &latencyAvg100},
 275     {"latency-avg1000", &latencyAvg1000},
 276     {"latency-avg10000", &latencyAvg10000},
 277     {"latency-avg1000000", &latencyAvg1000000},
 278     {"uptime", uptimeOfProcess},
 279     {"real-memory-usage", getRealMemoryUsage},
 280     {"special-memory-usage", getSpecialMemoryUsage},
 281     {"noncompliant-queries", &nonCompliantQueries},
 282     {"noncompliant-responses", &nonCompliantResponses},
 283     {"rdqueries", &rdQueries},
 284     {"empty-queries", &emptyQueries},
 285     {"cache-hits", &cacheHits},
 286     {"cache-misses", &cacheMisses},
 287     {"cpu-user-msec", getCPUTimeUser},
 288     {"cpu-sys-msec", getCPUTimeSystem},
 289     {"fd-usage", getOpenFileDescriptors},
 290     {"dyn-blocked", &dynBlocked},
 291     {"dyn-block-nmg-size", [](const std::string&) { return g_dynblockNMG.getLocal()->size(); }},
 292     {"security-status", &securityStatus}
 293   };
 294 };
 295
 296 // Metric types for Prometheus
 297 enum class PrometheusMetricType: int {
 298     counter = 1,
 299     gauge = 2
 300 };
 301
 302 // Keeps additional information about metrics
 303 struct MetricDefinition {
 304   MetricDefinition(PrometheusMetricType _prometheusType, const std::string& _description): description(_description), prometheusType(_prometheusType) {
 305   }
 306
 307   MetricDefinition() = default;
 308
 309   // Metric description
 310   std::string description;
 311   // Metric type for Prometheus
 312   PrometheusMetricType prometheusType;
 313 };
 314
 315 struct MetricDefinitionStorage {
 316   // Return metric definition by name
 317   bool getMetricDetails(std::string metricName, MetricDefinition& metric) {
 318   auto metricDetailsIter = metrics.find(metricName);
 319
 320   if (metricDetailsIter == metrics.end()) {
 321     return false;
 322   }
 323
 324   metric = metricDetailsIter->second;
 325     return true;
 326   };
 327
 328   // Return string representation of Prometheus metric type
 329   std::string getPrometheusStringMetricType(PrometheusMetricType metricType) {
 330     switch (metricType) {
 331       case PrometheusMetricType::counter:
 332         return "counter";
 333         break;
 334       case PrometheusMetricType::gauge:
 335         return "gauge";
 336         break;
 337       default:
 338         return "";
 339         break;
 340     }
 341   };
 342
 343   std::map<std::string, MetricDefinition> metrics = {
 344     { "responses",              MetricDefinition(PrometheusMetricType::counter, "Number of responses received from backends") },
 345     { "servfail-responses",     MetricDefinition(PrometheusMetricType::counter, "Number of SERVFAIL answers received from backends") },
 346     { "queries",                MetricDefinition(PrometheusMetricType::counter, "Number of received queries")},
 347     { "frontend-nxdomain",      MetricDefinition(PrometheusMetricType::counter, "Number of NXDomain answers sent to clients")},
 348     { "frontend-servfail",      MetricDefinition(PrometheusMetricType::counter, "Number of SERVFAIL answers sent to clients")},
 349     { "frontend-noerror",       MetricDefinition(PrometheusMetricType::counter, "Number of NoError answers sent to clients")},
 350     { "acl-drops",              MetricDefinition(PrometheusMetricType::counter, "Number of packets dropped because of the ACL")},
 351     { "rule-drop",              MetricDefinition(PrometheusMetricType::counter, "Number of queries dropped because of a rule")},
 352     { "rule-nxdomain",          MetricDefinition(PrometheusMetricType::counter, "Number of NXDomain answers returned because of a rule")},
 353     { "rule-refused",           MetricDefinition(PrometheusMetricType::counter, "Number of Refused answers returned because of a rule")},
 354     { "rule-servfail",          MetricDefinition(PrometheusMetricType::counter, "Number of SERVFAIL answers received because of a rule")},
 355     { "self-answered",          MetricDefinition(PrometheusMetricType::counter, "Number of self-answered responses")},
 356     { "downstream-timeouts",    MetricDefinition(PrometheusMetricType::counter, "Number of queries not answered in time by a backend")},
 357     { "downstream-send-errors", MetricDefinition(PrometheusMetricType::counter, "Number of errors when sending a query to a backend")},
 358     { "trunc-failures",         MetricDefinition(PrometheusMetricType::counter, "Number of errors encountered while truncating an answer")},
 359     { "no-policy",              MetricDefinition(PrometheusMetricType::counter, "Number of queries dropped because no server was available")},
 360     { "latency0-1",             MetricDefinition(PrometheusMetricType::counter, "Number of queries answered in less than 1ms")},
 361     { "latency1-10",            MetricDefinition(PrometheusMetricType::counter, "Number of queries answered in 1-10 ms")},
 362     { "latency10-50",           MetricDefinition(PrometheusMetricType::counter, "Number of queries answered in 10-50 ms")},
 363     { "latency50-100",          MetricDefinition(PrometheusMetricType::counter, "Number of queries answered in 50-100 ms")},
 364     { "latency100-1000",        MetricDefinition(PrometheusMetricType::counter, "Number of queries answered in 100-1000 ms")},
 365     { "latency-slow",           MetricDefinition(PrometheusMetricType::counter, "Number of queries answered in more than 1 second")},
 366     { "latency-avg100",         MetricDefinition(PrometheusMetricType::gauge,   "Average response latency in microseconds of the last 100 packets")},
 367     { "latency-avg1000",        MetricDefinition(PrometheusMetricType::gauge,   "Average response latency in microseconds of the last 1000 packets")},
 368     { "latency-avg10000",       MetricDefinition(PrometheusMetricType::gauge,   "Average response latency in microseconds of the last 10000 packets")},
 369     { "latency-avg1000000",     MetricDefinition(PrometheusMetricType::gauge,   "Average response latency in microseconds of the last 1000000 packets")},
 370     { "uptime",                 MetricDefinition(PrometheusMetricType::gauge,   "Uptime of the dnsdist process in seconds")},
 371     { "real-memory-usage",      MetricDefinition(PrometheusMetricType::gauge,   "Current memory usage in bytes")},
 372     { "noncompliant-queries",   MetricDefinition(PrometheusMetricType::counter, "Number of queries dropped as non-compliant")},
 373     { "noncompliant-responses", MetricDefinition(PrometheusMetricType::counter, "Number of answers from a backend dropped as non-compliant")},
 374     { "rdqueries",              MetricDefinition(PrometheusMetricType::counter, "Number of received queries with the recursion desired bit set")},
 375     { "empty-queries",          MetricDefinition(PrometheusMetricType::counter, "Number of empty queries received from clients")},
 376     { "cache-hits",             MetricDefinition(PrometheusMetricType::counter, "Number of times an answer was retrieved from cache")},
 377     { "cache-misses",           MetricDefinition(PrometheusMetricType::counter, "Number of times an answer not found in the cache")},
 378     { "cpu-user-msec",          MetricDefinition(PrometheusMetricType::counter, "Milliseconds spent by dnsdist in the user state")},
 379     { "cpu-sys-msec",           MetricDefinition(PrometheusMetricType::counter, "Milliseconds spent by dnsdist in the system state")},
 380     { "fd-usage",               MetricDefinition(PrometheusMetricType::gauge,   "Number of currently used file descriptors")},
 381     { "dyn-blocked",            MetricDefinition(PrometheusMetricType::counter, "Number of queries dropped because of a dynamic block")},
 382     { "dyn-block-nmg-size",     MetricDefinition(PrometheusMetricType::gauge,   "Number of dynamic blocks entries") },
 383     { "security-status",        MetricDefinition(PrometheusMetricType::gauge,   "Security status of this software. 0=unknown, 1=OK, 2=upgrade recommended, 3=upgrade mandatory") },
 384   };
 385 };
 386
 387 extern MetricDefinitionStorage g_metricDefinitions;
 388 extern struct DNSDistStats g_stats;
 389 void doLatencyStats(double udiff);
 390
 391
 392 struct StopWatch
 393 {
 394   StopWatch(bool realTime=false): d_needRealTime(realTime)
 395   {
 396   }
 397   struct timespec d_start{0,0};
 398   bool d_needRealTime{false};
 399
 400   void start() {
 401     if(gettime(&d_start, d_needRealTime) < 0)
 402       unixDie("Getting timestamp");
 403
 404   }
 405
 406   void set(const struct timespec& from) {
 407     d_start = from;
 408   }
 409
 410   double udiff() const {
 411     struct timespec now;
 412     if(gettime(&now, d_needRealTime) < 0)
 413       unixDie("Getting timestamp");
 414
 415     return 1000000.0*(now.tv_sec - d_start.tv_sec) + (now.tv_nsec - d_start.tv_nsec)/1000.0;
 416   }
 417
 418   double udiffAndSet() {
 419     struct timespec now;
 420     if(gettime(&now, d_needRealTime) < 0)
 421       unixDie("Getting timestamp");
 422
 423     auto ret= 1000000.0*(now.tv_sec - d_start.tv_sec) + (now.tv_nsec - d_start.tv_nsec)/1000.0;
 424     d_start = now;
 425     return ret;
 426   }
 427
 428 };
 429
 430 class BasicQPSLimiter
 431 {
 432 public:
 433   BasicQPSLimiter()
 434   {
 435   }
 436
 437   BasicQPSLimiter(unsigned int burst): d_tokens(burst)
 438   {
 439     d_prev.start();
 440   }
 441
 442   bool check(unsigned int rate, unsigned int burst) const // this is not quite fair
 443   {
 444     auto delta = d_prev.udiffAndSet();
 445
 446     if(delta > 0.0) // time, frequently, does go backwards..
 447       d_tokens += 1.0 * rate * (delta/1000000.0);
 448
 449     if(d_tokens > burst) {
 450       d_tokens = burst;
 451     }
 452
 453     bool ret=false;
 454     if(d_tokens >= 1.0) { // we need this because burst=1 is weird otherwise
 455       ret=true;
 456       --d_tokens;
 457     }
 458
 459     return ret;
 460   }
 461
 462   bool seenSince(const struct timespec& cutOff) const
 463   {
 464     return cutOff < d_prev.d_start;
 465   }
 466
 467 protected:
 468   mutable StopWatch d_prev;
 469   mutable double d_tokens;
 470 };
 471
 472 class QPSLimiter : public BasicQPSLimiter
 473 {
 474 public:
 475   QPSLimiter(): BasicQPSLimiter()
 476   {
 477   }
 478
 479   QPSLimiter(unsigned int rate, unsigned int burst): BasicQPSLimiter(burst), d_rate(rate), d_burst(burst), d_passthrough(false)
 480   {
 481     d_prev.start();
 482   }
 483
 484   unsigned int getRate() const
 485   {
 486     return d_passthrough ? 0 : d_rate;
 487   }
 488
 489   int getPassed() const
 490   {
 491     return d_passed;
 492   }
 493
 494   int getBlocked() const
 495   {
 496     return d_blocked;
 497   }
 498
 499   bool check() const // this is not quite fair
 500   {
 501     if (d_passthrough) {
 502       return true;
 503     }
 504
 505     bool ret = BasicQPSLimiter::check(d_rate, d_burst);
 506     if (ret) {
 507       d_passed++;
 508     }
 509     else {
 510       d_blocked++;
 511     }
 512
 513     return ret;
 514   }
 515 private:
 516   mutable unsigned int d_passed{0};
 517   mutable unsigned int d_blocked{0};
 518   unsigned int d_rate;
 519   unsigned int d_burst;
 520   bool d_passthrough{true};
 521 };
 522
 523 struct ClientState;
 524
 525 struct IDState
 526 {
 527   IDState() : origFD(-1), sentTime(true), delayMsec(0), tempFailureTTL(boost::none) { origDest.sin4.sin_family = 0;}
 528   IDState(const IDState& orig): origRemote(orig.origRemote), origDest(orig.origDest), age(orig.age)
 529   {
 530     origFD.store(orig.origFD.load());
 531     origID = orig.origID;
 532     delayMsec = orig.delayMsec;
 533     tempFailureTTL = orig.tempFailureTTL;
 534   }
 535
 536   std::atomic<int> origFD;  // set to <0 to indicate this state is empty   // 4
 537
 538   ComboAddress origRemote;                                    // 28
 539   ComboAddress origDest;                                      // 28
 540   StopWatch sentTime;                                         // 16
 541   DNSName qname;                                              // 80
 542   std::shared_ptr<DNSCryptQuery> dnsCryptQuery{nullptr};
 543 #ifdef HAVE_PROTOBUF
 544   boost::optional<boost::uuids::uuid> uniqueId;
 545 #endif
 546   boost::optional<Netmask> subnet{boost::none};
 547   std::shared_ptr<DNSDistPacketCache> packetCache{nullptr};
 548   std::shared_ptr<QTag> qTag{nullptr};
 549   const ClientState* cs{nullptr};
 550   uint32_t cacheKey;                                          // 4
 551   uint32_t cacheKeyNoECS;                                     // 4
 552   uint16_t age;                                               // 4
 553   uint16_t qtype;                                             // 2
 554   uint16_t qclass;                                            // 2
 555   uint16_t origID;                                            // 2
 556   uint16_t origFlags;                                         // 2
 557   int delayMsec;
 558   boost::optional<uint32_t> tempFailureTTL;
 559   bool ednsAdded{false};
 560   bool ecsAdded{false};
 561   bool skipCache{false};
 562   bool destHarvested{false}; // if true, origDest holds the original dest addr, otherwise the listening addr
 563   bool dnssecOK{false};
 564   bool useZeroScope;
 565 };
 566
 567 typedef std::unordered_map<string, unsigned int> QueryCountRecords;
 568 typedef std::function<std::tuple<bool, string>(DNSQuestion dq)> QueryCountFilter;
 569 struct QueryCount {
 570   QueryCount()
 571   {
 572     pthread_rwlock_init(&queryLock, nullptr);
 573   }
 574   QueryCountRecords records;
 575   QueryCountFilter filter;
 576   pthread_rwlock_t queryLock;
 577   bool enabled{false};
 578 };
 579
 580 extern QueryCount g_qcount;
 581
 582 struct ClientState
 583 {
 584   std::set<int> cpus;
 585   ComboAddress local;
 586   std::shared_ptr<DNSCryptContext> dnscryptCtx{nullptr};
 587   shared_ptr<TLSFrontend> tlsFrontend;
 588   std::atomic<uint64_t> queries{0};
 589   std::atomic<uint64_t> tcpDiedReadingQuery{0};
 590   std::atomic<uint64_t> tcpDiedSendingResponse{0};
 591   std::atomic<uint64_t> tcpGaveUp{0};
 592   std::atomic<uint64_t> tcpClientTimeouts{0};
 593   std::atomic<uint64_t> tcpDownstreamTimeouts{0};
 594   std::atomic<uint64_t> tcpCurrentConnections{0};
 595   std::atomic<double> tcpAvgQueriesPerConnection{0.0};
 596   /* in ms */
 597   std::atomic<double> tcpAvgConnectionDuration{0.0};
 598   int udpFD{-1};
 599   int tcpFD{-1};
 600   bool muted{false};
 601
 602   int getSocket() const
 603   {
 604     return udpFD != -1 ? udpFD : tcpFD;
 605   }
 606
 607   std::string getType() const
 608   {
 609     std::string result = udpFD != -1 ? "UDP" : "TCP";
 610
 611     if (tlsFrontend) {
 612       result += " (DNS over TLS)";
 613     }
 614     else if (dnscryptCtx) {
 615       result += " (DNSCrypt)";
 616     }
 617
 618     return result;
 619   }
 620
 621 #ifdef HAVE_EBPF
 622   shared_ptr<BPFFilter> d_filter;
 623
 624   void detachFilter()
 625   {
 626     if (d_filter) {
 627       d_filter->removeSocket(getSocket());
 628       d_filter = nullptr;
 629     }
 630   }
 631
 632   void attachFilter(shared_ptr<BPFFilter> bpf)
 633   {
 634     detachFilter();
 635
 636     bpf->addSocket(getSocket());
 637     d_filter = bpf;
 638   }
 639 #endif /* HAVE_EBPF */
 640
 641   void updateTCPMetrics(size_t queries, uint64_t durationMs)
 642   {
 643     tcpAvgQueriesPerConnection = (99.0 * tcpAvgQueriesPerConnection / 100.0) + (queries / 100.0);
 644     tcpAvgConnectionDuration = (99.0 * tcpAvgConnectionDuration / 100.0) + (durationMs / 100.0);
 645   }
 646 };
 647
 648 class TCPClientCollection {
 649   std::vector<int> d_tcpclientthreads;
 650   std::atomic<uint64_t> d_numthreads{0};
 651   std::atomic<uint64_t> d_pos{0};
 652   std::atomic<uint64_t> d_queued{0};
 653   const uint64_t d_maxthreads{0};
 654   std::mutex d_mutex;
 655   int d_singlePipe[2];
 656   const bool d_useSinglePipe;
 657 public:
 658
 659   TCPClientCollection(size_t maxThreads, bool useSinglePipe=false): d_maxthreads(maxThreads), d_singlePipe{-1,-1}, d_useSinglePipe(useSinglePipe)
 660
 661   {
 662     d_tcpclientthreads.reserve(maxThreads);
 663
 664     if (d_useSinglePipe) {
 665       if (pipe(d_singlePipe) < 0) {
 666         throw std::runtime_error("Error creating the TCP single communication pipe: " + string(strerror(errno)));
 667       }
 668
 669       if (!setNonBlocking(d_singlePipe[0])) {
 670         int err = errno;
 671         close(d_singlePipe[0]);
 672         close(d_singlePipe[1]);
 673         throw std::runtime_error("Error setting the TCP single communication pipe non-blocking: " + string(strerror(err)));
 674       }
 675
 676       if (!setNonBlocking(d_singlePipe[1])) {
 677         int err = errno;
 678         close(d_singlePipe[0]);
 679         close(d_singlePipe[1]);
 680         throw std::runtime_error("Error setting the TCP single communication pipe non-blocking: " + string(strerror(err)));
 681       }
 682     }
 683   }
 684   int getThread()
 685   {
 686     uint64_t pos = d_pos++;
 687     ++d_queued;
 688     return d_tcpclientthreads[pos % d_numthreads];
 689   }
 690   bool hasReachedMaxThreads() const
 691   {
 692     return d_numthreads >= d_maxthreads;
 693   }
 694   uint64_t getThreadsCount() const
 695   {
 696     return d_numthreads;
 697   }
 698   uint64_t getQueuedCount() const
 699   {
 700     return d_queued;
 701   }
 702   void decrementQueuedCount()
 703   {
 704     --d_queued;
 705   }
 706   void addTCPClientThread();
 707 };
 708
 709 extern std::unique_ptr<TCPClientCollection> g_tcpclientthreads;
 710
 711 struct DownstreamState
 712 {
 713    typedef std::function<std::tuple<DNSName, uint16_t, uint16_t>(const DNSName&, uint16_t, uint16_t, dnsheader*)> checkfunc_t;
 714
 715   DownstreamState(const ComboAddress& remote_, const ComboAddress& sourceAddr_, unsigned int sourceItf, size_t numberOfSockets);
 716   DownstreamState(const ComboAddress& remote_): DownstreamState(remote_, ComboAddress(), 0, 1) {}
 717   ~DownstreamState()
 718   {
 719     for (auto& fd : sockets) {
 720       if (fd >= 0) {
 721         close(fd);
 722         fd = -1;
 723       }
 724     }
 725   }
 726   boost::uuids::uuid id;
 727   std::set<unsigned int> hashes;
 728   mutable pthread_rwlock_t d_lock;
 729   std::vector<int> sockets;
 730   std::mutex socketsLock;
 731   std::mutex connectLock;
 732   std::unique_ptr<FDMultiplexer> mplexer{nullptr};
 733   std::thread tid;
 734   const ComboAddress remote;
 735   QPSLimiter qps;
 736   vector<IDState> idStates;
 737   const ComboAddress sourceAddr;
 738   checkfunc_t checkFunction;
 739   DNSName checkName{"a.root-servers.net."};
 740   QType checkType{QType::A};
 741   uint16_t checkClass{QClass::IN};
 742   std::atomic<uint64_t> idOffset{0};
 743   std::atomic<uint64_t> sendErrors{0};
 744   std::atomic<uint64_t> outstanding{0};
 745   std::atomic<uint64_t> reuseds{0};
 746   std::atomic<uint64_t> queries{0};
 747   struct {
 748     std::atomic<uint64_t> sendErrors{0};
 749     std::atomic<uint64_t> reuseds{0};
 750     std::atomic<uint64_t> queries{0};
 751   } prev;
 752   std::atomic<uint64_t> tcpDiedSendingQuery{0};
 753   std::atomic<uint64_t> tcpDiedReadingResponse{0};
 754   std::atomic<uint64_t> tcpGaveUp{0};
 755   std::atomic<uint64_t> tcpReadTimeouts{0};
 756   std::atomic<uint64_t> tcpWriteTimeouts{0};
 757   std::atomic<uint64_t> tcpCurrentConnections{0};
 758   std::atomic<double> tcpAvgQueriesPerConnection{0.0};
 759   /* in ms */
 760   std::atomic<double> tcpAvgConnectionDuration{0.0};
 761   string name;
 762   size_t socketsOffset{0};
 763   double queryLoad{0.0};
 764   double dropRate{0.0};
 765   double latencyUsec{0.0};
 766   int order{1};
 767   int weight{1};
 768   int tcpConnectTimeout{5};
 769   int tcpRecvTimeout{30};
 770   int tcpSendTimeout{30};
 771   unsigned int checkInterval{1};
 772   unsigned int lastCheck{0};
 773   const unsigned int sourceItf{0};
 774   uint16_t retries{5};
 775   uint16_t xpfRRCode{0};
 776   uint16_t checkTimeout{1000}; /* in milliseconds */
 777   uint8_t currentCheckFailures{0};
 778   uint8_t consecutiveSuccessfulChecks{0};
 779   uint8_t maxCheckFailures{1};
 780   uint8_t minRiseSuccesses{1};
 781   StopWatch sw;
 782   set<string> pools;
 783   enum class Availability { Up, Down, Auto} availability{Availability::Auto};
 784   bool mustResolve{false};
 785   bool upStatus{false};
 786   bool useECS{false};
 787   bool setCD{false};
 788   bool disableZeroScope{false};
 789   std::atomic<bool> connected{false};
 790   std::atomic_flag threadStarted;
 791   bool tcpFastOpen{false};
 792   bool ipBindAddrNoPort{true};
 793
 794   bool isUp() const
 795   {
 796     if(availability == Availability::Down)
 797       return false;
 798     if(availability == Availability::Up)
 799       return true;
 800     return upStatus;
 801   }
 802   void setUp() { availability = Availability::Up; }
 803   void setDown() { availability = Availability::Down; }
 804   void setAuto() { availability = Availability::Auto; }
 805   string getName() const {
 806     if (name.empty()) {
 807       return remote.toStringWithPort();
 808     }
 809     return name;
 810   }
 811   string getNameWithAddr() const {
 812     if (name.empty()) {
 813       return remote.toStringWithPort();
 814     }
 815     return name + " (" + remote.toStringWithPort()+ ")";
 816   }
 817   string getStatus() const
 818   {
 819     string status;
 820     if(availability == DownstreamState::Availability::Up)
 821       status = "UP";
 822     else if(availability == DownstreamState::Availability::Down)
 823       status = "DOWN";
 824     else
 825       status = (upStatus ? "up" : "down");
 826     return status;
 827   }
 828   bool reconnect();
 829   void hash();
 830   void setId(const boost::uuids::uuid& newId);
 831   void setWeight(int newWeight);
 832
 833   void updateTCPMetrics(size_t queries, uint64_t durationMs)
 834   {
 835     tcpAvgQueriesPerConnection = (99.0 * tcpAvgQueriesPerConnection / 100.0) + (queries / 100.0);
 836     tcpAvgConnectionDuration = (99.0 * tcpAvgConnectionDuration / 100.0) + (durationMs / 100.0);
 837   }
 838 };
 839 using servers_t =vector<std::shared_ptr<DownstreamState>>;
 840
 841 template <class T> using NumberedVector = std::vector<std::pair<unsigned int, T> >;
 842
 843 void responderThread(std::shared_ptr<DownstreamState> state);
 844 extern std::mutex g_luamutex;
 845 extern LuaContext g_lua;
 846 extern std::string g_outputBuffer; // locking for this is ok, as locked by g_luamutex
 847
 848 class DNSRule
 849 {
 850 public:
 851   virtual ~DNSRule ()
 852   {
 853   }
 854   virtual bool matches(const DNSQuestion* dq) const =0;
 855   virtual string toString() const = 0;
 856   mutable std::atomic<uint64_t> d_matches{0};
 857 };
 858
 859 using NumberedServerVector = NumberedVector<shared_ptr<DownstreamState>>;
 860 typedef std::function<shared_ptr<DownstreamState>(const NumberedServerVector& servers, const DNSQuestion*)> policyfunc_t;
 861
 862 struct ServerPolicy
 863 {
 864   string name;
 865   policyfunc_t policy;
 866   bool isLua;
 867   std::string toString() const {
 868     return string("ServerPolicy") + (isLua ? " (Lua)" : "") + " \"" + name + "\"";
 869   }
 870 };
 871
 872 struct ServerPool
 873 {
 874   ServerPool()
 875   {
 876     pthread_rwlock_init(&d_lock, nullptr);
 877   }
 878
 879   const std::shared_ptr<DNSDistPacketCache> getCache() const { return packetCache; };
 880
 881   bool getECS() const
 882   {
 883     return d_useECS;
 884   }
 885
 886   void setECS(bool useECS)
 887   {
 888     d_useECS = useECS;
 889   }
 890
 891   std::shared_ptr<DNSDistPacketCache> packetCache{nullptr};
 892   std::shared_ptr<ServerPolicy> policy{nullptr};
 893
 894   size_t countServers(bool upOnly)
 895   {
 896     size_t count = 0;
 897     ReadLock rl(&d_lock);
 898     for (const auto& server : d_servers) {
 899       if (!upOnly || std::get<1>(server)->isUp() ) {
 900         count++;
 901       }
 902     }
 903     return count;
 904   }
 905
 906   NumberedVector<shared_ptr<DownstreamState>> getServers()
 907   {
 908     NumberedVector<shared_ptr<DownstreamState>> result;
 909     {
 910       ReadLock rl(&d_lock);
 911       result = d_servers;
 912     }
 913     return result;
 914   }
 915
 916   void addServer(shared_ptr<DownstreamState>& server)
 917   {
 918     WriteLock wl(&d_lock);
 919     unsigned int count = (unsigned int) d_servers.size();
 920     d_servers.push_back(make_pair(++count, server));
 921     /* we need to reorder based on the server 'order' */
 922     std::stable_sort(d_servers.begin(), d_servers.end(), [](const std::pair<unsigned int,std::shared_ptr<DownstreamState> >& a, const std::pair<unsigned int,std::shared_ptr<DownstreamState> >& b) {
 923       return a.second->order < b.second->order;
 924     });
 925     /* and now we need to renumber for Lua (custom policies) */
 926     size_t idx = 1;
 927     for (auto& serv : d_servers) {
 928       serv.first = idx++;
 929     }
 930   }
 931
 932   void removeServer(shared_ptr<DownstreamState>& server)
 933   {
 934     WriteLock wl(&d_lock);
 935     size_t idx = 1;
 936     bool found = false;
 937     for (auto it = d_servers.begin(); it != d_servers.end();) {
 938       if (found) {
 939         /* we need to renumber the servers placed
 940            after the removed one, for Lua (custom policies) */
 941         it->first = idx++;
 942         it++;
 943       }
 944       else if (it->second == server) {
 945         it = d_servers.erase(it);
 946         found = true;
 947       } else {
 948         idx++;
 949         it++;
 950       }
 951     }
 952   }
 953
 954 private:
 955   NumberedVector<shared_ptr<DownstreamState>> d_servers;
 956   pthread_rwlock_t d_lock;
 957   bool d_useECS{false};
 958 };
 959 using pools_t=map<std::string,std::shared_ptr<ServerPool>>;
 960 void setPoolPolicy(pools_t& pools, const string& poolName, std::shared_ptr<ServerPolicy> policy);
 961 void addServerToPool(pools_t& pools, const string& poolName, std::shared_ptr<DownstreamState> server);
 962 void removeServerFromPool(pools_t& pools, const string& poolName, std::shared_ptr<DownstreamState> server);
 963
 964 struct CarbonConfig
 965 {
 966   ComboAddress server;
 967   std::string namespace_name;
 968   std::string ourname;
 969   std::string instance_name;
 970   unsigned int interval;
 971 };
 972
 973 enum ednsHeaderFlags {
 974   EDNS_HEADER_FLAG_NONE = 0,
 975   EDNS_HEADER_FLAG_DO = 32768
 976 };
 977
 978 struct DNSDistRuleAction
 979 {
 980   std::shared_ptr<DNSRule> d_rule;
 981   std::shared_ptr<DNSAction> d_action;
 982   boost::uuids::uuid d_id;
 983   uint64_t d_creationOrder;
 984 };
 985
 986 struct DNSDistResponseRuleAction
 987 {
 988   std::shared_ptr<DNSRule> d_rule;
 989   std::shared_ptr<DNSResponseAction> d_action;
 990   boost::uuids::uuid d_id;
 991   uint64_t d_creationOrder;
 992 };
 993
 994 extern GlobalStateHolder<SuffixMatchTree<DynBlock>> g_dynblockSMT;
 995 extern DNSAction::Action g_dynBlockAction;
 996
 997 extern GlobalStateHolder<vector<CarbonConfig> > g_carbon;
 998 extern GlobalStateHolder<ServerPolicy> g_policy;
 999 extern GlobalStateHolder<servers_t> g_dstates;
1000 extern GlobalStateHolder<pools_t> g_pools;
1001 extern GlobalStateHolder<vector<DNSDistRuleAction> > g_rulactions;
1002 extern GlobalStateHolder<vector<DNSDistResponseRuleAction> > g_resprulactions;
1003 extern GlobalStateHolder<vector<DNSDistResponseRuleAction> > g_cachehitresprulactions;
1004 extern GlobalStateHolder<vector<DNSDistResponseRuleAction> > g_selfansweredresprulactions;
1005 extern GlobalStateHolder<NetmaskGroup> g_ACL;
1006
1007 extern ComboAddress g_serverControl; // not changed during runtime
1008
1009 extern std::vector<std::tuple<ComboAddress, bool, bool, int, std::string, std::set<int>>> g_locals; // not changed at runtime (we hope XXX)
1010 extern std::vector<shared_ptr<TLSFrontend>> g_tlslocals;
1011 extern vector<ClientState*> g_frontends;
1012 extern bool g_truncateTC;
1013 extern bool g_fixupCase;
1014 extern int g_tcpRecvTimeout;
1015 extern int g_tcpSendTimeout;
1016 extern int g_udpTimeout;
1017 extern uint16_t g_maxOutstanding;
1018 extern std::atomic<bool> g_configurationDone;
1019 extern uint64_t g_maxTCPClientThreads;
1020 extern uint64_t g_maxTCPQueuedConnections;
1021 extern size_t g_maxTCPQueriesPerConn;
1022 extern size_t g_maxTCPConnectionDuration;
1023 extern size_t g_maxTCPConnectionsPerClient;
1024 extern std::atomic<uint16_t> g_cacheCleaningDelay;
1025 extern std::atomic<uint16_t> g_cacheCleaningPercentage;
1026 extern bool g_verboseHealthChecks;
1027 extern uint32_t g_staleCacheEntriesTTL;
1028 extern bool g_apiReadWrite;
1029 extern std::string g_apiConfigDirectory;
1030 extern bool g_servFailOnNoPolicy;
1031 extern uint32_t g_hashperturb;
1032 extern bool g_useTCPSinglePipe;
1033 extern uint16_t g_downstreamTCPCleanupInterval;
1034 extern size_t g_udpVectorSize;
1035 extern bool g_preserveTrailingData;
1036 extern bool g_allowEmptyResponse;
1037 extern bool g_roundrobinFailOnNoServer;
1038
1039 #ifdef HAVE_EBPF
1040 extern shared_ptr<BPFFilter> g_defaultBPFFilter;
1041 extern std::vector<std::shared_ptr<DynBPFFilter> > g_dynBPFFilters;
1042 #endif /* HAVE_EBPF */
1043
1044 struct LocalHolders
1045 {
1046   LocalHolders(): acl(g_ACL.getLocal()), policy(g_policy.getLocal()), rulactions(g_rulactions.getLocal()), cacheHitRespRulactions(g_cachehitresprulactions.getLocal()), selfAnsweredRespRulactions(g_selfansweredresprulactions.getLocal()), servers(g_dstates.getLocal()), dynNMGBlock(g_dynblockNMG.getLocal()), dynSMTBlock(g_dynblockSMT.getLocal()), pools(g_pools.getLocal())
1047   {
1048   }
1049
1050   LocalStateHolder<NetmaskGroup> acl;
1051   LocalStateHolder<ServerPolicy> policy;
1052   LocalStateHolder<vector<DNSDistRuleAction> > rulactions;
1053   LocalStateHolder<vector<DNSDistResponseRuleAction> > cacheHitRespRulactions;
1054   LocalStateHolder<vector<DNSDistResponseRuleAction> > selfAnsweredRespRulactions;
1055   LocalStateHolder<servers_t> servers;
1056   LocalStateHolder<NetmaskTree<DynBlock> > dynNMGBlock;
1057   LocalStateHolder<SuffixMatchTree<DynBlock> > dynSMTBlock;
1058   LocalStateHolder<pools_t> pools;
1059 };
1060
1061 struct dnsheader;
1062
1063 void controlThread(int fd, ComboAddress local);
1064 vector<std::function<void(void)>> setupLua(bool client, const std::string& config);
1065 std::shared_ptr<ServerPool> getPool(const pools_t& pools, const std::string& poolName);
1066 std::shared_ptr<ServerPool> createPoolIfNotExists(pools_t& pools, const string& poolName);
1067 NumberedServerVector getDownstreamCandidates(const pools_t& pools, const std::string& poolName);
1068
1069 std::shared_ptr<DownstreamState> firstAvailable(const NumberedServerVector& servers, const DNSQuestion* dq);
1070
1071 std::shared_ptr<DownstreamState> leastOutstanding(const NumberedServerVector& servers, const DNSQuestion* dq);
1072 std::shared_ptr<DownstreamState> wrandom(const NumberedServerVector& servers, const DNSQuestion* dq);
1073 std::shared_ptr<DownstreamState> whashed(const NumberedServerVector& servers, const DNSQuestion* dq);
1074 std::shared_ptr<DownstreamState> chashed(const NumberedServerVector& servers, const DNSQuestion* dq);
1075 std::shared_ptr<DownstreamState> roundrobin(const NumberedServerVector& servers, const DNSQuestion* dq);
1076
1077 struct WebserverConfig
1078 {
1079   std::string password;
1080   std::string apiKey;
1081   boost::optional<std::map<std::string, std::string> > customHeaders;
1082   std::mutex lock;
1083 };
1084
1085 void setWebserverAPIKey(const boost::optional<std::string> apiKey);
1086 void setWebserverPassword(const std::string& password);
1087 void setWebserverCustomHeaders(const boost::optional<std::map<std::string, std::string> > customHeaders);
1088
1089 void dnsdistWebserverThread(int sock, const ComboAddress& local);
1090 void tcpAcceptorThread(void* p);
1091
1092 void setLuaNoSideEffect(); // if nothing has been declared, set that there are no side effects
1093 void setLuaSideEffect();   // set to report a side effect, cancelling all _no_ side effect calls
1094 bool getLuaNoSideEffect(); // set if there were only explicit declarations of _no_ side effect
1095 void resetLuaSideEffect(); // reset to indeterminate state
1096
1097 bool responseContentMatches(const char* response, const uint16_t responseLen, const DNSName& qname, const uint16_t qtype, const uint16_t qclass, const ComboAddress& remote, unsigned int& consumed);
1098 bool processResponse(char** response, uint16_t* responseLen, size_t* responseSize, LocalStateHolder<vector<DNSDistResponseRuleAction> >& localRespRulactions, DNSResponse& dr, size_t addRoom, std::vector<uint8_t>& rewrittenResponse, bool muted);
1099
1100 bool checkQueryHeaders(const struct dnsheader* dh);
1101
1102 extern std::vector<std::tuple<ComboAddress, std::shared_ptr<DNSCryptContext>, bool, int, std::string, std::set<int> > > g_dnsCryptLocals;
1103 int handleDNSCryptQuery(char* packet, uint16_t len, std::shared_ptr<DNSCryptQuery> query, uint16_t* decryptedQueryLen, bool tcp, time_t now, std::vector<uint8_t>& response);
1104 boost::optional<std::vector<uint8_t>> checkDNSCryptQuery(const ClientState& cs, const char* query, uint16_t& len, std::shared_ptr<DNSCryptQuery>& dnsCryptQuery, time_t now, bool tcp);
1105
1106 bool addXPF(DNSQuestion& dq, uint16_t optionCode);
1107
1108 uint16_t getRandomDNSID();
1109
1110 #include "dnsdist-snmp.hh"
1111
1112 extern bool g_snmpEnabled;
1113 extern bool g_snmpTrapsEnabled;
1114 extern DNSDistSNMPAgent* g_snmpAgent;
1115 extern bool g_addEDNSToSelfGeneratedResponses;
1116
1117 static const size_t s_udpIncomingBufferSize{1500};
1118
1119 enum class ProcessQueryResult { Drop, SendAnswer, PassToBackend };
1120 ProcessQueryResult processQuery(DNSQuestion& dq, ClientState& cs, LocalHolders& holders, std::shared_ptr<DownstreamState>& selectedBackend);
1121
1122 DNSResponse makeDNSResponseFromIDState(IDState& ids, struct dnsheader* dh, size_t bufferSize, uint16_t responseLen, bool isTCP);
1123 void setIDStateFromDNSQuestion(IDState& ids, DNSQuestion& dq, DNSName&& qname);