From f2d4526021d880bccdf595ff9eadc6038c9080c4 Mon Sep 17 00:00:00 2001
From: Pieter Lexis <pieter.lexis@powerdns.com>
Date: Thu, 10 May 2018 13:12:12 +0100
Subject: [PATCH] ixfrdist: Load configuration from yaml file

---
 configure.ac              |   5 +
 pdns/Makefile.am          |   6 +-
 pdns/ixfrdist.cc          | 316 +++++++++++++++++++++++++++-----------
 pdns/ixfrdist.example.yml |  74 +++++++++
 4 files changed, 313 insertions(+), 88 deletions(-)
 create mode 100644 pdns/ixfrdist.example.yml

diff --git a/configure.ac b/configure.ac
index 3d868ddc93..0f14fbe3aa 100644
--- a/configure.ac
+++ b/configure.ac
@@ -224,6 +224,11 @@ AC_ARG_ENABLE([tools],
 
 AC_MSG_RESULT([$enable_tools])
 AM_CONDITIONAL([TOOLS], [test "x$enable_tools" != "xno"])
+AS_IF([test "x$enable_tools" != "xno"], [
+  PKG_CHECK_MODULES([YAML], [yaml-cpp >= 0.5],[],
+    AC_MSG_ERROR([Could not find yaml-cpp])
+  )]
+)
 
 PDNS_WITH_PROTOBUF
 
diff --git a/pdns/Makefile.am b/pdns/Makefile.am
index 641743705e..e3617525f7 100644
--- a/pdns/Makefile.am
+++ b/pdns/Makefile.am
@@ -5,7 +5,8 @@ AM_CPPFLAGS += \
 	$(YAHTTP_CFLAGS) \
 	$(LIBEDIT_CFLAGS) \
 	$(LIBCRYPTO_INCLUDES) \
-	$(SYSTEMD_CFLAGS)
+	$(SYSTEMD_CFLAGS) \
+	$(YAML_CFLAGS)
 
 AM_CXXFLAGS = \
 	-DSYSCONFDIR=\"$(sysconfdir)\" \
@@ -642,7 +643,8 @@ ixfrdist_SOURCES = \
 
 ixfrdist_LDADD = \
 	$(BOOST_PROGRAM_OPTIONS_LIBS) \
-	$(LIBCRYPTO_LIBS)
+	$(LIBCRYPTO_LIBS) \
+	$(YAML_LIBS)
 
 ixfrdist_LDFLAGS = \
 	$(AM_LDFLAGS) \
diff --git a/pdns/ixfrdist.cc b/pdns/ixfrdist.cc
index c93f3d0d94..f07fcc14a3 100644
--- a/pdns/ixfrdist.cc
+++ b/pdns/ixfrdist.cc
@@ -42,6 +42,7 @@
 #include "misc.hh"
 #include "iputils.hh"
 #include "logger.hh"
+#include <yaml-cpp/yaml.h>
 
 /* BEGIN Needed because of deeper dependencies */
 #include "arguments.hh"
@@ -55,12 +56,59 @@ ArgvMap &arg()
 }
 /* END Needed because of deeper dependencies */
 
+// Allows reading/writing ComboAddresses and DNSNames in YAML-cpp
+namespace YAML {
+template<>
+struct convert<ComboAddress> {
+  static Node encode(const ComboAddress& rhs) {
+    return Node(rhs.toStringWithPort());
+  }
+  static bool decode(const Node& node, ComboAddress& rhs) {
+    if (!node.IsScalar()) {
+      return false;
+    }
+    try {
+      rhs = ComboAddress(node.as<string>());
+      return true;
+    } catch(const runtime_error &e) {
+      return false;
+    } catch (const PDNSException &e) {
+      return false;
+    }
+  }
+};
+
+template<>
+struct convert<DNSName> {
+  static Node encode(const DNSName& rhs) {
+    return Node(rhs.toStringRootDot());
+  }
+  static bool decode(const Node& node, DNSName& rhs) {
+    if (!node.IsScalar()) {
+      return false;
+    }
+    try {
+      rhs = DNSName(node.as<string>());
+      return true;
+    } catch(const runtime_error &e) {
+      return false;
+    } catch (const PDNSException &e) {
+      return false;
+    }
+  }
+};
+} // namespace YAML
+
+// Why a struct? This way we can add more options to a domain in the future
+struct ixfrdistdomain_t {
+  set<ComboAddress> masters; // A set so we can do multiple master addresses in the future
+};
 
 // For all the listen-sockets
 FDMultiplexer* g_fdm;
 
-// The domains we support
-set<DNSName> g_domains;
+// This contains the configuration for each domain
+map<DNSName, ixfrdistdomain_t> g_domainConfigs;
 
 // Map domains and their data
 std::map<DNSName, ixfrinfo_t> g_soas;
@@ -76,7 +124,6 @@ using namespace boost::multi_index;
 namespace po = boost::program_options;
 po::variables_map g_vm;
 string g_workdir;
-ComboAddress g_master;
 
 bool g_exiting = false;
 
@@ -99,7 +146,7 @@ void handleSignal(int signum) {
 }
 
 void usage(po::options_description &desc) {
-  cerr << "Usage: ixfrdist [OPTION]... DOMAIN [DOMAIN]..."<<endl;
+  cerr << "Usage: ixfrdist [OPTION]..."<<endl;
   cerr << desc << "\n";
 }
 
@@ -176,7 +223,8 @@ void updateThread() {
   std::map<DNSName, time_t> lastCheck;
 
   // Initialize the serials we have
-  for (const auto &domain : g_domains) {
+  for (const auto &domainConfig : g_domainConfigs) {
+    DNSName domain = domainConfig.first;
     lastCheck[domain] = 0;
     string dir = g_workdir + "/" + domain.toString();
     try {
@@ -218,7 +266,8 @@ void updateThread() {
       break;
     }
     time_t now = time(nullptr);
-    for (const auto &domain : g_domains) {
+    for (const auto &domainConfig : g_domainConfigs) {
+      DNSName domain = domainConfig.first;
       shared_ptr<SOARecordContent> current_soa;
       {
         std::lock_guard<std::mutex> guard(g_soas_mutex);
@@ -230,14 +279,20 @@ void updateThread() {
           (current_soa == nullptr && now - lastCheck[domain] < 30))  {                       // Or if we could not get an update at all still, every 30 seconds
         continue;
       }
+
+      // TODO Keep track of 'down' masters
+      set<ComboAddress>::const_iterator it(g_domainConfigs[domain].masters.begin());
+      std::advance(it, random() % g_domainConfigs[domain].masters.size());
+      ComboAddress master = *it;
+
       string dir = g_workdir + "/" + domain.toString();
-      g_log<<Logger::Info<<"Attempting to retrieve SOA Serial update for '"<<domain<<"' from '"<<g_master.toStringWithPort()<<"'"<<endl;
+      g_log<<Logger::Info<<"Attempting to retrieve SOA Serial update for '"<<domain<<"' from '"<<master.toStringWithPort()<<"'"<<endl;
       shared_ptr<SOARecordContent> sr;
       try {
         lastCheck[domain] = now;
-        auto newSerial = getSerialFromMaster(g_master, domain, sr); // TODO TSIG
+        auto newSerial = getSerialFromMaster(master, domain, sr); // TODO TSIG
         if(current_soa != nullptr) {
-          g_log<<Logger::Info<<"Got SOA Serial for "<<domain<<" from "<<g_master.toStringWithPort()<<": "<< newSerial<<", had Serial: "<<current_soa->d_st.serial;
+          g_log<<Logger::Info<<"Got SOA Serial for "<<domain<<" from "<<master.toStringWithPort()<<": "<< newSerial<<", had Serial: "<<current_soa->d_st.serial;
           if (newSerial == current_soa->d_st.serial) {
             g_log<<Logger::Info<<", not updating."<<endl;
             continue;
@@ -250,13 +305,13 @@ void updateThread() {
       }
       // Now get the full zone!
       g_log<<Logger::Info<<"Attempting to receive full zonedata for '"<<domain<<"'"<<endl;
-      ComboAddress local = g_master.isIPv4() ? ComboAddress("0.0.0.0") : ComboAddress("::");
+      ComboAddress local = master.isIPv4() ? ComboAddress("0.0.0.0") : ComboAddress("::");
       TSIGTriplet tt;
 
       // The *new* SOA
       shared_ptr<SOARecordContent> soa;
       try {
-        AXFRRetriever axfr(g_master, domain, tt, &local);
+        AXFRRetriever axfr(master, domain, tt, &local);
         unsigned int nrecords=0;
         Resolver::res_t nop;
         vector<DNSRecord> chunk;
@@ -327,7 +382,7 @@ bool checkQuery(const MOADNSParser& mdp, const ComboAddress& saddr, const bool u
 
   {
     std::lock_guard<std::mutex> guard(g_soas_mutex);
-    if (g_domains.find(mdp.d_qname) == g_domains.end()) {
+    if (g_domainConfigs.find(mdp.d_qname) == g_domainConfigs.end()) {
       info_msg.push_back("Domain name '" + mdp.d_qname.toLogString() + "' is not configured for distribution");
     }
 
@@ -712,6 +767,142 @@ void tcpWorker(int tid) {
   }
 }
 
+/* Parses the configuration file in configpath into config, adding defaults for
+ * missing parameters (if applicable), returning true if the config file was
+ * good, false otherwise. Will log all issues with the config
+ */
+bool parseAndCheckConfig(const string& configpath, YAML::Node& config) {
+  g_log<<Logger::Info<<"Loading configuration file from "<<g_vm["config"].as<string>()<<endl;
+  try {
+    config = YAML::LoadFile(configpath);
+  } catch (const runtime_error &e) {
+    g_log<<Logger::Error<<"Unable to load configuration file '"<<g_vm["config"].as<string>()<<"': "<<e.what()<<endl;
+    return false;
+  }
+
+  bool retval = true;
+
+  if (config["keep"]) {
+    try {
+      config["keep"].as<uint16_t>();
+    } catch (const runtime_error &e) {
+      g_log<<Logger::Error<<"Unable to read 'keep' value: "<<e.what()<<endl;
+      retval = false;
+    }
+  } else {
+    config["keep"] = KEEP_DEFAULT;
+  }
+
+  if (config["axfr-timeout"]) {
+    try {
+      config["axfr-timeout"].as<uint16_t>();
+    } catch (const runtime_error &e) {
+      g_log<<Logger::Error<<"Unable to read 'axfr-timeout' value: "<<e.what()<<endl;
+    }
+  } else {
+    config["axfr-timeout"] = AXFRTIMEOUT_DEFAULT;
+  }
+
+  if (config["tcp-in-threads"]) {
+    try {
+      config["tcp-in-threads"].as<uint16_t>();
+    } catch (const runtime_error &e) {
+      g_log<<Logger::Error<<"Unable to read 'tcp-in-thread' value: "<<e.what()<<endl;
+    }
+  } else {
+    config["tcp-in-threads"] = 10;
+  }
+
+  if (config["listen"]) {
+    try {
+      config["listen"].as<vector<ComboAddress>>();
+    } catch (const runtime_error &e) {
+      g_log<<Logger::Error<<"Unable to read 'listen' value: "<<e.what()<<endl;
+      retval = false;
+    }
+  } else {
+    config["listen"].push_back("127.0.0.1:53");
+    config["listen"].push_back("[::1]:53");
+  }
+
+  if (config["acl"]) {
+    try {
+      config["acl"].as<vector<string>>();
+    } catch (const runtime_error &e) {
+      g_log<<Logger::Error<<"Unable to read 'acl' value: "<<e.what()<<endl;
+      retval = false;
+    }
+  } else {
+    config["acl"].push_back("127.0.0.0/8");
+    config["acl"].push_back("::1/128");
+  }
+
+  if (config["work-dir"]) {
+    try {
+      config["work-dir"].as<string>();
+    } catch(const runtime_error &e) {
+      g_log<<Logger::Error<<"Unable to read 'work-dir' value: "<<e.what()<<endl;
+      retval = false;
+    }
+  } else {
+    char tmp[512];
+    config["work-dir"] = getcwd(tmp, sizeof(tmp)) ? string(tmp) : "";;
+  }
+
+  if (config["uid"]) {
+    try {
+      config["uid"].as<string>();
+    } catch(const runtime_error &e) {
+      g_log<<Logger::Error<<"Unable to read 'uid' value: "<<e.what()<<endl;
+      retval = false;
+    }
+  }
+
+  if (config["gid"]) {
+    try {
+      config["gid"].as<string>();
+    } catch(const runtime_error &e) {
+      g_log<<Logger::Error<<"Unable to read 'gid' value: "<<e.what()<<endl;
+      retval = false;
+    }
+  }
+
+  if (config["domains"]) {
+    if (config["domains"].size() == 0) {
+      g_log<<Logger::Error<<"No domains configured"<<endl;
+      retval = false;
+    }
+    for (auto const &domain : config["domains"]) {
+      try {
+        if (!domain["domain"]) {
+          g_log<<Logger::Error<<"An entry in 'domains' is missing a 'domain' key!"<<endl;
+          retval = false;
+          continue;
+        }
+        domain["domain"].as<DNSName>();
+      } catch (const runtime_error &e) {
+        g_log<<Logger::Error<<"Unable to read domain '"<<domain["domain"].as<string>()<<"': "<<e.what()<<endl;
+      }
+      try {
+        if (!domain["master"]) {
+          g_log<<Logger::Error<<"Domain '"<<domain["domain"].as<string>()<<"' has no master configured!"<<endl;
+          retval = false;
+          continue;
+        }
+        domain["master"].as<ComboAddress>();
+      } catch (const runtime_error &e) {
+        g_log<<Logger::Error<<"Unable to read domain '"<<domain["domain"].as<string>()<<"' master address: "<<e.what()<<endl;
+        retval = false;
+      }
+    }
+  } else {
+    g_log<<Logger::Error<<"No domains configured"<<endl;
+    retval = false;
+  }
+
+  return retval;
+}
+
 int main(int argc, char** argv) {
   g_log.setLoglevel(Logger::Notice);
   g_log.toConsole(Logger::Notice);
@@ -725,26 +916,10 @@ int main(int argc, char** argv) {
       ("version", "Display the version of ixfrdist")
       ("verbose", "Be verbose")
       ("debug", "Be even more verbose")
-      ("uid", po::value<string>(), "Drop privileges to this user after binding the listen sockets")
-      ("gid", po::value<string>(), "Drop privileges to this group after binding the listen sockets")
-      ("listen-address", po::value< vector< string>>(), "IP Address(es) to listen on")
-      ("acl", po::value<vector<string>>(), "IP Address masks that are allowed access, by default only loopback addresses are allowed")
-      ("server-address", po::value<string>()->default_value("127.0.0.1:5300"), "server address")
-      ("work-dir", po::value<string>()->default_value("."), "Directory for storing AXFR and IXFR data")
-      ("keep", po::value<uint16_t>()->default_value(KEEP_DEFAULT), "Number of old zone versions to retain")
-      ("axfr-timeout", po::value<uint16_t>()->default_value(AXFRTIMEOUT_DEFAULT), "Timeout in seconds for an inbound AXFR to complete")
-      ("tcp-in-threads", po::value<uint16_t>()->default_value(10), "Number of maximum simultaneous inbound TCP connections. Limits simultaneous AXFR/IXFR transactions")
+      ("config", po::value<string>()->default_value(SYSCONFDIR + string("/ixfrdist.yml")), "Configuration file to use")
       ;
-    po::options_description alloptions;
-    po::options_description hidden("hidden options");
-    hidden.add_options()
-      ("domains", po::value< vector<string> >(), "domains");
-
-    alloptions.add(desc).add(hidden);
-    po::positional_options_description p;
-    p.add("domains", -1);
 
-    po::store(po::command_line_parser(argc, argv).options(alloptions).positional(p).run(), g_vm);
+    po::store(po::command_line_parser(argc, argv).options(desc).run(), g_vm);
     po::notify(g_vm);
 
     if (g_vm.count("help") > 0) {
@@ -773,60 +948,23 @@ int main(int argc, char** argv) {
     g_log.toConsole(Logger::Debug);
   }
 
-  if (g_vm.count("keep") > 0) {
-    g_keep = g_vm["keep"].as<uint16_t>();
-  }
-
-  if (g_vm.count("axfr-timeout") > 0) {
-    g_axfrTimeout = g_vm["axfr-timeout"].as<uint16_t>();
-  }
-
-  vector<ComboAddress> listen_addresses = {ComboAddress("127.0.0.1:53")};
-
-  if (g_vm.count("listen-address") > 0) {
-    listen_addresses.clear();
-    for (const auto &addr : g_vm["listen-address"].as< vector< string> >()) {
-      try {
-        listen_addresses.push_back(ComboAddress(addr, 53));
-      } catch(PDNSException &e) {
-        g_log<<Logger::Error<<"listen-address '"<<addr<<"' is not an IP address: "<<e.reason<<endl;
-        had_error = true;
-      }
-    }
+  YAML::Node config;
+  if (!parseAndCheckConfig(g_vm["config"].as<string>(), config)) {
+    // parseAndCheckConfig already logged whatever was wrong
+    return EXIT_FAILURE;
   }
 
-  try {
-    g_master = ComboAddress(g_vm["server-address"].as<string>(), 53);
-  } catch(PDNSException &e) {
-    g_log<<Logger::Error<<"server-address '"<<g_vm["server-address"].as<string>()<<"' is not an IP address: "<<e.reason<<endl;
-    had_error = true;
-  }
+  /*  From hereon out, we known that all the values in config are valid. */
 
-  if (!g_vm.count("domains")) {
-    g_log<<Logger::Error<<"No domain(s) specified!"<<endl;
-    had_error = true;
-  } else {
-    for (const auto &domain : g_vm["domains"].as<vector<string>>()) {
-      try {
-        g_domains.insert(DNSName(domain));
-      } catch (PDNSException &e) {
-        g_log<<Logger::Error<<"'"<<domain<<"' is not a valid domain name: "<<e.reason<<endl;
-        had_error = true;
-      }
-    }
+  for (auto const &domain : config["domains"]) {
+    set<ComboAddress> s;
+    s.insert(domain["master"].as<ComboAddress>());
+    g_domainConfigs[domain["domain"].as<DNSName>()].masters = s;
   }
 
-  g_fdm = FDMultiplexer::getMultiplexerSilent();
-  if (g_fdm == nullptr) {
-    g_log<<Logger::Error<<"Could not enable a multiplexer for the listen sockets!"<<endl;
-    return EXIT_FAILURE;
-  }
+  g_workdir = config["work-dir"].as<string>();
 
-  vector<string> acl = {"127.0.0.0/8", "::1/128"};
-  if (g_vm.count("acl") > 0) {
-    acl = g_vm["acl"].as<vector<string>>();
-  }
-  for (const auto &addr : acl) {
+  for (const auto &addr : config["acl"].as<vector<string>>()) {
     try {
       g_acl.addMask(addr);
     } catch (const NetmaskException &e) {
@@ -836,8 +974,14 @@ int main(int argc, char** argv) {
   }
   g_log<<Logger::Notice<<"ACL set to "<<g_acl.toString()<<"."<<endl;
 
+  g_fdm = FDMultiplexer::getMultiplexerSilent();
+  if (g_fdm == nullptr) {
+    g_log<<Logger::Error<<"Could not enable a multiplexer for the listen sockets!"<<endl;
+    return EXIT_FAILURE;
+  }
+
   set<int> allSockets;
-  for (const auto& addr : listen_addresses) {
+  for (const auto& addr : config["listen"].as<vector<ComboAddress>>()) {
     for (const auto& stype : {SOCK_DGRAM, SOCK_STREAM}) {
       try {
         int s = SSocket(addr.sin4.sin_family, stype, 0);
@@ -857,12 +1001,10 @@ int main(int argc, char** argv) {
     }
   }
 
-  g_workdir = g_vm["work-dir"].as<string>();
-
   int newgid = 0;
 
-  if (g_vm.count("gid") > 0) {
-    string gid = g_vm["gid"].as<string>();
+  if (config["gid"]) {
+    string gid = config["gid"].as<string>();
     if (!(newgid = atoi(gid.c_str()))) {
       struct group *gr = getgrnam(gid.c_str());
       if (gr == nullptr) {
@@ -881,8 +1023,8 @@ int main(int argc, char** argv) {
 
   int newuid = 0;
 
-  if (g_vm.count("uid") > 0) {
-    string uid = g_vm["uid"].as<string>();
+  if (config["uid"]) {
+    string uid = config["uid"].as<string>();
     if (!(newuid = atoi(uid.c_str()))) {
       struct passwd *pw = getpwnam(uid.c_str());
       if (pw == nullptr) {
@@ -933,8 +1075,10 @@ int main(int argc, char** argv) {
   g_log<<Logger::Notice<<"IXFR distributor starting up!"<<endl;
 
   std::thread ut(updateThread);
+
   vector<std::thread> tcpHandlers;
-  for (int i = 0; i < g_vm["tcp-in-threads"].as<uint16_t>(); ++i) {
+  tcpHandlers.reserve(config["tcp-in-threads"].as<uint16_t>());
+  for (size_t i = 0; i < tcpHandlers.capacity(); ++i) {
     tcpHandlers.push_back(std::thread(tcpWorker, i));
   }
 
diff --git a/pdns/ixfrdist.example.yml b/pdns/ixfrdist.example.yml
new file mode 100644
index 0000000000..6d991c6ed0
--- /dev/null
+++ b/pdns/ixfrdist.example.yml
@@ -0,0 +1,74 @@
+# Listen addresses. ixfrdist will listen on both UDP and TCP.
+# When no port is specified, 53 is used. When specifying ports for IPv6, use the
+# "bracket" notation:
+#
+#    listen:
+#      - '127.0.0.1'
+#      - '::1'
+#      - '192.0.2.3:5300'
+#      - '[2001:DB8:1234::334]:5353'
+#
+# By default, or when unset, ixfrdist listens on local loopback addresses.
+listen:
+  - '127.0.0.1'
+  - '::1'
+
+# Netmasks or IP addresses of hosts that are allowed to query ixfrdist. Hosts
+# do not need a netmask:
+#
+#    acl:
+#      - '127.0.0.0/8'
+#      - '::1'
+#      - '192.0.2.55'
+#      - '2001:DB8:ABCD::/48'
+#
+# By default (or when unset), only loopback addresses are allowed.
+#
+acl:
+  - '127.0.0.0/8'
+  - '::1'
+
+# Timeout in seconds an AXFR transaction requested by ixfrdist may take.
+# Increase this when the network to the authoritative servers is slow or the
+# domains are very large and you experience timeouts. Set to 20 by default or
+# when unset.
+#
+axfr-timeout: 20
+
+# Amount of older copies/IXFR diffs to keep for every domain. This is set to
+# 20 by default or when unset.
+#
+keep: 20
+
+# Number of threads to spawn for TCP connections (AXFRs) from downstream hosts.
+# This is set to 10 by default or when unset.
+#
+tcp-in-threads: 10
+
+# The directory where the domain data is stored. When unset, the current
+# working directory is used. Note that this directory must be writable for the
+# user or group ixfrdist runs as.
+#
+# work-dir: '/var/lib/ixfrdist'
+
+# User to drop privileges to once all listen-sockets are bound. May be either
+# a username or numerical ID.
+#
+# uid: ixfrdist
+
+# Group to drop privileges to once all listen-sockets are bound. May be either
+# a username or numerical ID.
+#
+# gid: ixfrdist
+
+# The domains to redistribute, the 'master' and 'domains' keys are mandatory.
+# When no port is specified, 53 is used. When specifying ports for IPv6, use the
+# "bracket" notation:
+#
+#    domains:
+#      - domain: example.com
+#        master: 192.0.2.15
+#      - domain: rpz.example
+#        master: [2001:DB8:a34:543::53]:5353
+#
+domains: []
-- 
2.47.2