rec: introduce a setting to allow duplicates, including a dup handling fix

diff --git a/pdns/recursordist/docs/lua-config/rpz.rst b/pdns/recursordist/docs/lua-config/rpz.rst
index 47e1ac957f29fac60c07ec803420766932d4ace1..cd420505f9982a31d130dcc782b38ea6df870771 100644 (file)
--- a/pdns/recursordist/docs/lua-config/rpz.rst
+++ b/pdns/recursordist/docs/lua-config/rpz.rst
@@ -156,7 +156,14 @@ includeSOA
 .. versionadded:: 4.9.0
 
 Include the RPZ's SOA record to the reply's additional section if modified by a policy hit.
-Defaults to ``no``.
+Defaults to ``false``.
+
+ignoreDuplicates
+^^^^^^^^^^^^^^^^
+.. versionadded:: 5.0.0
+
+When loading an RPZ, ignore duplicate entries.
+Defaults to ``false``, duplicate entries will cause failure to load the zone.
 
 maxTTL
 ^^^^^^

diff --git a/pdns/recursordist/filterpo.cc b/pdns/recursordist/filterpo.cc
index e1b0e61640037cfb2f6a03730c3848ebb0c3d010..66a29401eade90abc1b569e86585329b7a737a87 100644 (file)
--- a/pdns/recursordist/filterpo.cc
+++ b/pdns/recursordist/filterpo.cc
@@ -383,11 +383,17 @@ void DNSFilterEngine::Zone::addNameTrigger(std::unordered_map<DNSName, Policy>&
     auto& existingPol = it->second;
 
     if (pol.d_kind != PolicyKind::Custom && !ignoreDuplicate) {
+      if (d_zoneData->d_ignoreDuplicates) {
+        return;
+      }
       throw std::runtime_error("Adding a " + getTypeToString(ptype) + "-based filter policy of kind " + getKindToString(pol.d_kind) + " but a policy of kind " + getKindToString(existingPol.d_kind) + " already exists for the following name: " + n.toLogString());
     }
 
-    if (existingPol.d_kind != PolicyKind::Custom && ignoreDuplicate) {
-      throw std::runtime_error("Adding a " + getTypeToString(ptype) + "-based filter policy of kind " + getKindToString(existingPol.d_kind) + " but there was already an existing policy for the following name: " + n.toLogString());
+    if (existingPol.d_kind != PolicyKind::Custom && !ignoreDuplicate) {
+      if (d_zoneData->d_ignoreDuplicates) {
+        return;
+      }
+      throw std::runtime_error("Adding a " + getTypeToString(ptype) + "-based filter policy of kind " + getKindToString(pol.d_kind) + " but a policy of kind " + getKindToString(existingPol.d_kind) + " already exists for for the following name: " + n.toLogString());
     }
 
     existingPol.d_custom.reserve(existingPol.d_custom.size() + pol.d_custom.size());
@@ -411,11 +417,17 @@ void DNSFilterEngine::Zone::addNetmaskTrigger(NetmaskTree<Policy>& nmt, const Ne
     auto& existingPol = const_cast<Policy&>(nmt.lookup(nm)->second);
 
     if (pol.d_kind != PolicyKind::Custom && !ignoreDuplicate) {
+      if (d_zoneData->d_ignoreDuplicates) {
+        return;
+      }
       throw std::runtime_error("Adding a " + getTypeToString(ptype) + "-based filter policy of kind " + getKindToString(pol.d_kind) + " but a policy of kind " + getKindToString(existingPol.d_kind) + " already exists for the following netmask: " + nm.toString());
     }
 
-    if (existingPol.d_kind != PolicyKind::Custom && ignoreDuplicate) {
-      throw std::runtime_error("Adding a " + getTypeToString(ptype) + "-based filter policy of kind " + getKindToString(existingPol.d_kind) + " but there was already an existing policy for the following netmask: " + nm.toString());
+    if (existingPol.d_kind != PolicyKind::Custom && !ignoreDuplicate) {
+      if (d_zoneData->d_ignoreDuplicates) {
+        return;
+      }
+      throw std::runtime_error("Adding a " + getTypeToString(ptype) + "-based filter policy of kind " + getKindToString(pol.d_kind) + " but a policy of kind " + getKindToString(existingPol.d_kind) + " already exists for the following netmask: " + nm.toString());
     }
 
     existingPol.d_custom.reserve(existingPol.d_custom.size() + pol.d_custom.size());

diff --git a/pdns/recursordist/filterpo.hh b/pdns/recursordist/filterpo.hh
index b5aa306bf737023d7e89f6d6ea0d66cbfb06f5f9..d0a89cebd24416d1cd1841cb48e6046d907dd3d9 100644 (file)
--- a/pdns/recursordist/filterpo.hh
+++ b/pdns/recursordist/filterpo.hh
@@ -102,6 +102,7 @@ public:
     Priority d_priority{maximumPriority};
     bool d_policyOverridesGettag{true};
     bool d_includeSOA{false};
+    bool d_ignoreDuplicates{false};
   };
 
   struct Policy
@@ -306,6 +307,11 @@ public:
       d_zoneData->d_includeSOA = flag;
     }
 
+    void setIgnoreDuplicates(bool flag)
+    {
+      d_zoneData->d_ignoreDuplicates = flag;
+    }
+
     void dump(FILE* fp) const;
 
     void addClientTrigger(const Netmask& nm, Policy&& pol, bool ignoreDuplicate = false);

diff --git a/pdns/recursordist/rec-lua-conf.cc b/pdns/recursordist/rec-lua-conf.cc
index 83c48cd432c3a4826032a5f8f914892b26aa1d40..8349a5fabcd01db12d05e3f686752c2876c5852b 100644 (file)
--- a/pdns/recursordist/rec-lua-conf.cc
+++ b/pdns/recursordist/rec-lua-conf.cc
@@ -158,6 +158,9 @@ static void parseRPZParameters(rpzOptions_t& have, std::shared_ptr<DNSFilterEngi
   if (have.count("includeSOA") != 0) {
     zone->setIncludeSOA(boost::get<bool>(have["includeSOA"]));
   }
+  if (have.count("ignoreDuplicates") != 0) {
+    zone->setIgnoreDuplicates(boost::get<bool>(have["ignoreDuplicates"]));
+  }
 }
 
 typedef std::unordered_map<std::string, boost::variant<bool, uint64_t, std::string, std::vector<std::pair<int, std::string>>>> protobufOptions_t;

diff --git a/pdns/recursordist/test-rpzloader_cc.cc b/pdns/recursordist/test-rpzloader_cc.cc
index 8d4d70345361fe8a6d431b450ae185c867ea8aea..04793abfb11b39756d0c80ff88123dcb3ad9bb3f 100644 (file)
--- a/pdns/recursordist/test-rpzloader_cc.cc
+++ b/pdns/recursordist/test-rpzloader_cc.cc
@@ -4,6 +4,9 @@
 #include "config.h"
 #endif
 
+#include <array>
+
+#include "arguments.hh"
 #include "rpzloader.hh"
 #include "syncres.hh"
 
@@ -37,4 +40,135 @@ BOOST_AUTO_TEST_CASE(test_rpz_loader)
   }
 }
 
+static string makeFile(const string& lines)
+{
+  std::array<char, 20> temp{"/tmp/rpzXXXXXXXXXX"};
+  int fileDesc = mkstemp(temp.data());
+  BOOST_REQUIRE(fileDesc > 0);
+  auto filePtr = std::unique_ptr<FILE, decltype(&fclose)>(fdopen(fileDesc, "w"), fclose);
+  BOOST_REQUIRE(filePtr);
+  size_t written = fwrite(lines.data(), 1, lines.length(), filePtr.get());
+  BOOST_REQUIRE(written == lines.length());
+  BOOST_REQUIRE(fflush(filePtr.get()) == 0);
+  return temp.data();
+}
+
+BOOST_AUTO_TEST_CASE(load_rpz_ok)
+{
+  const string lines = "\n"
+                       "$ORIGIN rpz.example.net.\n"
+                       "$TTL 1H\n"
+                       "@                   SOA LOCALHOST. named-mgr.example.net. (\n"
+                       "                                        1 1h 15m 30d 2h)\n"
+                       "                    NS LOCALHOST.\n"
+                       "\n"
+                       "; QNAME policy records.\n"
+                       "; There are no periods (.) after the relative owner names.\n"
+                       "nxdomain.example.com        CNAME   .       ; NXDOMAIN policy\n"
+                       "nodata.example.com          CNAME   *.      ; NODATA policy\n"
+                       "\n"
+                       "; Redirect to walled garden\n"
+                       "bad.example.com             A       10.0.0.1\n"
+                       "                            AAAA    2001:db8::1\n"
+                       "\n"
+                       "; Rewrite all names inside \"AZONE.EXAMPLE.COM\"\n"
+                       "; except \"OK.AZONE.EXAMPLE.COM\"\n"
+                       "*.azone.example.com         CNAME   garden.example.net.\n"
+                       "ok.azone.example.com        CNAME   rpz-passthru.\n"
+                       "\n"
+                       "; Redirect \"BZONE.EXAMPLE.COM\" and \"X.BZONE.EXAMPLE.COM\"\n"
+                       "; to \"BZONE.EXAMPLE.COM.GARDEN.EXAMPLE.NET\" and\n"
+                       "; \"X.BZONE.EXAMPLE.COM.GARDEN.EXAMPLE.NET\", respectively.\n"
+                       "bzone.example.com           CNAME   *.garden.example.net.\n"
+                       "*.bzone.example.com         CNAME   *.garden.example.net.\n"
+                       "\n"
+                       "; Rewrite all answers containing addresses in 192.0.2.0/24,\n"
+                       "; except 192.0.2.1\n"
+                       "24.0.2.0.192.rpz-ip         CNAME   .\n"
+                       "32.1.2.0.192.rpz-ip         CNAME   rpz-passthru.\n"
+                       "\n"
+                       "; Rewrite to NXDOMAIN all responses for domains for which\n"
+                       "; \"NS.EXAMPLE.COM\" is an authoritative DNS server for that domain\n"
+                       "; or any of its ancestors, or that have an authoritative server\n"
+                       "; in 2001:db8::/32\n"
+                       "ns.example.com.rpz-nsdname  CNAME   .\n"
+                       "32.zz.db8.2001.rpz-nsip     CNAME   .\n"
+                       "\n"
+                       "; Local Data can include many record types\n"
+                       "25.128.2.0.192.rpz-ip       A       172.16.0.1\n"
+                       "25.128.2.0.192.rpz-ip       A       172.16.0.2\n"
+                       "25.128.2.0.192.rpz-ip       A       172.16.0.3\n"
+                       "25.128.2.0.192.rpz-ip       MX      10 mx1.example.com\n"
+                       "25.128.2.0.192.rpz-ip       MX      20 mx2.example.com\n"
+                       "25.128.2.0.192.rpz-ip       TXT     \"Contact Central Services\"\n"
+                       "25.128.2.0.192.rpz-ip       TXT     \"Your system is infected.\"\n";
+
+  auto rpz = makeFile(lines);
+
+  ::arg().set("max-generate-steps") = "1";
+  ::arg().set("max-include-depth") = "20";
+  auto zone = std::make_shared<DNSFilterEngine::Zone>();
+  cerr << "rpz " << rpz << endl;
+  auto soa = loadRPZFromFile(rpz, zone, boost::none, false, 3600);
+  unlink(rpz.c_str());
+
+  BOOST_CHECK_EQUAL(soa->d_st.serial, 1U);
+  BOOST_CHECK_EQUAL(zone->getDomain(), DNSName("rpz.example.net."));
+  BOOST_CHECK_EQUAL(zone->size(), 12U);
+}
+
+BOOST_AUTO_TEST_CASE(load_rpz_dups)
+{
+  const string lines = "\n"
+                       "$TTL 300\n"
+                       "\n"
+                       "@              IN SOA  need.to.know.only.  hostmaster.spamhaus.org. (\n"
+                       "                       1000000000 ; Serial number\n"
+                       "                       60         ; Refresh every 1 minutes\n"
+                       "                       60         ; Retry every minute\n"
+                       "                       432000     ; Expire in 5 days\n"
+                       "                       60 )       ; negative caching ttl 1 minute\n"
+                       "               IN NS   LOCALHOST.\n"
+                       "qqq.powerdns.net   CNAME .\n"
+                       "qqq.powerdns.net   IN A 3.4.5.6\n";
+
+  auto rpz = makeFile(lines);
+
+  ::arg().set("max-generate-steps") = "1";
+  ::arg().set("max-include-depth") = "20";
+  auto zone = std::make_shared<DNSFilterEngine::Zone>();
+
+  BOOST_CHECK_THROW(loadRPZFromFile(rpz, zone, boost::none, false, 3600),
+                    std::runtime_error);
+  unlink(rpz.c_str());
+}
+
+BOOST_AUTO_TEST_CASE(load_rpz_dups_allow)
+{
+  const string lines = "\n"
+                       "$TTL 300\n"
+                       "\n"
+                       "@              IN SOA  need.to.know.only.  hostmaster.powerdns.org. (\n"
+                       "                       1000000000 ; Serial number\n"
+                       "                       60         ; Refresh every 1 minutes\n"
+                       "                       60         ; Retry every minute\n"
+                       "                       432000     ; Expire in 5 days\n"
+                       "                       60 )       ; negative caching ttl 1 minute\n"
+                       "               IN NS   LOCALHOST.\n"
+                       "qqq.powerdns.net   CNAME .\n"
+                       "qqq.powerdns.net   CNAME rpz-passthru\n";
+
+  auto rpz = makeFile(lines);
+
+  ::arg().set("max-generate-steps") = "1";
+  ::arg().set("max-include-depth") = "20";
+  auto zone = std::make_shared<DNSFilterEngine::Zone>();
+  zone->setIgnoreDuplicates(true);
+  auto soa = loadRPZFromFile(rpz, zone, boost::none, false, 3600);
+  unlink(rpz.c_str());
+  BOOST_CHECK_EQUAL(soa->d_st.serial, 1000000000U);
+  BOOST_CHECK_EQUAL(zone->getDomain(), DNSName("."));
+  BOOST_CHECK_EQUAL(zone->size(), 1U);
+}
+
 BOOST_AUTO_TEST_SUITE_END()