rec: Add support for loading the Public Suffix List from a file

diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc
index 28360e42b2c47b4a2f83f3756ce28adf022943fd..783a847e335b0914883df56825c40cb92fa48858 100644 (file)
--- a/pdns/pdns_recursor.cc
+++ b/pdns/pdns_recursor.cc
@@ -87,6 +87,7 @@
 #include "rec-lua-conf.hh"
 #include "ednsoptions.hh"
 #include "gettime.hh"
+#include "pubsuffix.hh"
 #ifdef NOD_ENABLED
 #include "nod.hh"
 #endif /* NOD_ENABLED */
@@ -3517,7 +3518,7 @@ static int serviceMain(int argc, char*argv[])
   }
 
   parseACLs();
-  sortPublicSuffixList();
+  initPublicSuffixList(::arg()["public-suffix-list-file"]);
 
   if(!::arg()["dont-query"].empty()) {
     vector<string> ips;
@@ -4215,6 +4216,7 @@ int main(int argc, char **argv)
     ::arg().set("udp-source-port-max", "Maximum UDP port to bind on")="65535";
     ::arg().set("udp-source-port-avoid", "List of comma separated UDP port number to avoid")="11211";
     ::arg().set("rng", "Specify random number generator to use. Valid values are auto,sodium,openssl,getrandom,arc4random,urandom.")="auto";
+    ::arg().set("public-suffix-list-file", "Path to the Public Suffix List file, if any")="";
 #ifdef NOD_ENABLED
     ::arg().set("new-domain-tracking", "Track newly observed domains (i.e. never seen before).")="no";
     ::arg().set("new-domain-log", "Log newly observed domains.")="yes";

diff --git a/pdns/pubsuffix.hh b/pdns/pubsuffix.hh
index 721200ded73271b18178ae03a69938a1e0e8d011..cc5f7c4272ca529677bc23c92d000f5ce24cb6c0 100644 (file)
--- a/pdns/pubsuffix.hh
+++ b/pdns/pubsuffix.hh
@@ -19,6 +19,15 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
-#ifndef PDNS_PUBSUFFIX_HH
-extern const char* g_pubsuffix[];
-#endif
+#pragma once
+
+#include <string>
+#include <vector>
+
+extern std::vector<std::vector<std::string>> g_pubs;
+
+/* initialize the g_pubs variable with the public suffix list,
+   using the file passed in parameter if any, or the built-in
+   list otherwise.
+*/
+void initPublicSuffixList(const std::string& file);

diff --git a/pdns/rec_channel.hh b/pdns/rec_channel.hh
index 0fadd4125986dd89cd8cd1be0a5aff6ec9ce508e..0f1dd7973e7f4413000ff1e20b1f1aaf11fc1e2f 100644 (file)
--- a/pdns/rec_channel.hh
+++ b/pdns/rec_channel.hh
@@ -65,7 +65,6 @@ public:
 
 std::map<std::string, std::string> getAllStatsMap();
 extern pthread_mutex_t g_carbon_config_lock;
-void sortPublicSuffixList();
 std::vector<std::pair<DNSName, uint16_t> >* pleaseGetQueryRing();
 std::vector<std::pair<DNSName, uint16_t> >* pleaseGetServfailQueryRing();
 std::vector<std::pair<DNSName, uint16_t> >* pleaseGetBogusQueryRing();

diff --git a/pdns/rec_channel_rec.cc b/pdns/rec_channel_rec.cc
index 53eebc3d8cd0cab91f5b1d0237a4c937c5314022..c318948cb87ab26067a2495b7f1baadec0871920 100644 (file)
--- a/pdns/rec_channel_rec.cc
+++ b/pdns/rec_channel_rec.cc
@@ -1208,24 +1208,6 @@ string doGenericTopRemotes(pleaseremotefunc_t func)
   return ret.str();
 }
 
-namespace {
-  typedef vector<vector<string> > pubs_t;
-  pubs_t g_pubs;
-}
-
-void sortPublicSuffixList()
-{
-  for(const char** p=g_pubsuffix; *p; ++p) {
-    string low=toLower(*p);
-
-    vector<string> parts;
-    stringtok(parts, low, ".");
-    reverse(parts.begin(), parts.end());
-    g_pubs.push_back(parts);
-  }
-  sort(g_pubs.begin(), g_pubs.end());
-}
-
 // XXX DNSName Pain - this function should benefit from native DNSName methods
 DNSName getRegisteredName(const DNSName& dom)
 {

diff --git a/pdns/recursordist/Makefile.am b/pdns/recursordist/Makefile.am
index f3ae8afe2a687633bcf9a3b5d116c2d144101850..cca58a8ad8af318671453cbfd17e55e171091ee8 100644 (file)
--- a/pdns/recursordist/Makefile.am
+++ b/pdns/recursordist/Makefile.am
@@ -134,6 +134,7 @@ pdns_recursor_SOURCES = \
        pollmplexer.cc \
        protobuf.cc protobuf.hh \
        pubsuffix.hh pubsuffix.cc \
+       pubsuffixloader.cc \
        qtype.hh qtype.cc \
        rcpgenerator.cc rcpgenerator.hh \
        rec-carbon.cc \

diff --git a/pdns/recursordist/docs/settings.rst b/pdns/recursordist/docs/settings.rst
index 31f66737def8f398fc969cde4c130c4d72dd264b..4dcec27b0fa72baa654577e0ffa2ab0b7adf4a5c 100644 (file)
--- a/pdns/recursordist/docs/settings.rst
+++ b/pdns/recursordist/docs/settings.rst
@@ -1075,6 +1075,17 @@ maximizing the cache hit ratio. Starting with version 4.2.0, more than one distr
 setting.
 Improves performance on Linux.
 
+.. _settting-public-suffix-list-file:
+
+``public-suffix-list-file``
+---------------------------
+.. versionadded:: 4.2.0
+
+- Path
+- Default: unset
+
+Path to the Public Suffix List file, if any. If set, PowerDNS will try to load the Public Suffix List from this file instead of using the built-in list. The PSL is used to group the queries by relevant domain names when displaying the top queries.
+
 .. _setting-query-local-address:
 
 ``query-local-address``

diff --git a/pdns/recursordist/pubsuffixloader.cc b/pdns/recursordist/pubsuffixloader.cc
new file mode 100644 (file)
index 0000000..1bf1f54
--- /dev/null
+++ b/pdns/recursordist/pubsuffixloader.cc
@@ -0,0 +1,84 @@
+/*
+ * This file is part of PowerDNS or dnsdist.
+ * Copyright -- PowerDNS.COM B.V. and its contributors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * In addition, for the avoidance of any doubt, permission is granted to
+ * link this program with OpenSSL and to (re)distribute the binaries
+ * produced as the result of such linking.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <fstream>
+#include <stdexcept>
+
+#include "dnsname.hh"
+#include "logger.hh"
+#include "misc.hh"
+#include "pubsuffix.hh"
+
+extern const char* g_pubsuffix[];
+std::vector<std::vector<std::string>> g_pubs;
+
+void initPublicSuffixList(const std::string& file)
+{
+  std::vector<std::vector<std::string>> pbList;
+  
+  bool loaded = false;
+  if (!file.empty()) {
+    try {
+      std::ifstream suffixFile(file);
+      if (!suffixFile.is_open()) {
+        throw std::runtime_error("Error opening the public suffix list file");
+      }
+
+      std::string line;
+      while (std::getline(suffixFile, line)) {
+        if (line.empty() || (line.rfind("//", 0) == 0)) {
+          /* skip empty and commented lines */
+          continue;
+        }
+        try {
+          DNSName name(toLower(line));
+          pbList.push_back(name.labelReverse().getRawLabels());
+        }
+        catch(...) {
+          /* not a DNS name, ignoring */
+        }
+      }
+
+      g_log<<Logger::Info<<"Loaded the Public Suffix List from '"<<file<<"'"<<endl;
+      loaded = true;
+    }
+    catch (const std::exception& e) {
+      g_log<<Logger::Warning<<"Error while loading the Public Suffix List from '"<<file<<"', falling back to the built-in list: "<<e.what()<<endl;
+    }
+  }
+
+  if (!loaded) {
+    pbList.clear();
+
+    for(const char** p = g_pubsuffix; *p; ++p) {
+      string low=toLower(*p);
+
+      vector<string> parts;
+      stringtok(parts, low, ".");
+      reverse(parts.begin(), parts.end());
+      pbList.push_back(parts);
+    }
+  }
+
+  sort(pbList.begin(), pbList.end());
+  g_pubs = std::move(pbList);
+}