rec: don't use a vector of string for internal pubsuffixlist

The construct

std::vector<std::string> x  { not event that many string literals };

blows up with some compilers. Worst I have seen is (with not even
8k strings): g++12 develops a resident size of 26G.

This just creates a (blank line and comments stripped) in-memory version of the file that is
fed to the same code as an external file.

Problem noted by @wojas.

Signed-off-by: Otto Moerbeek <otto.moerbeek@open-xchange.com>

diff --git a/pdns/recursordist/mkpubsuffixcc b/pdns/recursordist/mkpubsuffixcc
index e2d8958e1e8250f158561407206fe18a00fc5103..880c087bd5231ea9c1dbeb3cc42ce5f9a3aac69a 100755 (executable)
--- a/pdns/recursordist/mkpubsuffixcc
+++ b/pdns/recursordist/mkpubsuffixcc
@@ -10,9 +10,9 @@ trap "rm -f $temp" 0 1 2 3 15
 set -e
 curl -o $temp -s -S https://publicsuffix.org/list/public_suffix_list.dat
 (echo "#include \"pubsuffix.hh\""
- echo "const std::vector<std::string> g_pubsuffix = {";
+ echo "const std::string g_pubsuffix = ";
        for a in $(grep -v "//" "$temp" | grep \\. | egrep "^[.0-9a-z-]*$")
        do
-               echo \"$a\",
+               echo \"$a\\\\n\"
        done
-echo "};") > "$1"
+echo ";") > "$1"

diff --git a/pdns/recursordist/pubsuffix.hh b/pdns/recursordist/pubsuffix.hh
index 1179cec2621c93a694052fcf8a22462e68a6cdf8..d65c6af9d0dacc7332b53cb6614af9fd6c0f07d6 100644 (file)
--- a/pdns/recursordist/pubsuffix.hh
+++ b/pdns/recursordist/pubsuffix.hh
@@ -25,7 +25,7 @@
 #include <vector>
 
 extern std::vector<std::vector<std::string>> g_pubs;
-extern const std::vector<std::string> g_pubsuffix;
+extern const std::string g_pubsuffix;
 
 /* initialize the g_pubs variable with the public suffix list,
    using the file passed in parameter if any, or the built-in

diff --git a/pdns/recursordist/pubsuffixloader.cc b/pdns/recursordist/pubsuffixloader.cc
index 719151ec9d06a7f5417a61cef2d09476b202b112..667a3a4dd7c57a30790165f4edef30946c7ce6f2 100644 (file)
--- a/pdns/recursordist/pubsuffixloader.cc
+++ b/pdns/recursordist/pubsuffixloader.cc
@@ -30,44 +30,58 @@
 
 std::vector<std::vector<std::string>> g_pubs;
 
-void initPublicSuffixList(const std::string& file)
+static bool initPublicSuffixList(const std::string& file, std::istream& stream, std::vector<std::vector<std::string>>& pbList)
 {
-  std::vector<std::vector<std::string>> pbList;
 
-  bool loaded = false;
-  if (!file.empty()) {
-    try {
-      Regex reg("^[.0-9a-z-]*$");
-      std::ifstream suffixFile(file);
-      if (!suffixFile.is_open()) {
-        throw std::runtime_error("Error opening the public suffix list file");
-      }
+  try {
+    Regex reg("^[.0-9a-z-]*$");
 
-      std::string line;
-      while (std::getline(suffixFile, line)) {
-        if (line.empty() || (line.rfind("//", 0) == 0)) {
-          /* skip empty and commented lines */
+    std::string line;
+    while (std::getline(stream, line)) {
+      if (line.empty() || (line.rfind("//", 0) == 0)) {
+        /* skip empty and commented lines */
+        continue;
+      }
+      try {
+        line = toLower(line);
+        if (!reg.match(line)) {
           continue;
         }
-        try {
-          line = toLower(line);
-          if (!reg.match(line)) {
-            continue;
-          }
-          DNSName name(toLower(line));
-          if (name.countLabels() < 2) {
-            continue;
-          }
-          pbList.emplace_back(name.labelReverse().getRawLabels());
-        }
-        catch (...) {
-          /* not a DNS name, ignoring */
+        DNSName name(line);
+        if (name.countLabels() < 2) {
           continue;
         }
+        pbList.emplace_back(name.labelReverse().getRawLabels());
+      }
+      catch (...) {
+        /* not a DNS name, ignoring */
+        continue;
       }
+    }
 
+    if (file != "internal") {
       g_slog->withName("runtime")->info(Logr::Info, "Loaded the Public Suffix List", "file", Logging::Loggable(file));
-      loaded = true;
+    }
+    return true;
+  }
+  catch (const std::exception& e) {
+    g_slog->withName("runtime")->error(Logr::Error, e.what(), "Error while loading the Public Suffix List", "file", Logging::Loggable(file));
+  }
+  return false;
+}
+
+void initPublicSuffixList(const std::string& file)
+{
+  bool loaded = false;
+  std::vector<std::vector<std::string>> pbList;
+
+  if (!file.empty()) {
+    try {
+      std::ifstream suffixFile(file);
+      if (!suffixFile.is_open()) {
+        throw std::runtime_error("Error opening the public suffix list file");
+      }
+      loaded = initPublicSuffixList(file, suffixFile, pbList);
     }
     catch (const std::exception& e) {
       g_slog->withName("runtime")->error(Logr::Error, e.what(), "Error while loading the Public Suffix List", "file", Logging::Loggable(file));
@@ -76,17 +90,9 @@ void initPublicSuffixList(const std::string& file)
 
   if (!loaded) {
     pbList.clear();
-
-    for (const auto& entry : g_pubsuffix) {
-      const auto low = toLower(entry);
-      std::vector<std::string> parts;
-      stringtok(parts, low, ".");
-      std::reverse(parts.begin(), parts.end());
-      parts.shrink_to_fit();
-      pbList.emplace_back(std::move(parts));
-    }
+    std::istringstream stream(g_pubsuffix);
+    initPublicSuffixList("internal", stream, pbList);
   }
-
   std::sort(pbList.begin(), pbList.end());
   g_pubs = std::move(pbList);
 }