From: Ensar Sarajčić <dev@ensarsarajcic.com>
Date: Wed, 5 Jun 2024 13:16:53 +0000 (+0200)
Subject: Add file option for new domain ignore list
X-Git-Tag: rec-5.2.0-alpha0~19^2~13
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=000c969985060672ef81adf6de88d2585fefb995;p=thirdparty%2Fpdns.git

Add file option for new domain ignore list
---

diff --git a/pdns/recursordist/docs/nod_udr.rst b/pdns/recursordist/docs/nod_udr.rst
index 69d6bd2f19..92e7ee5faa 100644
--- a/pdns/recursordist/docs/nod_udr.rst
+++ b/pdns/recursordist/docs/nod_udr.rst
@@ -19,7 +19,7 @@ NOD is disabled by default, and must be enabled through the use of the following
 
 Once enabled the recursor will keep track of previously seen domains using the SBF data structure, which is periodically persisted to the directory specified in the ``new-domain-history-dir``, which defaults to /var/lib/pdns-recursor/nod.
 
-Administrators may wish to prevent certain domains or subdomains from ever triggering the NOD algorithm, in which case those domains must be added to the ``new-domain-ignore-list`` setting as a comma separated list. No domain (or subdomain of a domain) listed will be considered a newly observed domain.
+Administrators may wish to prevent certain domains or subdomains from ever triggering the NOD algorithm, in which case those domains must be added to the ``new-domain-ignore-list`` setting as a comma separated list. No domain (or subdomain of a domain) listed will be considered a newly observed domain. It is also possible to use ``new-domain-ignore-list-file`` to read a file with ignored domains, one domain per line.
 
 There are several ways to receive the information about newly observed domains:
 
diff --git a/pdns/recursordist/rec-main.cc b/pdns/recursordist/rec-main.cc
index db8fa554d2..f425525f06 100644
--- a/pdns/recursordist/rec-main.cc
+++ b/pdns/recursordist/rec-main.cc
@@ -859,33 +859,24 @@ static void setupNODThread(Logr::log_t log)
   }
 }
 
-static void parseNODIgnorelist(const std::string& wlist)
+static void parseIgnorelist(const std::string& wlist, SuffixMatchNode& matchNode)
 {
   vector<string> parts;
   stringtok(parts, wlist, ",; ");
   for (const auto& part : parts) {
-    g_nodDomainWL.add(DNSName(part));
+    matchNode.add(DNSName(part));
   }
 }
 
-static void parseUDRIgnorelist(const std::string& wlist)
-{
-  vector<string> parts;
-  stringtok(parts, wlist, ",; ");
-  for (const auto& part : parts) {
-    g_udrDomainWL.add(DNSName(part));
-  }
-}
-
-static void parseUDRIgnorelistFile(const std::string& fname)
+static void parseIgnorelistFile(const std::string& fname, SuffixMatchNode& matchNode)
 {
   string line;
-  std::ifstream udrIgnorelistFileStream(fname);
-  if (!udrIgnorelistFileStream) {
+  std::ifstream ignorelistFileStream(fname);
+  if (!ignorelistFileStream) {
     throw ArgException(fname + " could not be parsed");
   }
 
-  while (getline(udrIgnorelistFileStream, line)) {
+  while (getline(ignorelistFileStream, line)) {
     boost::trim_right(line);
 
     // strip everything after a #
@@ -906,7 +897,7 @@ static void parseUDRIgnorelistFile(const std::string& fname)
       line = line.substr(pos);
     }
 
-    g_udrDomainWL.add(DNSName(line));
+    matchNode.add(DNSName(line));
   }
 }
 
@@ -916,16 +907,19 @@ static void setupNODGlobal()
   g_nodEnabled = ::arg().mustDo("new-domain-tracking");
   g_nodLookupDomain = DNSName(::arg()["new-domain-lookup"]);
   g_nodLog = ::arg().mustDo("new-domain-log");
-  parseNODIgnorelist(::arg()["new-domain-whitelist"]);
-  parseNODIgnorelist(::arg()["new-domain-ignore-list"]);
+  parseIgnorelist(::arg()["new-domain-whitelist"], g_nodDomainWL);
+  parseIgnorelist(::arg()["new-domain-ignore-list"], g_nodDomainWL);
+  if (!::arg().isEmpty("new-domain-ignore-list-file"))
+    parseIgnorelistFile(::arg()["new-domain-ignore-list-file"], g_nodDomainWL);
 
   // Setup Unique DNS Response subsystem
   g_udrEnabled = ::arg().mustDo("unique-response-tracking");
   g_udrLog = ::arg().mustDo("unique-response-log");
   g_nod_pbtag = ::arg()["new-domain-pb-tag"];
   g_udr_pbtag = ::arg()["unique-response-pb-tag"];
-  parseUDRIgnorelist(::arg()["udr-ignore-list"]);
-  parseUDRIgnorelistFile(::arg()["udr-ignore-list-file"]);
+  parseIgnorelist(::arg()["udr-ignore-list"], g_udrDomainWL);
+  if (!::arg().isEmpty("udr-ignore-list-file"))
+    parseIgnorelistFile(::arg()["udr-ignore-list-file"], g_udrDomainWL);
 }
 #endif /* NOD_ENABLED */
 
diff --git a/pdns/recursordist/settings/table.py b/pdns/recursordist/settings/table.py
index 3f8ac72e9f..05fe1d5ff6 100644
--- a/pdns/recursordist/settings/table.py
+++ b/pdns/recursordist/settings/table.py
@@ -1795,6 +1795,23 @@ feature.
  ''',
     'versionadded': '4.5.0'
     },
+    {
+        'name' : 'ignore_list_file',
+        'section' : 'nod',
+        'oldname' : 'new-domain-ignore-list-file',
+        'type' : LType.String,
+        'default' : '',
+        'help' : 'File with a list of domains (and implicitly all subdomains) which will never be considered a new domain',
+        'doc' : '''
+This setting is a list of all domains (and implicitly all subdomains)
+that will never be considered a new domain. For example, if the domain
+'xyz123.tv' is in the list, then 'foo.bar.xyz123.tv' will never be
+considered a new domain. One use-case for the ignore list is to never
+reveal details of internal subdomains via the new-domain-lookup
+feature.
+ ''',
+    'versionadded': '5.1.0'
+    },
     {
         'name' : 'udr_ignore_list',
         'section' : 'nod',