Add file option for new domain ignore list

diff --git a/pdns/recursordist/docs/nod_udr.rst b/pdns/recursordist/docs/nod_udr.rst
index 69d6bd2f194eaceb9144bc8a6a4d0e3dc71047dd..92e7ee5faa9654f5b102ce06cd8908e926ffce91 100644 (file)
--- a/pdns/recursordist/docs/nod_udr.rst
+++ b/pdns/recursordist/docs/nod_udr.rst
@@ -19,7 +19,7 @@ NOD is disabled by default, and must be enabled through the use of the following
 
 Once enabled the recursor will keep track of previously seen domains using the SBF data structure, which is periodically persisted to the directory specified in the ``new-domain-history-dir``, which defaults to /var/lib/pdns-recursor/nod.
 
-Administrators may wish to prevent certain domains or subdomains from ever triggering the NOD algorithm, in which case those domains must be added to the ``new-domain-ignore-list`` setting as a comma separated list. No domain (or subdomain of a domain) listed will be considered a newly observed domain.
+Administrators may wish to prevent certain domains or subdomains from ever triggering the NOD algorithm, in which case those domains must be added to the ``new-domain-ignore-list`` setting as a comma separated list. No domain (or subdomain of a domain) listed will be considered a newly observed domain. It is also possible to use ``new-domain-ignore-list-file`` to read a file with ignored domains, one domain per line.
 
 There are several ways to receive the information about newly observed domains:
 

diff --git a/pdns/recursordist/rec-main.cc b/pdns/recursordist/rec-main.cc
index db8fa554d245e809fbae9109fbf9e87eb2c3fe5f..f425525f06701af5fe9e80749e1d86ad44636fcb 100644 (file)
--- a/pdns/recursordist/rec-main.cc
+++ b/pdns/recursordist/rec-main.cc
@@ -859,33 +859,24 @@ static void setupNODThread(Logr::log_t log)
   }
 }
 
-static void parseNODIgnorelist(const std::string& wlist)
+static void parseIgnorelist(const std::string& wlist, SuffixMatchNode& matchNode)
 {
   vector<string> parts;
   stringtok(parts, wlist, ",; ");
   for (const auto& part : parts) {
-    g_nodDomainWL.add(DNSName(part));
+    matchNode.add(DNSName(part));
   }
 }
 
-static void parseUDRIgnorelist(const std::string& wlist)
-{
-  vector<string> parts;
-  stringtok(parts, wlist, ",; ");
-  for (const auto& part : parts) {
-    g_udrDomainWL.add(DNSName(part));
-  }
-}
-
-static void parseUDRIgnorelistFile(const std::string& fname)
+static void parseIgnorelistFile(const std::string& fname, SuffixMatchNode& matchNode)
 {
   string line;
-  std::ifstream udrIgnorelistFileStream(fname);
-  if (!udrIgnorelistFileStream) {
+  std::ifstream ignorelistFileStream(fname);
+  if (!ignorelistFileStream) {
     throw ArgException(fname + " could not be parsed");
   }
 
-  while (getline(udrIgnorelistFileStream, line)) {
+  while (getline(ignorelistFileStream, line)) {
     boost::trim_right(line);
 
     // strip everything after a #
@@ -906,7 +897,7 @@ static void parseUDRIgnorelistFile(const std::string& fname)
       line = line.substr(pos);
     }
 
-    g_udrDomainWL.add(DNSName(line));
+    matchNode.add(DNSName(line));
   }
 }
 
@@ -916,16 +907,19 @@ static void setupNODGlobal()
   g_nodEnabled = ::arg().mustDo("new-domain-tracking");
   g_nodLookupDomain = DNSName(::arg()["new-domain-lookup"]);
   g_nodLog = ::arg().mustDo("new-domain-log");
-  parseNODIgnorelist(::arg()["new-domain-whitelist"]);
-  parseNODIgnorelist(::arg()["new-domain-ignore-list"]);
+  parseIgnorelist(::arg()["new-domain-whitelist"], g_nodDomainWL);
+  parseIgnorelist(::arg()["new-domain-ignore-list"], g_nodDomainWL);
+  if (!::arg().isEmpty("new-domain-ignore-list-file"))
+    parseIgnorelistFile(::arg()["new-domain-ignore-list-file"], g_nodDomainWL);
 
   // Setup Unique DNS Response subsystem
   g_udrEnabled = ::arg().mustDo("unique-response-tracking");
   g_udrLog = ::arg().mustDo("unique-response-log");
   g_nod_pbtag = ::arg()["new-domain-pb-tag"];
   g_udr_pbtag = ::arg()["unique-response-pb-tag"];
-  parseUDRIgnorelist(::arg()["udr-ignore-list"]);
-  parseUDRIgnorelistFile(::arg()["udr-ignore-list-file"]);
+  parseIgnorelist(::arg()["udr-ignore-list"], g_udrDomainWL);
+  if (!::arg().isEmpty("udr-ignore-list-file"))
+    parseIgnorelistFile(::arg()["udr-ignore-list-file"], g_udrDomainWL);
 }
 #endif /* NOD_ENABLED */
 

diff --git a/pdns/recursordist/settings/table.py b/pdns/recursordist/settings/table.py
index 3f8ac72e9fa985f9486c49510dce4ce7c97d2a46..05fe1d5ff6ae068a1de711ea8b680f280d20a658 100644 (file)
--- a/pdns/recursordist/settings/table.py
+++ b/pdns/recursordist/settings/table.py
@@ -1795,6 +1795,23 @@ feature.
  ''',
     'versionadded': '4.5.0'
     },
+    {
+        'name' : 'ignore_list_file',
+        'section' : 'nod',
+        'oldname' : 'new-domain-ignore-list-file',
+        'type' : LType.String,
+        'default' : '',
+        'help' : 'File with a list of domains (and implicitly all subdomains) which will never be considered a new domain',
+        'doc' : '''
+This setting is a list of all domains (and implicitly all subdomains)
+that will never be considered a new domain. For example, if the domain
+'xyz123.tv' is in the list, then 'foo.bar.xyz123.tv' will never be
+considered a new domain. One use-case for the ignore list is to never
+reveal details of internal subdomains via the new-domain-lookup
+feature.
+ ''',
+    'versionadded': '5.1.0'
+    },
     {
         'name' : 'udr_ignore_list',
         'section' : 'nod',