sources: Don't import anything that isn't globally resolvable

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/configure.ac b/configure.ac
index e23326136a46e0538b276a20ff3f500ef9effe60..a6db88602e52ddbd4d5a6a250a0846f4446a0a0e 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -56,6 +56,7 @@ AX_PYTHON_MODULE([dns], [fatal])
 AX_PYTHON_MODULE([babel], [fatal])
 AX_PYTHON_MODULE([fastapi], [fatal])
 AX_PYTHON_MODULE([httpx], [fatal])
+AX_PYTHON_MODULE([publicsuffix2], [fatal])
 AX_PYTHON_MODULE([rich], [fatal])
 AX_PYTHON_MODULE([sqlmodel], [fatal])
 

diff --git a/src/dnsbl/__init__.py b/src/dnsbl/__init__.py
index af1f72f72200637480b4e7e18e1e44be5ca57b08..a378b089717bd4076573448349e66a13a87ca88d 100644 (file)
--- a/src/dnsbl/__init__.py
+++ b/src/dnsbl/__init__.py
@@ -23,6 +23,7 @@ import functools
 import httpx
 import io
 import logging
+import publicsuffix2
 import sqlmodel
 
 # Initialize logging as early as possible
@@ -85,6 +86,13 @@ class Backend(object):
                        follow_redirects=True,
                )
 
+       @functools.cached_property
+       def psl(self):
+               """
+                       The Public Suffix List
+               """
+               return publicsuffix2.PublicSuffixList()
+
        @functools.cached_property
        def auth(self):
                return auth.Auth(self)

diff --git a/src/dnsbl/sources.py b/src/dnsbl/sources.py
index 86c06ceaac244bf314ede2248d0e90e4a42a1d50..a1035bffda78bf013004d35c3f5d2fadaf5c8cb9 100644 (file)
--- a/src/dnsbl/sources.py
+++ b/src/dnsbl/sources.py
@@ -283,6 +283,11 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True):
                                                                log.debug("Skipping ignored domain: %s" % domain)
                                                                continue
 
+                                                       # Is the domain actually publicly resolvable?
+                                                       if not self.backend.psl.get_tld(domain, strict=True):
+                                                               log.debug("Skipping non-public domain: %s" % domain)
+                                                               continue
+
                                                        # Add the domain
                                                        domains.add(domain)