exporters: Compute Suricata SIDs by a hash

This function is not perfect, but substantially faster when writing out
the list. We will have some somewhat-stable hash here which will always
return the same result per domain unless there has already been a small
collision.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/dnsbl/exporters.py b/src/dnsbl/exporters.py
index 90268022d11310653fd0f2ac357daf3e3143efee..618369f8944c5b251ae8013a4555e44103d96f1f 100644 (file)
--- a/src/dnsbl/exporters.py
+++ b/src/dnsbl/exporters.py
@@ -22,6 +22,7 @@ import abc
 import datetime
 import io
 import tarfile
+import zlib
 
 from . import util
 from .i18n import _
@@ -406,8 +407,14 @@ class SuricataExporter(TextExporter):
        """
                Export domains as a set of rules for Suricata
        """
+       def __init__(self, *args, **kwargs):
+               super().__init__(*args, **kwargs)
+
+               # Cache any used SIDs
+               self._used_sids = set()
+
        @abc.abstractproperty
-       def sid_offset(self):
+       def sid_prefix(self):
                """
                        The value that is being added to any SIDs
                """
@@ -431,20 +438,47 @@ class SuricataExporter(TextExporter):
                for domain in self.list.domains:
                        args = {
                                "list"   : "%s" % self.list,
-                               "domain" : domain.name,
-                               "sid"    : domain.id + self.sid_offset,
+                               "domain" : domain,
+                               "sid"    : self.compute_sid(domain),
                                "rev"    : self.rev,
                        }
 
                        # Write the rule
                        f.write(self.rule % args)
 
+       def compute_sid(self, domain):
+               """
+                       Implements a simple hash function for a domain.
+
+                       The hash can only use up to
+               """
+               # Convert the domain into bytes()
+               domain = domain.encode()
+
+               # Compute the CRC32 checksum
+               h = zlib.crc32(domain)
+
+               # Truncate to make space for the prefix
+               h %= 0x0fffffff
+
+               # Add the prefix
+               h |= self.sid_prefix
+
+               # Check if we have a collision, if so, increment the hash by one
+               while h in self._used_sids:
+                       h += 1
+
+               # Store the hash for the next collision check
+               self._used_sids.add(h)
+
+               return h
+
 
 class SuricataDNSExporter(SuricataExporter):
        """
                Exports the lists as a Suricata ruleset that filters DNS queries.
        """
-       sid_offset = 1000000000
+       sid_prefix = 0x10000000
 
        rule = (
                "alert dns any any -> any any ("
@@ -463,7 +497,7 @@ class SuricataHTTPExporter(SuricataExporter):
        """
                Exports the lists as a Suricata ruleset that filters HTTP requests.
        """
-       sid_offset = 2000000000
+       sid_prefix = 0x20000000
 
        rule = (
                "alert dns any any -> any any ("
@@ -482,7 +516,7 @@ class SuricataTLSExporter(SuricataExporter):
        """
                Exports the lists as a Suricata ruleset that filters TLS connections.
        """
-       sid_offset = 3000000000
+       sid_prefix = 0x30000000
 
        rule = (
                "alert dns any any -> any any ("
@@ -501,7 +535,7 @@ class SuricataQUICExporter(SuricataExporter):
        """
                Exports the lists as a Suricata ruleset that filters QUIC connections.
        """
-       sid_offset = 4000000000
+       sid_prefix = 0x40000000
 
        rule = (
                "alert dns any any -> any any ("