matching: consolidate sid matchers into a set matcher

Consolidate SID matchers into a single SID set matcher which stores a
dict of all SIDs to be matched. An array of many SID matchers to a
single matcher with much faster lookup.

This can reduce a many minute runtime down to 10s of seconds.

Ticket: #7415

diff --git a/suricata/update/main.py b/suricata/update/main.py
index 78145184739c3454ca15efaecc2e856d0f505a8a..eb761ae502c2be3284997670568c950a5273c117 100644 (file)
--- a/suricata/update/main.py
+++ b/suricata/update/main.py
@@ -278,6 +278,8 @@ def load_drop_filters(filename):
 def parse_matchers(fileobj):
     matchers = []
 
+    id_set_matcher = matchers_mod.IdSetRuleMatcher()
+
     for line in fileobj:
         line = line.strip()
         if not line or line.startswith("#"):
@@ -287,10 +289,19 @@ def parse_matchers(fileobj):
         if not matcher:
             logger.warn("Failed to parse: \"%s\"" % (line))
         else:
-            matchers.append(matcher)
+            # If matcher is an IdRuleMatcher
+            if isinstance(matcher, matchers_mod.IdRuleMatcher):
+                for (gid, sid) in matcher.signatureIds:
+                    id_set_matcher.add(gid, sid)
+            else:
+                matchers.append(matcher)
+
+    if len(id_set_matcher.sids) > 0:
+        matchers.append(id_set_matcher)
 
     return matchers
 
+
 def load_matchers(filename):
     with open(filename) as fileobj:
         return parse_matchers(fileobj)

diff --git a/suricata/update/matchers.py b/suricata/update/matchers.py
index 56a9e294e038a60ecd3084b19751fb4d4c8b4cde..79c38663089d3d827ae5de65ab198cddaed1ee72 100644 (file)
--- a/suricata/update/matchers.py
+++ b/suricata/update/matchers.py
@@ -51,6 +51,25 @@ class ProtoRuleMatcher:
         return rule.proto == self.proto
 
 
+class IdSetRuleMatcher(object):
+    """Matcher object that matches on a set of rule SIDs.
+
+    This matcher is not directly parsed, but instead constructed after
+    loading individual IdRuleMatchers and consolidated into one
+    matcher that is much faster.
+
+    """
+    def __init__(self):
+        self.sids = {}
+
+    def add(self, generatorId, signatureId):
+        self.sids[(generatorId, signatureId)] = True
+
+    def match(self, rule):
+        key = (rule.gid, rule.sid)
+        return key in self.sids
+
+
 class IdRuleMatcher(object):
     """Matcher object to match an idstools rule object by its signature
     ID."""

diff --git a/tests/test_matchers.py b/tests/test_matchers.py
index ad64a540283b867fdd556de7730d5ac584406ef4..ee360be8633a87b4e5d5337f0bb02e09c9db4cf7 100644 (file)
--- a/tests/test_matchers.py
+++ b/tests/test_matchers.py
@@ -66,7 +66,7 @@ re:.# This is a comment*
         self.assertEqual(
             matchers[1].__class__, matchers_mod.ReRuleMatcher)
         self.assertEqual(
-            matchers[2].__class__, matchers_mod.IdRuleMatcher)
+            matchers[2].__class__, matchers_mod.IdSetRuleMatcher)
 
 class IdRuleMatcherTestCase(unittest.TestCase):