matching: consolidate sid matchers into a set matcher

Consolidate SID matchers into a single SID set matcher which stores a
dict of all SIDs to be matched. An array of many SID matchers to a
single matcher with much faster lookup.

This can reduce a many minute runtime down to 10s of seconds.

Ticket: #7415

suricata/update/main.py

@@ -278,6 +278,8 @@ def load_drop_filters(filename):
 def parse_matchers(fileobj):
     matchers = []
 
+    id_set_matcher = matchers_mod.IdSetRuleMatcher()
+
     for line in fileobj:
         line = line.strip()
         if not line or line.startswith("#"):
@@ -287,10 +289,19 @@ def parse_matchers(fileobj):
         if not matcher:
             logger.warn("Failed to parse: \"%s\"" % (line))
         else:
-            matchers.append(matcher)
+            # If matcher is an IdRuleMatcher
+            if isinstance(matcher, matchers_mod.IdRuleMatcher):
+                for (gid, sid) in matcher.signatureIds:
+                    id_set_matcher.add(gid, sid)
+            else:
+                matchers.append(matcher)
+
+    if len(id_set_matcher.sids) > 0:
+        matchers.append(id_set_matcher)
 
     return matchers
 
+
 def load_matchers(filename):
     with open(filename) as fileobj:
         return parse_matchers(fileobj)

suricata/update/matchers.py

@@ -51,6 +51,25 @@ def match(self, rule):
         return rule.proto == self.proto
 
 
+class IdSetRuleMatcher(object):
+    """Matcher object that matches on a set of rule SIDs.
+
+    This matcher is not directly parsed, but instead constructed after
+    loading individual IdRuleMatchers and consolidated into one
+    matcher that is much faster.
+
+    """
+    def __init__(self):
+        self.sids = {}
+
+    def add(self, generatorId, signatureId):
+        self.sids[(generatorId, signatureId)] = True
+
+    def match(self, rule):
+        key = (rule.gid, rule.sid)
+        return key in self.sids
+
+
 class IdRuleMatcher(object):
     """Matcher object to match an idstools rule object by its signature
     ID."""

tests/test_matchers.py

@@ -66,7 +66,7 @@ def test_trailing_comment(self):
         self.assertEqual(
             matchers[1].__class__, matchers_mod.ReRuleMatcher)
         self.assertEqual(
-            matchers[2].__class__, matchers_mod.IdRuleMatcher)
+            matchers[2].__class__, matchers_mod.IdSetRuleMatcher)
 
 class IdRuleMatcherTestCase(unittest.TestCase):