From f506c37c49cf53d759bf2c6f15b466344cf9c9fe Mon Sep 17 00:00:00 2001
From: Jason Ish <jason.ish@oisf.net>
Date: Wed, 27 Nov 2024 16:00:49 -0600
Subject: [PATCH] matching: consolidate sid matchers into a set matcher

Consolidate SID matchers into a single SID set matcher which stores a
dict of all SIDs to be matched. An array of many SID matchers to a
single matcher with much faster lookup.

This can reduce a many minute runtime down to 10s of seconds.

Ticket: #7415
---
 suricata/update/main.py     | 13 ++++++++++++-
 suricata/update/matchers.py | 19 +++++++++++++++++++
 tests/test_matchers.py      |  2 +-
 3 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/suricata/update/main.py b/suricata/update/main.py
index 7814518..eb761ae 100644
--- a/suricata/update/main.py
+++ b/suricata/update/main.py
@@ -278,6 +278,8 @@ def load_drop_filters(filename):
 def parse_matchers(fileobj):
     matchers = []
 
+    id_set_matcher = matchers_mod.IdSetRuleMatcher()
+
     for line in fileobj:
         line = line.strip()
         if not line or line.startswith("#"):
@@ -287,10 +289,19 @@ def parse_matchers(fileobj):
         if not matcher:
             logger.warn("Failed to parse: \"%s\"" % (line))
         else:
-            matchers.append(matcher)
+            # If matcher is an IdRuleMatcher
+            if isinstance(matcher, matchers_mod.IdRuleMatcher):
+                for (gid, sid) in matcher.signatureIds:
+                    id_set_matcher.add(gid, sid)
+            else:
+                matchers.append(matcher)
+
+    if len(id_set_matcher.sids) > 0:
+        matchers.append(id_set_matcher)
 
     return matchers
 
+
 def load_matchers(filename):
     with open(filename) as fileobj:
         return parse_matchers(fileobj)
diff --git a/suricata/update/matchers.py b/suricata/update/matchers.py
index 56a9e29..79c3866 100644
--- a/suricata/update/matchers.py
+++ b/suricata/update/matchers.py
@@ -51,6 +51,25 @@ def match(self, rule):
         return rule.proto == self.proto
 
 
+class IdSetRuleMatcher(object):
+    """Matcher object that matches on a set of rule SIDs.
+
+    This matcher is not directly parsed, but instead constructed after
+    loading individual IdRuleMatchers and consolidated into one
+    matcher that is much faster.
+
+    """
+    def __init__(self):
+        self.sids = {}
+
+    def add(self, generatorId, signatureId):
+        self.sids[(generatorId, signatureId)] = True
+
+    def match(self, rule):
+        key = (rule.gid, rule.sid)
+        return key in self.sids
+
+
 class IdRuleMatcher(object):
     """Matcher object to match an idstools rule object by its signature
     ID."""
diff --git a/tests/test_matchers.py b/tests/test_matchers.py
index ad64a54..ee360be 100644
--- a/tests/test_matchers.py
+++ b/tests/test_matchers.py
@@ -66,7 +66,7 @@ def test_trailing_comment(self):
         self.assertEqual(
             matchers[1].__class__, matchers_mod.ReRuleMatcher)
         self.assertEqual(
-            matchers[2].__class__, matchers_mod.IdRuleMatcher)
+            matchers[2].__class__, matchers_mod.IdSetRuleMatcher)
 
 class IdRuleMatcherTestCase(unittest.TestCase):