Merge pull request #2057 from blacklanternsecurity/fix-blacklist-logging

Fix blacklist logging bug

bbot/modules/internal/dnsresolve.py

@@ -83,9 +83,14 @@ async def handle_event(self, event, **kwargs):
                     event_data_changed = await self.handle_wildcard_event(main_host_event)
                     if event_data_changed:
                         # since data has changed, we check again whether it's a duplicate
-                        if event.type == "DNS_NAME" and self.scan.ingress_module.is_incoming_duplicate(event, add=True):
+                        if event.type == "DNS_NAME" and self.scan.ingress_module.is_incoming_duplicate(
+                            event, add=True
+                        ):
                             if not event._graph_important:
-                                return False, "it's a DNS wildcard, and its module already emitted a similar wildcard event"
+                                return (
+                                    False,
+                                    "it's a DNS wildcard, and its module already emitted a similar wildcard event",
+                                )
                             else:
                                 self.debug(
                                     f"Event {event} was already emitted by its module, but it's graph-important so it gets a pass"

bbot/modules/internal/excavate.py

@@ -656,8 +656,10 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte
                         continue
                     if parsed_url.scheme in ["http", "https"]:
                         continue
+
                     def abort_if(e):
                         return e.scope_distance > 0
+
                     finding_data = {"host": str(host), "description": f"Non-HTTP URI: {parsed_url.geturl()}"}
                     await self.report(finding_data, event, yara_rule_settings, discovery_context, abort_if=abort_if)
                     protocol_data = {"protocol": parsed_url.scheme, "host": str(host)}

bbot/scanner/target.py

@@ -192,7 +192,6 @@ def __init__(self, *args, **kwargs):
     @special_target_type(r"^(?:RE|REGEX):(.*)")
     def handle_regex(self, match):
         pattern = match.group(1)
-        log.info(f"Blacklisting by custom regex: {pattern}")
         blacklist_regex = re.compile(pattern, re.IGNORECASE)
         self.blacklist_regexes.add(blacklist_regex)
         return []
@@ -225,6 +224,12 @@ def _hash_value(self):
         hosts = [str(h).encode() for h in self.sorted_hosts]
         return hosts + regex_patterns
 
+    def __len__(self):
+        return super().__len__() + len(self.blacklist_regexes)
+
+    def __bool__(self):
+        return bool(len(self))
+
 
 class BBOTTarget:
     """

bbot/test/test_step_1/test_target.py

@@ -395,6 +395,7 @@ async def test_blacklist_regex(bbot_scanner, bbot_httpserver):
         config={"excavate": True},
         debug=True,
     )
+    assert len(scan.target.blacklist) == 2
     assert scan.target.blacklist.blacklist_regexes
     assert {r.pattern for r in scan.target.blacklist.blacklist_regexes} == {
         r"evil[0-9]{3}",