diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index c746b03451..5bb5c5bc40 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -83,9 +83,14 @@ async def handle_event(self, event, **kwargs): event_data_changed = await self.handle_wildcard_event(main_host_event) if event_data_changed: # since data has changed, we check again whether it's a duplicate - if event.type == "DNS_NAME" and self.scan.ingress_module.is_incoming_duplicate(event, add=True): + if event.type == "DNS_NAME" and self.scan.ingress_module.is_incoming_duplicate( + event, add=True + ): if not event._graph_important: - return False, "it's a DNS wildcard, and its module already emitted a similar wildcard event" + return ( + False, + "it's a DNS wildcard, and its module already emitted a similar wildcard event", + ) else: self.debug( f"Event {event} was already emitted by its module, but it's graph-important so it gets a pass" diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 9d33621815..209b96eefb 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -656,8 +656,10 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte continue if parsed_url.scheme in ["http", "https"]: continue + def abort_if(e): return e.scope_distance > 0 + finding_data = {"host": str(host), "description": f"Non-HTTP URI: {parsed_url.geturl()}"} await self.report(finding_data, event, yara_rule_settings, discovery_context, abort_if=abort_if) protocol_data = {"protocol": parsed_url.scheme, "host": str(host)} diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index 5608fdf4fe..ba4226ec09 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -192,7 +192,6 @@ def __init__(self, *args, **kwargs): @special_target_type(r"^(?:RE|REGEX):(.*)") def handle_regex(self, match): pattern = match.group(1) - log.info(f"Blacklisting by custom regex: {pattern}") blacklist_regex = re.compile(pattern, re.IGNORECASE) self.blacklist_regexes.add(blacklist_regex) return [] @@ -225,6 +224,12 @@ def _hash_value(self): hosts = [str(h).encode() for h in self.sorted_hosts] return hosts + regex_patterns + def __len__(self): + return super().__len__() + len(self.blacklist_regexes) + + def __bool__(self): + return bool(len(self)) + class BBOTTarget: """ diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index 8f2a6bf91f..f5c28c3596 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -395,6 +395,7 @@ async def test_blacklist_regex(bbot_scanner, bbot_httpserver): config={"excavate": True}, debug=True, ) + assert len(scan.target.blacklist) == 2 assert scan.target.blacklist.blacklist_regexes assert {r.pattern for r in scan.target.blacklist.blacklist_regexes} == { r"evil[0-9]{3}",