url parser and fqdn analyzer for ip blacklist inspection of urls

etc/saq.default.ini

@@ -1010,6 +1010,18 @@ enabled = yes
 ; use this if you are in AWS and your target is inside target network
 ssh_host = 
 
+[analysis_module_parse_url]
+; Parse URL and add FQDN observable
+module = saq.modules.url
+class = ParseURLAnalyzer
+enabled = no
+
+[analysis_module_fqdn_analyzer]
+; Add ip observables for FQDN resolutions
+module = saq.modules.dns
+class = FQDNAnalyzer
+enabled = no
+
 [analysis_module_dns_analyzer]
 module = saq.modules.asset
 class = DNSAnalyzer

lib/saq/modules/dns.py

@@ -3,14 +3,15 @@
 import csv
 import logging
 import os.path
+import socket
 
 from urllib.parse import urlparse
 
 import saq
 
 from saq.analysis import Analysis, Observable
 from saq.constants import *
-from saq.modules import SplunkAnalysisModule, splunktime_to_saqtime
+from saq.modules import AnalysisModule, SplunkAnalysisModule, splunktime_to_saqtime
 
 KEY_SOURCE_COUNT = 'src_count'
 KEY_REQUEST_BREAKDOWN = 'request_breakdown'
@@ -20,6 +21,52 @@
 KEY_REQUEST_BREAKDOWN_TOTAL_COUNT = 'total_count'
 KEY_DNS_REQUESTS = 'dns_requests'
 
+
+class FQDNAnalysis(Analysis):
+    """What IP adderss does this FQDN resolve to?"""
+
+    def initialize_details(self):
+        self.details = { 'ip_address': None,
+                         'resolution_count': None,
+                         'aliaslist': [],
+                         'all_resolutions': []}
+
+    def generate_summary(self):
+        message = f"Resolved to {self.details['ip_address']}"
+        if self.details['resolution_count'] > 1:
+            message += f", and {self.details['resolution_count']-1} other IP addresses"
+        return message
+
+class FQDNAnalyzer(AnalysisModule):
+    """What IP address does this FQDN resolve to?"""
+    # Add anything else you want to this FQDN Analyzer.
+
+    @property
+    def generated_analysis_type(self):
+        return FQDNAnalysis
+
+    @property
+    def valid_observable_types(self):
+        return F_FQDN
+
+    def execute_analysis(self, observable):
+        try:
+            _hostname, _aliaslist, ipaddrlist = socket.gethostbyname_ex(observable.value)
+            if ipaddrlist:
+                # ipaddrlist should always be a list of strings
+                analysis = self.create_analysis(observable)
+                analysis.details['resolution_count'] = len(ipaddrlist)
+                analysis.details['all_resolutions'] = ipaddrlist
+                analysis.details['aliaslist'] = _aliaslist
+                # for now, just add the first ip address
+                analysis.details['ip_address'] = ipaddrlist[0]
+                analysis.add_observable(F_IPV4, ipaddrlist[0])
+                return True
+            return False
+        except Exception as e:
+            logging.error(f"Problem resolving FQDN: {e}")
+            return False
+
 #
 # Module:   DNS Request Analysis
 # Question: Who requested DNS resolution for this FQDN?

lib/saq/modules/url.py

@@ -46,6 +46,46 @@
 KEY_PROXY = 'proxy'
 KEY_PROXY_NAME = 'proxy_name'
 
+class ParseURLAnalysis(Analysis):
+    """Add the FQDN of the URL as an observable."""
+    def initialize_details(self):
+        self.details = { 'netloc': None,
+                         'scheme': None,
+                         'path': None,
+                         'query': None,
+                         'params': None,
+                         'fragment': None }
+
+    #def generate_summary(self):
+    #    return f"Parsed: {self.details['netloc']}"
+
+class ParseURLAnalyzer(AnalysisModule):
+    """Parse the URL and add the FQDN as an observable."""
+
+    @property
+    def generated_analysis_type(self):
+        return ParseURLAnalysis
+
+    @property
+    def valid_observable_types(self):
+        return F_URL
+
+    def execute_analysis(self, observable):
+        try:
+            parsed_url = urlparse(observable.value)
+            analysis = self.create_analysis(observable)
+            analysis.details['netloc'] = parsed_url.netloc
+            analysis.details['scheme'] = parsed_url.scheme
+            analysis.details['path'] = parsed_url.path
+            analysis.details['query'] = parsed_url.query
+            analysis.details['params'] = parsed_url.params
+            analysis.details['fragment'] = parsed_url.fragment
+            analysis.add_observable(F_FQDN, parsed_url.netloc)
+            return True
+        except Exception as e:
+            logging.error(f"Problem parsing URL: {e}")
+            return False
+
 class GglsblAnalysis(Analysis):
     """URL matches against Google's SafeBrowsing List using the [gglsbl-rest](https://github.com/mlsecproject/gglsbl-rest) service.
     """