Merge pull request #67 from seanmcfeely/master

ParseURLAnalyzer && FQDNAnalyzer && more configurable IPIAnalyzer

etc/saq.default.ini

@@ -873,6 +873,8 @@ use_proxy = yes
 #>>> FIELDS
 #['IP', 'ASN', 'ORG', 'Continent', 'Country', 'Region', 'City', 'Time Zone', 'Latitude', 'Longitude', 'Accuracy Radius']
 tag_list = Country
+; Point to a local YAML or JSON config for any customizations 
+override_config_path = 
 
 [analysis_module_mailbox_email_analyzer]
 module = saq.modules.email
@@ -1008,6 +1010,18 @@ enabled = yes
 ; use this if you are in AWS and your target is inside target network
 ssh_host = 
 
+[analysis_module_parse_url]
+; Parse URL and add FQDN observable
+module = saq.modules.url
+class = ParseURLAnalyzer
+enabled = no
+
+[analysis_module_fqdn_analyzer]
+; Add ip observables for FQDN resolutions
+module = saq.modules.dns
+class = FQDNAnalyzer
+enabled = no
+
 [analysis_module_dns_analyzer]
 module = saq.modules.asset
 class = DNSAnalyzer

lib/saq/modules/dns.py

@@ -3,14 +3,15 @@
 import csv
 import logging
 import os.path
+import socket
 
 from urllib.parse import urlparse
 
 import saq
 
 from saq.analysis import Analysis, Observable
 from saq.constants import *
-from saq.modules import SplunkAnalysisModule, splunktime_to_saqtime
+from saq.modules import AnalysisModule, SplunkAnalysisModule, splunktime_to_saqtime
 
 KEY_SOURCE_COUNT = 'src_count'
 KEY_REQUEST_BREAKDOWN = 'request_breakdown'
@@ -20,6 +21,52 @@
 KEY_REQUEST_BREAKDOWN_TOTAL_COUNT = 'total_count'
 KEY_DNS_REQUESTS = 'dns_requests'
 
+
+class FQDNAnalysis(Analysis):
+    """What IP adderss does this FQDN resolve to?"""
+
+    def initialize_details(self):
+        self.details = { 'ip_address': None,
+                         'resolution_count': None,
+                         'aliaslist': [],
+                         'all_resolutions': []}
+
+    def generate_summary(self):
+        message = f"Resolved to {self.details['ip_address']}"
+        if self.details['resolution_count'] > 1:
+            message += f", and {self.details['resolution_count']-1} other IP addresses"
+        return message
+
+class FQDNAnalyzer(AnalysisModule):
+    """What IP address does this FQDN resolve to?"""
+    # Add anything else you want to this FQDN Analyzer.
+
+    @property
+    def generated_analysis_type(self):
+        return FQDNAnalysis
+
+    @property
+    def valid_observable_types(self):
+        return F_FQDN
+
+    def execute_analysis(self, observable):
+        try:
+            _hostname, _aliaslist, ipaddrlist = socket.gethostbyname_ex(observable.value)
+            if ipaddrlist:
+                # ipaddrlist should always be a list of strings
+                analysis = self.create_analysis(observable)
+                analysis.details['resolution_count'] = len(ipaddrlist)
+                analysis.details['all_resolutions'] = ipaddrlist
+                analysis.details['aliaslist'] = _aliaslist
+                # for now, just add the first ip address
+                analysis.details['ip_address'] = ipaddrlist[0]
+                analysis.add_observable(F_IPV4, ipaddrlist[0])
+                return True
+            return False
+        except Exception as e:
+            logging.error(f"Problem resolving FQDN: {e}")
+            return False
+
 #
 # Module:   DNS Request Analysis
 # Question: Who requested DNS resolution for this FQDN?

lib/saq/modules/ip_address.py

@@ -1,8 +1,9 @@
-
+import os
 import sys
 import logging
 
 from ip_inspector import maxmind
+from ip_inspector.config import load as load_ipi_config
 from ip_inspector import Inspector, Inspected_IP
 
 import saq
@@ -110,6 +111,10 @@ class IPIAnalyzer(AnalysisModule):
     """Lookup an IP address in MaxMind's free GeoLite2 databases and wrap those results around a whitelist/blacklist check.
     """
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.__ipi_config = None
+
     @property
     def generated_analysis_type(self):
         return IpInspectorAnalysis
@@ -127,6 +132,54 @@ def tag_list(self):
         tag_list = self.config['tag_list']
         return tag_list.split(',')
 
+    @property
+    def override_config_path(self):
+        if 'override_config_path' not in self.config:
+            logging.warning("Missing expected default config field.")
+            return False
+        ocp = self.config['override_config_path']
+        if not ocp:
+            # value not set
+            return None
+        if os.path.exists(ocp):
+            return ocp
+        ocp = os.path.join(saq.SAQ_HOME, ocp)
+        if os.path.exists(ocp):
+            return ocp
+        logging.warning("Can't find '{}'".format(self.config['override_config_path']))
+        return False
+
+    @property
+    def ipi_config(self):
+        if not self.__ipi_config:
+            if self.override_config_path:
+                self.__ipi_config = load_ipi_config(saved_config_path=self.override_config_path)
+            else:
+                self.__ipi_config = load_ipi_config()
+        return self.__ipi_config
+
+    @property
+    def blacklist_maps(self):
+        _bl_map = {}
+        for bl_type, bl_path in self.ipi_config['default']['blacklists'].items():
+            _bl_map[bl_type] = bl_path
+            if os.path.exists(bl_path):
+                continue
+            if os.path.exists(os.path.join(saq.SAQ_HOME, bl_path)):
+                _bl_map[bl_type] = os.path.exists(os.path.join(saq.SAQ_HOME, bl_path))
+        return _bl_map
+
+    @property
+    def whitelist_maps(self):
+        _bl_map = {}
+        for bl_type, bl_path in self.ipi_config['default']['whitelists'].items():
+            _bl_map[bl_type] = bl_path
+            if os.path.exists(bl_path):
+                continue
+            if os.path.exists(os.path.join(saq.SAQ_HOME, bl_path)):
+                _bl_map[bl_type] = os.path.exists(os.path.join(saq.SAQ_HOME, bl_path))
+        return _bl_map
+
     @property
     def use_proxy(self):
         return self.config['use_proxy']
@@ -141,7 +194,9 @@ def execute_analysis(self, observable):
         try:
             proxies = saq.PROXIES if self.use_proxy else None
             # Create Inspector with MaxMind API
-            mmi = Inspector(maxmind.Client(license_key=self.license_key, proxies=proxies))
+            mmi = Inspector(maxmind.Client(license_key=self.license_key, proxies=proxies),
+                            blacklists=self.blacklist_maps,
+                            whitelists=self.whitelist_maps)
         except Exception as e:
             logging.error("Failed to create MaxMind Inspector: {}".format(e))
             return False

lib/saq/modules/url.py

@@ -46,6 +46,46 @@
 KEY_PROXY = 'proxy'
 KEY_PROXY_NAME = 'proxy_name'
 
+class ParseURLAnalysis(Analysis):
+    """Add the FQDN of the URL as an observable."""
+    def initialize_details(self):
+        self.details = { 'netloc': None,
+                         'scheme': None,
+                         'path': None,
+                         'query': None,
+                         'params': None,
+                         'fragment': None }
+
+    #def generate_summary(self):
+    #    return f"Parsed: {self.details['netloc']}"
+
+class ParseURLAnalyzer(AnalysisModule):
+    """Parse the URL and add the FQDN as an observable."""
+
+    @property
+    def generated_analysis_type(self):
+        return ParseURLAnalysis
+
+    @property
+    def valid_observable_types(self):
+        return F_URL
+
+    def execute_analysis(self, observable):
+        try:
+            parsed_url = urlparse(observable.value)
+            analysis = self.create_analysis(observable)
+            analysis.details['netloc'] = parsed_url.netloc
+            analysis.details['scheme'] = parsed_url.scheme
+            analysis.details['path'] = parsed_url.path
+            analysis.details['query'] = parsed_url.query
+            analysis.details['params'] = parsed_url.params
+            analysis.details['fragment'] = parsed_url.fragment
+            analysis.add_observable(F_FQDN, parsed_url.netloc)
+            return True
+        except Exception as e:
+            logging.error(f"Problem parsing URL: {e}")
+            return False
+
 class GglsblAnalysis(Analysis):
     """URL matches against Google's SafeBrowsing List using the [gglsbl-rest](https://github.com/mlsecproject/gglsbl-rest) service.
     """