WIP: Creating a Python package

.gitignore

@@ -26,3 +26,5 @@ __pycache__
 scripts/pulse-results/*.json
 .DS_Store
 venv
+build/
+dist/

gatherers/censys.py → domain_scan/gatherers/censys.py

@@ -8,8 +8,8 @@
 from google.oauth2 import service_account
 import google.api_core.exceptions
 
-from gatherers.gathererabc import Gatherer
-from utils import utils
+from domain_scan.gatherers.gathererabc import Gatherer
+from domain_scan.utils import utils
 
 # Options:
 #

gatherers/rdns.py → domain_scan/gatherers/rdns.py

@@ -3,7 +3,7 @@
 import re
 from typing import Generator, List, Pattern
 
-from gatherers.gathererabc import Gatherer
+from domain_scan.gatherers.gathererabc import Gatherer
 
 # Reverse DNS
 #

gatherers/url.py → domain_scan/gatherers/url.py

@@ -3,8 +3,8 @@
 
 import requests
 
-from gatherers.gathererabc import Gatherer
-from utils import utils
+from domain_scan.gatherers.gathererabc import Gatherer
+from domain_scan.utils import utils
 
 
 class Gatherer(Gatherer):

runner/runner.py → domain_scan/runner/runner.py

@@ -1,4 +1,4 @@
-from utils import utils
+from domain_scan.utils import utils
 
 
 def write_rows(rows, domain, base_domain, scanner, csv_writer, meta=None):

scanners/a11y.py → domain_scan/scanners/a11y.py

@@ -4,7 +4,7 @@
 import requests
 import yaml
 
-from utils import utils
+from domain_scan.utils import utils
 
 
 workers = 3

scanners/analytics.py → domain_scan/scanners/analytics.py

@@ -1,7 +1,7 @@
 import logging
 import os
 
-from utils import utils
+from domain_scan.utils import utils
 
 # Check whether a domain is present in a CSV, set in --analytics.
 

scanners/csp.py → domain_scan/scanners/csp.py

@@ -1,6 +1,6 @@
 import logging
 import requests
-from scanners import utils
+from domain_scan.scanners import utils
 
 ###
 # CSP Scanner - check the presence of CSP headers

scanners/headless/local_bridge.py → domain_scan/scanners/headless/local_bridge.py

@@ -1,7 +1,7 @@
 import logging
 import json
 
-from utils import utils
+from domain_scan.utils import utils
 
 ###
 # Local Python bridge to the JS bridge to the JS scanner.

scanners/pshtt.py → domain_scan/scanners/pshtt.py

@@ -4,7 +4,7 @@
 import re
 
 from pshtt import pshtt
-from utils import utils
+from domain_scan.utils import utils
 
 ###
 # Measure a site's HTTP behavior using DHS NCATS' pshtt tool.

scanners/sslyze.py → domain_scan/scanners/sslyze.py

@@ -27,7 +27,7 @@
 from cryptography.hazmat.primitives.serialization import Encoding
 from cryptography.hazmat.primitives.asymmetric import ec, dsa, rsa
 
-from utils import utils
+from domain_scan.utils import utils
 
 # Number of seconds to wait during sslyze connection check.
 # Not much patience here, and very willing to move on.

scanners/third_parties.py → domain_scan/scanners/third_parties.py

@@ -1,6 +1,6 @@
 import logging
 
-from utils import utils
+from domain_scan.utils import utils
 
 # Evaluate third party service usage using Chrome headless.
 

gather

@@ -9,7 +9,7 @@ import requests
 import logging
 import importlib
 
-from utils import utils
+from domain_scan.utils import utils
 
 # some metadata about the scan itself
 start_time = utils.local_now()
@@ -55,14 +55,14 @@ def run(options=None, cache_dir="./cache", results_dir="./results"):
 
         try:
             gatherer_module = importlib.import_module(
-                "gatherers.%s" % source)
+                "domain_scan.gatherers.%s" % source)
             gatherer = gatherer_module.Gatherer(suffixes, options, extra)
         except ImportError:
             # If it's not a registered module, allow it to be "hot registered"
             # as long as the user gave us a flag with that name that can be
             # used as the --url option to the URL module.
             if options.get(source):
-                gatherer_module = importlib.import_module("gatherers.url")
+                gatherer_module = importlib.import_module("domain_scan.gatherers.url")
                 extra['name'] = source
                 gatherer = gatherer_module.Gatherer(suffixes, options, extra)
             else:

lambda/lambda_handler.py

@@ -2,7 +2,7 @@
 import sys
 import logging
 
-from utils import utils
+from domain_scan.utils import utils
 
 # Central handler for all Lambda events.
 def handler(event, context):
@@ -19,7 +19,7 @@ def handler(event, context):
 
     # Might be acceptable to let this crash the module, in Lambda.
     try:
-        scanner = importlib.import_module("scanners.%s" % name)
+        scanner = importlib.import_module("domain_scan.scanners.%s" % name)
     except ImportError:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         logging.error("[%s] Scanner not found, or had an error during loading.\n\tERROR: %s\n\t%s" % (name, exc_type, exc_value))
@@ -49,4 +49,3 @@ def handler(event, context):
     # date transform functions in one place, before Amazon's built-in
     # JSON serialization prepares the data for transport.
     return utils.from_json(utils.json_for(response))
-

scan

@@ -16,9 +16,9 @@ import boto3
 import botocore
 from concurrent.futures import ThreadPoolExecutor
 
-from scanners.headless.local_bridge import headless_scan
-from utils import utils
-from runner import runner
+from domain_scan.scanners.headless.local_bridge import headless_scan
+from domain_scan.utils import utils
+from domain_scan.runner import runner
 
 
 # Default and maximum for local workers (threads) per-scanner.
@@ -130,7 +130,7 @@ def run(options=None):
 
     for name in options.get("scan").split(","):
         try:
-            scanner = importlib.import_module("scanners.%s" % name)
+            scanner = importlib.import_module("domain_scan.scanners.%s" % name)
         except ImportError:
             exc_type, exc_value, exc_traceback = sys.exc_info()
             logging.error("[%s] Scanner not found, or had an error during loading.\n\tERROR: %s\n\t%s" % (name, exc_type, exc_value))

setup.cfg

@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal = true

setup.py

@@ -0,0 +1,84 @@
+"""
+setup module for domain-scan
+
+Based on:
+
+- https://packaging.python.org/distributing/
+- https://github.com/pypa/sampleproject/blob/master/setup.py
+- https://github.com/dhs-ncats/pshtt/blob/master/setup.py
+"""
+
+from setuptools import setup, find_packages
+
+setup(
+    name='domain-scan',
+
+    # Versions should comply with PEP440
+    version='0.1.0-dev1',
+    description='lightweight scan pipeline for orchestrating third party tools, at scale and (optionally) using serverless infrastructure',
+
+    # NCATS "homepage"
+    url='https://18f.gsa.gov',
+    # The project's main homepage
+    download_url='https://github.com/18F/domain-scan',
+
+    # Author details
+    author='GSA 18F',
+    author_email='[email protected]',
+
+    license='License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication',
+
+    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
+    classifiers=[
+        # How mature is this project? Common values are
+        #   3 - Alpha
+        #   4 - Beta
+        #   5 - Production/Stable
+        'Development Status :: 4 - Beta',
+
+        # Indicate who your project is intended for
+        'Intended Audience :: Developers',
+
+        # Pick your license as you wish (should match "license" above)
+        'License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication',
+
+        # Specify the Python versions you support here. In particular, ensure
+        # that you indicate whether you support Python 2, Python 3 or both.
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+    ],
+
+    # What does your project relate to?
+    keywords='https best practices web-crawling domain scanning',
+
+    packages=find_packages(),
+
+    install_requires=[
+        'strict-rfc3339',
+        'publicsuffix',
+        'boto3',
+        'ipython',
+        'sslyze>=1.3.4,<1.4.0',
+        'cryptography',
+        'pyyaml',
+        'requests',
+        'google-cloud-bigquery',
+        'google-auth-oauthlib'
+    ],
+
+    extras_require={
+        'test': [
+            'pytest'
+        ],
+    },
+
+    # Conveniently allows one to run the CLI scripts
+    scripts=[
+        'gather',
+        'scan',
+    ]
+)

tests/test_utils.py

@@ -2,7 +2,7 @@
 import sys
 import pytest
 from .context import utils  # noqa
-from utils import utils as subutils
+from domain_scan.utils import utils as subutils
 
 
 def get_default_false_values(parser):