From 9edc1f0b92475a160524c8a4852f6682e7b03a7b Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Sat, 14 Nov 2020 21:10:49 +0530 Subject: [PATCH 01/38] layout some init. groundwork for rewrite (#33) --- .github/workflows/pull_request_automation.yml | 2 +- README.md | 76 ++++++------ cli.py | 41 +++++++ cli/__init__.py | 16 +++ cli/_utils.py | 7 ++ cli/analyse.py | 114 ++++++++++++++++++ cli/commands.py | 10 ++ cli/config.py | 10 ++ cli/jobs.py | 14 +++ cli/tags.py | 9 ++ pyintelowl/pyintelowl.py | 51 +++++--- setup.py | 53 +++++++- 12 files changed, 345 insertions(+), 58 deletions(-) create mode 100644 cli.py create mode 100644 cli/__init__.py create mode 100644 cli/_utils.py create mode 100644 cli/analyse.py create mode 100644 cli/commands.py create mode 100644 cli/config.py create mode 100644 cli/jobs.py create mode 100644 cli/tags.py diff --git a/.github/workflows/pull_request_automation.yml b/.github/workflows/pull_request_automation.yml index 0db2799..416e88c 100644 --- a/.github/workflows/pull_request_automation.yml +++ b/.github/workflows/pull_request_automation.yml @@ -18,7 +18,7 @@ jobs: run: | sudo apt-get update pip3 install --upgrade pip - pip3 install -r test-requirements.txt + pip install -e .[test] - name: Black formatter run: | diff --git a/README.md b/README.md index 977c12e..c4aff43 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,50 @@ -# pyintelowl +# PyIntelOwl [![PyPI version](https://badge.fury.io/py/pyintelowl.svg)](https://badge.fury.io/py/pyintelowl) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/intelowlproject/pyintelowl.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/mlodic/pyintelowl/context:python) [![CodeFactor](https://www.codefactor.io/repository/github/intelowlproject/pyintelowl/badge)](https://www.codefactor.io/repository/github/intelowlproject/pyintelowl) -Simple Client for the [Intel Owl Project](https://github.com/intelowlproject/IntelOwl) - -2 ways to use it: -* as a library -* as a command line script +Robust Python **SDK** and **Command Line Client** for interacting with [IntelOwl](https://github.com/intelowlproject/IntelOwl)'s API. You can select which analyzers you want to run for every analysis you perform. For additional help, we suggest to check the ["How to use pyintelowl" Youtube Video](https://www.youtube.com/watch?v=fpd6Kt9EZdI) by [Kostas](https://github.com/tsale). -## Generate API key -You need a valid API key to interact with the IntelOwl server. -Keys should be created from the admin interface of [IntelOwl](https://github.com/intelowlproject/intelowl): you have to go in the *Durin* section (click on `Auth tokens`) and generate a key there. +## Installation -You can use the with the parameter `-k ` from CLI +```bash +$ pip3 install pyintelowl +``` -#### (old auth method) JWT Token Authentication -> this auth was available in IntelOwl versions <1.8.0 and pyintelowl versions <2.0.0 +For development/testing, `pip3 install pyintelowl[dev]` -From the admin interface of IntelOwl, you have to go in the *Outstanding tokens* section and generate a token there. +## Usage + +### As Command Line Client + +```bash +$ python3 cli.py -h +Usage: cli.py [OPTIONS] COMMAND [ARGS]... + +Options: + -k, --api-key TEXT API key to authenticate against a IntelOwl instance + [required] + + -u, --instance-url TEXT IntelOwl instance URL [required] + -c, --certificate PATH Path to SSL client certificate file (.pem) + --debug / --no-debug Set log level to DEBUG + -h, --help Show this message and exit. -You can use it by pasting it into the file [api_token.txt](api_token.txt). +Commands: + analyse Send new analysis request + config Set or view config variables + get-analyzer-config Get current state of analyzer_config.json from the IntelOwl instance + jobs List jobs + tags List tags +``` -## Library -`pip3 install pyintelowl` +## As a library / SDK `from pyintelowl.pyintelowl import IntelOwl` @@ -45,28 +60,17 @@ You can use it by pasting it into the file [api_token.txt](api_token.txt). `get_analyzer_configs` -> get the analyzers configuration -## Command line Client +## FAQ -#### Help - -`python3 intel_owl_client.py -h` - -#### Analyze -2 Submodules: `file` and `observable` - -##### Sample -Example: - -`python3 intel_owl_client.py -k -i -a PE_Info -a File_Info file -f ` - -Run all available analyzers (some of them could fail if you did not implemented the required configuration in the IntelOwl server): +### Generate API key +You need a valid API key to interact with the IntelOwl server. +Keys should be created from the admin interface of [IntelOwl](https://github.com/intelowlproject/intelowl): you have to go in the *Durin* section (click on `Auth tokens`) and generate a key there. -`python3 intel_owl_client.py -k -i -aa file -f ` +You can use the with the parameter `-k ` from CLI -##### Observable -Example: +#### (old auth method) JWT Token Authentication +> this auth was available in IntelOwl versions <1.8.0 and pyintelowl versions <2.0.0 -`python3 intel_owl_client.py -k -i -a AbuseIPDB -a OTXQuery observable -v google.com` +From the admin interface of IntelOwl, you have to go in the *Outstanding tokens* section and generate a token there. -#### Get Analyzers Configuration -`python3 intel_owl_client.py -k -i -gc` +You can use it by pasting it into the file [api_token.txt](api_token.txt). \ No newline at end of file diff --git a/cli.py b/cli.py new file mode 100644 index 0000000..b2364d3 --- /dev/null +++ b/cli.py @@ -0,0 +1,41 @@ +import click +from pyintelowl.pyintelowl import IntelOwl +from cli import groups, cmds + + +@click.group(context_settings=dict(help_option_names=["-h", "--help"])) +@click.option( + "-k", + "--api-key", + required=True, + default="", + help="API key to authenticate against a IntelOwl instance", +) +@click.option( + "-u", + "--instance-url", + required=True, + default="http://localhost:80", + help="IntelOwl's instance URL", +) +@click.option( + "-c", + "--certificate", + required=False, + type=click.Path(exists=True), + help="Path to SSL client certificate file (.pem)", +) +@click.option("--debug/--no-debug", default=False, help="Set log level to DEBUG") +@click.pass_context +def cli(ctx, api_key, instance_url, certificate, debug): + ctx.obj: "IntelOwl" = IntelOwl(api_key, instance_url, certificate, debug) + + +# Compile all groups and commands +for c in groups + cmds: + cli.add_command(c) + + +# Entrypoint/executor +if __name__ == "__main__": + cli() diff --git a/cli/__init__.py b/cli/__init__.py new file mode 100644 index 0000000..c4175eb --- /dev/null +++ b/cli/__init__.py @@ -0,0 +1,16 @@ +from .analyse import analyse +from .jobs import jobs +from .tags import tags +from .config import config +from .commands import get_analyzer_config + +groups = [ + analyse, + jobs, + tags, + config, +] + +cmds = [get_analyzer_config] + +__all__ = [groups, cmds] diff --git a/cli/_utils.py b/cli/_utils.py new file mode 100644 index 0000000..abbef7c --- /dev/null +++ b/cli/_utils.py @@ -0,0 +1,7 @@ +def add_options(options): + def _add_options(func): + for option in reversed(options): + func = option(func) + return func + + return _add_options diff --git a/cli/analyse.py b/cli/analyse.py new file mode 100644 index 0000000..c6497a6 --- /dev/null +++ b/cli/analyse.py @@ -0,0 +1,114 @@ +import click +from ._utils import add_options + +__analyse_options = [ + click.option( + "-al", + "--analyzers-list", + multiple=True, + type=str, + default=(), + help=""" + List of analyzer names to invoke. Should not be used with + --run-all-available-analyzers + """, + ), + click.option( + "-aa", + "--run-all-available-analyzers", + is_flag=True, + help=""" + Run all available and compatible analyzers. Should not be used with + --analyzers-list. + """, + ), + click.option( + "-fp", + "--force-privacy", + is_flag=True, + help="Disable analyzers that could impact privacy", + ), + click.option( + "-p", + "--private-job", + is_flag=True, + help="Limit view permissions to my group", + ), + click.option( + "-de", + "--disable-external-analyzers", + is_flag=True, + help="Disable analyzers that use external services", + ), + click.option( + "-c", + "--check", + type=click.Choice(["reported", "running", "force-new"], case_sensitive=False), + default="reported", + show_default=True, + help="""\n + 1. 'reported': analysis won't be repeated if already exists as running or failed.\n + 2. 'running': analysis won't be repeated if already running.\n + 3. 'force_new': force new analysis + """, + ), +] + + +@click.group("analyse") +def analyse(): + """ + Send new analysis request + """ + pass + + +@analyse.command(short_help="Send analysis request for an observable") +@click.argument("value") +@add_options(__analyse_options) +@click.pass_context +def observable( + ctx, + value, + analyzers_list, + run_all_available_analyzers, + force_privacy, + private_job, + disable_external_analyzers, + check, +): + if not analyzers_list: + if not run_all_available_analyzers: + click.echo( + """ + One of --analyzers-list, + --run-all-available-analyzers should be specified + """, + err=True, + color="RED", + ) + ans, errs = ctx.obj.send_observable_analysis_request( + analyzers_requested=analyzers_list, + observable_name=value, + force_privacy=force_privacy, + private_job=private_job, + disable_external_analyzers=disable_external_analyzers, + run_all_available_analyzers=run_all_available_analyzers, + ) + + +@analyse.command(short_help="Send analysis request for a file") +@click.argument("filename", type=click.Path(exists=True)) +@add_options(__analyse_options) +@click.pass_context +def file( + ctx, + filename, + analyzers_list, + run_all_available_analyzers, + force_privacy, + private_job, + disable_external_analyzers, + check, +): + pass diff --git a/cli/commands.py b/cli/commands.py new file mode 100644 index 0000000..84d39a0 --- /dev/null +++ b/cli/commands.py @@ -0,0 +1,10 @@ +import click + + +@click.command( + short_help="Get current state of `analyzer_config.json` from the IntelOwl instance" +) +@click.pass_context +def get_analyzer_config(ctx): + res = ctx.obj.get_analyzer_configs() + print(res) diff --git a/cli/config.py b/cli/config.py new file mode 100644 index 0000000..83aed5a --- /dev/null +++ b/cli/config.py @@ -0,0 +1,10 @@ +import click + + +@click.group("config", invoke_without_command=True) +@click.pass_context +def config(): + """ + Set or view config variables + """ + pass diff --git a/cli/jobs.py b/cli/jobs.py new file mode 100644 index 0000000..0516a1d --- /dev/null +++ b/cli/jobs.py @@ -0,0 +1,14 @@ +import click + + +@click.group("jobs", invoke_without_command=True) +def jobs(): + """ + List jobs + """ + pass + + +@jobs.command(short_help="HTTP Poll for a job who's status is `running`") +def poll(): + pass diff --git a/cli/tags.py b/cli/tags.py new file mode 100644 index 0000000..29577c0 --- /dev/null +++ b/cli/tags.py @@ -0,0 +1,9 @@ +import click + + +@click.group("tags", invoke_without_command=True) +def tags(): + """ + List tags + """ + pass diff --git a/pyintelowl/pyintelowl.py b/pyintelowl/pyintelowl.py index bfcd8d9..9f619b6 100644 --- a/pyintelowl/pyintelowl.py +++ b/pyintelowl/pyintelowl.py @@ -3,6 +3,9 @@ import re import requests import sys +import hashlib + +from typing import List, Dict from json import dumps as json_dumps @@ -12,20 +15,34 @@ class IntelOwl: - def __init__(self, token, certificate, instance, debug): + def __init__( + self, + token: str, + instance_url: str, + certificate: str = None, + debug: bool = False, + ): self.token = token + self.instance = instance_url self.certificate = certificate - self.instance = instance - if debug: + self.__debug__hndlr = logging.StreamHandler(sys.stdout) + self.debug(debug) + + def debug(self, on: bool) -> None: + if on: # if debug add stdout logging logger.setLevel(logging.DEBUG) - logger.addHandler(logging.StreamHandler(sys.stdout)) + logger.addHandler(self.__debug__hndlr) + else: + logger.setLevel(logging.INFO) + logger.removeHandler(self.__debug__hndlr) @property def session(self): if not hasattr(self, "_session"): session = requests.Session() - session.verify = self.certificate + if self.certificate: + session.verify = self.certificate session.headers.update( { "Authorization": f"Token {self.token}", @@ -102,28 +119,28 @@ def send_file_analysis_request( def send_observable_analysis_request( self, - md5, - analyzers_requested, - observable_name, - force_privacy=False, - private_job=False, - disable_external_analyzers=False, - run_all_available_analyzers=False, - runtime_configuration=None, + analyzers_requested: List[str], + observable_name: str, + md5: str = None, + force_privacy: bool = False, + private_job: bool = False, + disable_external_analyzers: bool = False, + run_all_available_analyzers: bool = False, + runtime_configuration: Dict = {}, ): - if runtime_configuration is None: - runtime_configuration = {} answer = {} errors = [] + if not md5: + md5 = hashlib.md5(observable_name.encode("utf-8")).hexdigest() try: data = { + "is_sample": False, "md5": md5, "analyzers_requested": analyzers_requested, "run_all_available_analyzers": run_all_available_analyzers, "force_privacy": force_privacy, "private": private_job, "disable_external_analyzers": disable_external_analyzers, - "is_sample": False, "observable_name": observable_name, "observable_classification": get_observable_classification( observable_name @@ -138,7 +155,7 @@ def send_observable_analysis_request( answer = response.json() except Exception as e: errors.append(str(e)) - return {"errors": errors, "answer": answer} + return answer, errors def ask_analysis_result(self, job_id): answer = {} diff --git a/setup.py b/setup.py index 3cec36e..fb1cecd 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,9 @@ +""" +# PyIntelOwl +Robust Python SDK and CLI for interacting with IntelOwl's API. +## Docs & Example Usage: https://github.com/intelowlproject/pyintelowl +""" + import pathlib from setuptools import setup @@ -7,21 +13,60 @@ # The text of the README file README = (HERE / "README.md").read_text() +GITHUB_URL = "https://github.com/intelowlproject/pyintelowl" + +requirements = [ + "requests==2.25.0", + "geocoder==1.38.1", + "click==7.1.2", + "rich==9.2.0", +] + # This call to setup() does all the work setup( name="pyintelowl", - version="2.0.0", - description="Client and Library for Intel Owl", + version="3.0.0", + description="Robust Python SDK and CLI for IntelOwl's API", long_description=README, long_description_content_type="text/markdown", - url="https://github.com/intelowlproject/pyintelowl", + url=GITHUB_URL, author="Matteo Lodi", classifiers=[ + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 5 - Production/Stable", + # Indicate who your project is intended for + "Intended Audience :: Developers", + "Environment :: Web Environment", + # Pick your license as you wish (should match "license" above) "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Topic :: Software Development :: Libraries :: Python Modules", ], packages=["pyintelowl"], python_requires="~=3.6", include_package_data=True, - install_requires=["requests", "geocoder"], + install_requires=requirements, + project_urls={ + "Documentation": "https://django-rest-durin.readthedocs.io/", + "Funding": "https://liberapay.com/IntelOwlProject/", + "Source": GITHUB_URL, + "Tracker": "{}/issues".format(GITHUB_URL), + }, + keywords="intelowl sdk python command line osint threat intel", + # List additional groups of dependencies here (e.g. development + # dependencies). You can install these using the following syntax, + # for example: + # $ pip install -e .[dev,test] + extras_require={ + "dev": ["black==20.8b1", "flake8"] + requirements, + "test": ["black==20.8b1", "flake8"] + requirements, + }, ) From b57ee22e198d5f645f218e5b18595d2627121582 Mon Sep 17 00:00:00 2001 From: Appaji Chintimi Date: Mon, 16 Nov 2020 13:23:06 +0530 Subject: [PATCH 02/38] Added job Listing functionality Update cli/jobs.py Co-authored-by: Eshaan Bansal --- cli/__init__.py | 3 +- cli/jobs.py | 106 ++++++++++++++++++++++++++++++++++++--- pyintelowl/pyintelowl.py | 25 +++++++++ 3 files changed, 126 insertions(+), 8 deletions(-) diff --git a/cli/__init__.py b/cli/__init__.py index c4175eb..ec95936 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -6,11 +6,10 @@ groups = [ analyse, - jobs, tags, config, ] -cmds = [get_analyzer_config] +cmds = [get_analyzer_config, jobs] __all__ = [groups, cmds] diff --git a/cli/jobs.py b/cli/jobs.py index 0516a1d..a28706d 100644 --- a/cli/jobs.py +++ b/cli/jobs.py @@ -1,14 +1,108 @@ import click +from rich.console import Console +from rich.table import Table +from rich.text import Text +from rich import box +from json import dumps as json_dumps +from pprint import pprint -@click.group("jobs", invoke_without_command=True) -def jobs(): +@click.command(short_help="Manage jobs") +@click.option("-a", "--all", is_flag=True, help="List all jobs") +@click.option("--id", help="Retrieve Job details by ID") +@click.pass_context +def jobs(ctx, id, all): """ List jobs """ - pass + if all: + errors, answer = ctx.obj.get_all_jobs() + if not errors: + display_all_jobs(answer) + else: + pprint(errors) + elif id: + errors, answer = ctx.obj.get_job_by_id(id) + if not errors: + display_single_job(answer) + else: + pprint(errors) -@jobs.command(short_help="HTTP Poll for a job who's status is `running`") -def poll(): - pass +def display_single_job(data): + console = Console() + style = "bold #31DDCF" + console.print(Text("Id: ", style=style, end=""), Text(str(data["id"]))) + console.print(Text("Tags: ", style=style, end=""), Text(", ".join(data["tags"]))) + console.print(Text("User: ", style=style, end=""), Text(data["source"])) + console.print(Text("MD5: ", style=style, end=""), Text(data["md5"])) + console.print( + Text("Name: ", style=style, end=""), + Text(data["observable_name"] if data["observable_name"] else data["file_name"]), + ) + console.print( + Text("Classification: ", style=style, end=""), + Text( + data["observable_classification"] + if data["observable_classification"] + else data["file_mimetype"] + ), + ) + console.print(Text("Status: ", style=style, end=""), job_status(data["status"])) + + header_style = "bold blue" + table = Table(show_header=True) + table.add_column("Name", header_style=header_style) + table.add_column("Errors", header_style=header_style) + table.add_column("Report", header_style=header_style) + table.add_column("Status", header_style=header_style) + + for element in data["analysis_reports"]: + table.add_row( + element["name"], + json_dumps(element["errors"], indent=2), + json_dumps(element["report"], indent=2), + str(element["success"]), + ) + console.print(table) + + +def job_status(status): + styles = { + "pending": "#CE5C00", + "running": "#CE5C00", + "reported_without_fails": "#73D216", + "reported_with_fails": "#CC0000", + "failed": "#CC0000", + } + return Text(status, style=styles[status]) + + +def display_all_jobs(data): + console = Console() + table = Table(show_header=True) + header_style = "bold blue" + table.add_column(header="Id", header_style=header_style) + table.add_column(header="Name", header_style=header_style) + table.add_column(header="Type", header_style=header_style) + table.add_column( + header="Analyzers\nCalled", justify="center", header_style=header_style + ) + table.add_column( + header="Process\nTime(s)", justify="center", header_style=header_style + ) + table.add_column(header="Status", header_style=header_style) + try: + for element in data: + table.add_row( + str(element["id"]), + element["observable_name"], + element["observable_classification"], + element["no_of_analyzers_executed"], + str(element["process_time"]), + job_status(element["status"]), + ) + table.box = box.DOUBLE + console.print(table, justify="center") + except Exception as e: + print(e) diff --git a/pyintelowl/pyintelowl.py b/pyintelowl/pyintelowl.py index 9f619b6..e6550b3 100644 --- a/pyintelowl/pyintelowl.py +++ b/pyintelowl/pyintelowl.py @@ -184,6 +184,31 @@ def get_analyzer_configs(self): errors.append(str(e)) return {"errors": errors, "answer": answer} + def get_all_jobs(self): + answer = [] + errors = [] + try: + url = self.instance + "/api/jobs" + response = self.session.get(url) + logger.debug(response.url) + response.raise_for_status() + answer = response.json() + except Exception as e: + errors.append(str(e)) + return (errors, answer) + + def get_job_by_id(self, job_id): + answer = {} + errors = [] + try: + url = self.instance + "/api/jobs/" + str(job_id) + response = self.session.get(url) + logger.debug(response.url) + answer = response.json() + except Exception as e: + errors.append(str(e)) + return (errors, answer) + def get_observable_classification(value): # only following types are supported: From 15298ade7e911ebc1d4d1c29f20387b2e069f446 Mon Sep 17 00:00:00 2001 From: Ramnath Kumar Date: Sun, 15 Nov 2020 22:16:23 +0530 Subject: [PATCH 03/38] Rewrote tags using click and rich Added tags Optimized code Ran flake8 and black Color columns are also rich formatted Prettyprint erros Updated table format Fix merge issues --- cli/__init__.py | 3 +-- cli/tags.py | 43 ++++++++++++++++++++++++++++++++++------ pyintelowl/pyintelowl.py | 26 ++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/cli/__init__.py b/cli/__init__.py index c4175eb..36a7616 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -7,10 +7,9 @@ groups = [ analyse, jobs, - tags, config, ] -cmds = [get_analyzer_config] +cmds = [get_analyzer_config, tags] __all__ = [groups, cmds] diff --git a/cli/tags.py b/cli/tags.py index 29577c0..b00e07f 100644 --- a/cli/tags.py +++ b/cli/tags.py @@ -1,9 +1,40 @@ import click +from rich.console import Console +from rich.table import Table +from rich import box +from pprint import pprint -@click.group("tags", invoke_without_command=True) -def tags(): - """ - List tags - """ - pass +@click.command(short_help="Manage tags") +@click.option("-a", "--all", is_flag=True, help="List all tags") +@click.option("--id", help="Retrieve tag by ID") +@click.pass_context +def tags(ctx, id, all): + if all: + ans, errs = ctx.obj.get_all_tags() + else: + if id: + ans, errs = ctx.obj.get_tag_by_id(id) + if errs: + pprint(errs) + print_table(ans) + + +def print_table(data): + console = Console() + table = Table(show_header=True) + table.add_column("Id", no_wrap=True, header_style="bold blue") + table.add_column("Label", no_wrap=True, header_style="bold blue") + table.add_column("Color", no_wrap=True, header_style="bold blue") + try: + for elem in data: + color = str(elem["color"]) + table.add_row( + str(elem["id"]), + str(elem["label"]), + f"[{color.lower()}]{color}[/{color.lower()}]", + ) + table.box = box.DOUBLE + console.print(table, justify="center") + except Exception as e: + pprint(e) diff --git a/pyintelowl/pyintelowl.py b/pyintelowl/pyintelowl.py index 9f619b6..d3a1975 100644 --- a/pyintelowl/pyintelowl.py +++ b/pyintelowl/pyintelowl.py @@ -184,6 +184,32 @@ def get_analyzer_configs(self): errors.append(str(e)) return {"errors": errors, "answer": answer} + def get_all_tags(self): + answer = [] + errors = [] + try: + url = self.instance + "/api/tags" + response = self.session.get(url) + logger.debug(response.url) + response.raise_for_status() + answer = response.json() + except Exception as e: + errors.append(str(e)) + return answer, errors + + def get_tag_by_id(self, tag_id): + answer = [] + errors = [] + try: + url = self.instance + "/api/tags/" + response = self.session.get(url + str(tag_id)) + logger.debug(response.url) + response.raise_for_status() + answer.append(response.json()) + except Exception as e: + errors.append(str(e)) + return answer, errors + def get_observable_classification(value): # only following types are supported: From 55ba0f8bcf621387a366b0408c703971927d6e6f Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Mon, 16 Nov 2020 22:35:33 +0530 Subject: [PATCH 04/38] Refactor code for tags/jobs/analyzer_config CLI interfaces (#38) --- cli.py | 11 ++- cli/__init__.py | 6 +- cli/_utils.py | 48 ++++++++++++ cli/analyse.py | 6 +- cli/commands.py | 11 ++- cli/config.py | 4 +- cli/jobs.py | 155 +++++++++++++++++++++++---------------- cli/tags.py | 35 +++++---- pyintelowl/pyintelowl.py | 28 +++---- setup.py | 1 + 10 files changed, 200 insertions(+), 105 deletions(-) diff --git a/cli.py b/cli.py index b2364d3..c8ba2e0 100644 --- a/cli.py +++ b/cli.py @@ -1,6 +1,6 @@ import click from pyintelowl.pyintelowl import IntelOwl -from cli import groups, cmds +from cli import groups, cmds, ClickContext @click.group(context_settings=dict(help_option_names=["-h", "--help"])) @@ -16,6 +16,7 @@ "--instance-url", required=True, default="http://localhost:80", + show_default=True, help="IntelOwl's instance URL", ) @click.option( @@ -25,10 +26,12 @@ type=click.Path(exists=True), help="Path to SSL client certificate file (.pem)", ) -@click.option("--debug/--no-debug", default=False, help="Set log level to DEBUG") +@click.option("-d", "--debug", is_flag=True, help="Set log level to DEBUG") @click.pass_context -def cli(ctx, api_key, instance_url, certificate, debug): - ctx.obj: "IntelOwl" = IntelOwl(api_key, instance_url, certificate, debug) +def cli( + ctx: ClickContext, api_key: str, instance_url: str, certificate: str, debug: bool +): + ctx.obj = IntelOwl(api_key, instance_url, certificate, debug) # Compile all groups and commands diff --git a/cli/__init__.py b/cli/__init__.py index 957d4ef..85268d3 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -3,14 +3,16 @@ from .tags import tags from .config import config from .commands import get_analyzer_config +from ._utils import ClickContext groups = [ analyse, config, + jobs, ] -cmds = [get_analyzer_config, jobs, tags] +cmds = [get_analyzer_config, tags] -__all__ = [groups, cmds] +__all__ = [ClickContext, groups, cmds] diff --git a/cli/_utils.py b/cli/_utils.py index abbef7c..fa2fa16 100644 --- a/cli/_utils.py +++ b/cli/_utils.py @@ -1,3 +1,51 @@ +import click +import json +from rich.emoji import Emoji +from rich.text import Text +from rich.syntax import Syntax + +from pyintelowl.pyintelowl import IntelOwl + + +class ClickContext(click.Context): + obj: IntelOwl + """ + IntelOwl instance + """ + + +def get_status_text(status: str): + styles = { + "pending": ("#CE5C00", str(Emoji("gear"))), + "running": ("#CE5C00", str(Emoji("gear"))), + "reported_without_fails": ("#73D216", str(Emoji("heavy_check_mark"))), + "reported_with_fails": ("#CC0000", str(Emoji("warning"))), + "failed": ("#CC0000", str(Emoji("cross_mark"))), + } + color, emoji = styles[status] + return Text(status + " " + emoji, style=color) + + +def get_success_text(success): + success = str(success) + styles = { + "True": ("#73D216", str(Emoji("heavy_check_mark"))), + "False": ("#CC0000", str(Emoji("cross_mark"))), + } + color, emoji = styles[success] + return Text(emoji, style=color) + + +def get_json_syntax(obj): + return Syntax( + json.dumps(obj, indent=2), + "json", + theme="ansi_dark", + word_wrap=True, + tab_size=2, + ) + + def add_options(options): def _add_options(func): for option in reversed(options): diff --git a/cli/analyse.py b/cli/analyse.py index c6497a6..9c7521e 100644 --- a/cli/analyse.py +++ b/cli/analyse.py @@ -1,5 +1,5 @@ import click -from ._utils import add_options +from ._utils import add_options, ClickContext __analyse_options = [ click.option( @@ -68,7 +68,7 @@ def analyse(): @add_options(__analyse_options) @click.pass_context def observable( - ctx, + ctx: ClickContext, value, analyzers_list, run_all_available_analyzers, @@ -102,7 +102,7 @@ def observable( @add_options(__analyse_options) @click.pass_context def file( - ctx, + ctx: ClickContext, filename, analyzers_list, run_all_available_analyzers, diff --git a/cli/commands.py b/cli/commands.py index 84d39a0..ea970e1 100644 --- a/cli/commands.py +++ b/cli/commands.py @@ -1,10 +1,15 @@ import click +from ._utils import ClickContext +from rich import print @click.command( short_help="Get current state of `analyzer_config.json` from the IntelOwl instance" ) @click.pass_context -def get_analyzer_config(ctx): - res = ctx.obj.get_analyzer_configs() - print(res) +def get_analyzer_config(ctx: ClickContext): + res, err = ctx.obj.get_analyzer_configs() + if err: + print(err) + else: + print(res) diff --git a/cli/config.py b/cli/config.py index 83aed5a..9976f4f 100644 --- a/cli/config.py +++ b/cli/config.py @@ -1,9 +1,11 @@ import click +from ._utils import ClickContext + @click.group("config", invoke_without_command=True) @click.pass_context -def config(): +def config(ctx: ClickContext): """ Set or view config variables """ diff --git a/cli/jobs.py b/cli/jobs.py index a28706d..c5771ce 100644 --- a/cli/jobs.py +++ b/cli/jobs.py @@ -1,90 +1,117 @@ import click +import click_spinner from rich.console import Console from rich.table import Table from rich.text import Text -from rich import box -from json import dumps as json_dumps -from pprint import pprint +from rich import box, print as rprint +from ._utils import ClickContext, get_status_text, get_success_text, get_json_syntax -@click.command(short_help="Manage jobs") + +@click.group("jobs", short_help="Manage Jobs", invoke_without_command=True) @click.option("-a", "--all", is_flag=True, help="List all jobs") -@click.option("--id", help="Retrieve Job details by ID") +@click.option("--id", type=int, default=0, help="Retrieve Job details by ID") @click.pass_context -def jobs(ctx, id, all): +def jobs(ctx: ClickContext, id: int, all: bool): """ - List jobs + Manage Jobs """ + errs = None if all: - errors, answer = ctx.obj.get_all_jobs() - if not errors: - display_all_jobs(answer) - else: - pprint(errors) + with click_spinner.spinner(): + ans, errs = ctx.obj.get_all_jobs() + if not errs: + _display_all_jobs(ans) elif id: - errors, answer = ctx.obj.get_job_by_id(id) - if not errors: - display_single_job(answer) - else: - pprint(errors) + with click_spinner.spinner(): + ans, errs = ctx.obj.get_job_by_id(id) + if not errs: + _display_single_job(ans) + else: + print("Use -h or --help for help.") + if errs: + rprint(errs) -def display_single_job(data): - console = Console() - style = "bold #31DDCF" - console.print(Text("Id: ", style=style, end=""), Text(str(data["id"]))) - console.print(Text("Tags: ", style=style, end=""), Text(", ".join(data["tags"]))) - console.print(Text("User: ", style=style, end=""), Text(data["source"])) - console.print(Text("MD5: ", style=style, end=""), Text(data["md5"])) - console.print( - Text("Name: ", style=style, end=""), - Text(data["observable_name"] if data["observable_name"] else data["file_name"]), - ) - console.print( - Text("Classification: ", style=style, end=""), - Text( - data["observable_classification"] - if data["observable_classification"] - else data["file_mimetype"] - ), - ) - console.print(Text("Status: ", style=style, end=""), job_status(data["status"])) +@jobs.command("poll", short_help="HTTP poll a currently running job's details") +@click.option( + "-t", + "--max-tries", + type=int, + default=0, + show_default=True, + help="maximum number of tries (in sec)", +) +@click.option( + "-i", + "--interval", + type=int, + default=5, + show_default=True, + help="sleep interval before subsequent requests (in sec)", +) +@click.pass_context +def poll(ctx: ClickContext, max_tries: int, interval: int): + pass - header_style = "bold blue" - table = Table(show_header=True) - table.add_column("Name", header_style=header_style) - table.add_column("Errors", header_style=header_style) - table.add_column("Report", header_style=header_style) - table.add_column("Status", header_style=header_style) - for element in data["analysis_reports"]: - table.add_row( - element["name"], - json_dumps(element["errors"], indent=2), - json_dumps(element["report"], indent=2), - str(element["success"]), +def _display_single_job(data): + console = Console() + style = "bold #31DDCF" + headers = ["Name", "Status", "Report", "Errors"] + with console.pager(styles=True): + # print job attributes + console.print(Text("Id: ", style=style, end=""), Text(str(data["id"]))) + tags = ", ".join([t["label"] for t in data["tags"]]) + console.print(Text("Tags: ", style=style, end=""), Text(tags)) + console.print(Text("User: ", style=style, end=""), Text(data["source"])) + console.print(Text("MD5: ", style=style, end=""), Text(data["md5"])) + console.print( + Text("Name: ", style=style, end=""), + (data["observable_name"] if data["observable_name"] else data["file_name"]), + ) + console.print( + Text("Classification: ", style=style, end=""), + ( + data["observable_classification"] + if data["observable_classification"] + else data["file_mimetype"] + ), + ) + console.print( + Text("Status: ", style=style, end=""), get_status_text(data["status"]) ) - console.print(table) + # construct job analysis table -def job_status(status): - styles = { - "pending": "#CE5C00", - "running": "#CE5C00", - "reported_without_fails": "#73D216", - "reported_with_fails": "#CC0000", - "failed": "#CC0000", - } - return Text(status, style=styles[status]) + table = Table( + show_header=True, + title="Analysis Data", + box=box.DOUBLE_EDGE, + show_lines=True, + ) + # add headers + for h in headers: + table.add_column(h, header_style="bold blue") + # add rows + for element in data["analysis_reports"]: + table.add_row( + element["name"], + get_success_text((element["success"])), + get_json_syntax(element["report"]) if element["report"] else None, + get_json_syntax(element["errors"]) if element["errors"] else None, + ) + console.print(table) -def display_all_jobs(data): +def _display_all_jobs(data): console = Console() - table = Table(show_header=True) + table = Table(show_header=True, title="List of Jobs", box=box.DOUBLE_EDGE) header_style = "bold blue" table.add_column(header="Id", header_style=header_style) table.add_column(header="Name", header_style=header_style) table.add_column(header="Type", header_style=header_style) + table.add_column(header="Tags", header_style=header_style) table.add_column( header="Analyzers\nCalled", justify="center", header_style=header_style ) @@ -98,11 +125,11 @@ def display_all_jobs(data): str(element["id"]), element["observable_name"], element["observable_classification"], + ", ".join([t["label"] for t in element["tags"]]), element["no_of_analyzers_executed"], str(element["process_time"]), - job_status(element["status"]), + get_status_text(element["status"]), ) - table.box = box.DOUBLE console.print(table, justify="center") except Exception as e: - print(e) + rprint(e) diff --git a/cli/tags.py b/cli/tags.py index b00e07f..a32815f 100644 --- a/cli/tags.py +++ b/cli/tags.py @@ -1,40 +1,45 @@ import click from rich.console import Console from rich.table import Table -from rich import box -from pprint import pprint +from rich.text import Text +from rich import box, print as rprint + +from ._utils import ClickContext @click.command(short_help="Manage tags") @click.option("-a", "--all", is_flag=True, help="List all tags") -@click.option("--id", help="Retrieve tag by ID") +@click.option("--id", type=int, default=0, help="Retrieve tag details by ID") @click.pass_context -def tags(ctx, id, all): +def tags(ctx: ClickContext, id: int, all: bool): + """ + Manage tags + """ if all: ans, errs = ctx.obj.get_all_tags() - else: - if id: - ans, errs = ctx.obj.get_tag_by_id(id) + elif id: + ans, errs = ctx.obj.get_tag_by_id(id) + ans = [ans] if errs: - pprint(errs) - print_table(ans) + rprint(errs) + else: + _print_tags_table(ans) -def print_table(data): +def _print_tags_table(data): console = Console() - table = Table(show_header=True) + table = Table(show_header=True, title="List of tags", box=box.DOUBLE_EDGE) table.add_column("Id", no_wrap=True, header_style="bold blue") table.add_column("Label", no_wrap=True, header_style="bold blue") table.add_column("Color", no_wrap=True, header_style="bold blue") try: for elem in data: - color = str(elem["color"]) + color = str(elem["color"]).lower() table.add_row( str(elem["id"]), str(elem["label"]), - f"[{color.lower()}]{color}[/{color.lower()}]", + Text(color, style=f"on {color}") ) - table.box = box.DOUBLE console.print(table, justify="center") except Exception as e: - pprint(e) + rprint(e) diff --git a/pyintelowl/pyintelowl.py b/pyintelowl/pyintelowl.py index 1a27eff..9cc92df 100644 --- a/pyintelowl/pyintelowl.py +++ b/pyintelowl/pyintelowl.py @@ -172,7 +172,7 @@ def ask_analysis_result(self, job_id): return {"errors": errors, "answer": answer} def get_analyzer_configs(self): - answer = {} + answer = None errors = [] try: url = self.instance + "/api/get_analyzer_configs" @@ -182,10 +182,10 @@ def get_analyzer_configs(self): answer = response.json() except Exception as e: errors.append(str(e)) - return {"errors": errors, "answer": answer} + return answer, errors def get_all_tags(self): - answer = [] + answer = None errors = [] try: url = self.instance + "/api/tags" @@ -198,42 +198,44 @@ def get_all_tags(self): return answer, errors def get_all_jobs(self): - answer = [] + answer = None errors = [] try: url = self.instance + "/api/jobs" response = self.session.get(url) - logger.debug(response.url) + logger.debug(msg=(response.url, response.status_code)) response.raise_for_status() answer = response.json() except Exception as e: errors.append(str(e)) - return (errors, answer) + return answer, errors def get_tag_by_id(self, tag_id): - answer = [] + answer = None errors = [] try: url = self.instance + "/api/tags/" response = self.session.get(url + str(tag_id)) - logger.debug(response.url) + logger.debug(msg=(response.url, response.status_code)) response.raise_for_status() - answer.append(response.json()) + answer = response.json() except Exception as e: errors.append(str(e)) return answer, errors - + def get_job_by_id(self, job_id): - answer = {} + answer = None errors = [] try: url = self.instance + "/api/jobs/" + str(job_id) response = self.session.get(url) - logger.debug(response.url) + logger.debug(msg=(response.url, response.status_code)) + response.raise_for_status() answer = response.json() except Exception as e: errors.append(str(e)) - return (errors, answer) + return answer, errors + def get_observable_classification(value): # only following types are supported: diff --git a/setup.py b/setup.py index fb1cecd..718a0dc 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ "geocoder==1.38.1", "click==7.1.2", "rich==9.2.0", + "click-spinner==0.1.10", ] # This call to setup() does all the work From 217aa1f3fd785c9c705762f79e1c000db4ffa6dd Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Tue, 17 Nov 2020 12:37:27 +0530 Subject: [PATCH 05/38] use netrc for storing config vars (#39) --- cli.py | 33 ++++++++-------------------- cli/_utils.py | 7 ++++++ cli/analyse.py | 1 + cli/config.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++---- cli/jobs.py | 2 +- setup.py | 1 + 6 files changed, 73 insertions(+), 29 deletions(-) diff --git a/cli.py b/cli.py index c8ba2e0..4cbfb31 100644 --- a/cli.py +++ b/cli.py @@ -1,37 +1,22 @@ import click +from tinynetrc import Netrc from pyintelowl.pyintelowl import IntelOwl from cli import groups, cmds, ClickContext @click.group(context_settings=dict(help_option_names=["-h", "--help"])) -@click.option( - "-k", - "--api-key", - required=True, - default="", - help="API key to authenticate against a IntelOwl instance", -) -@click.option( - "-u", - "--instance-url", - required=True, - default="http://localhost:80", - show_default=True, - help="IntelOwl's instance URL", -) -@click.option( - "-c", - "--certificate", - required=False, - type=click.Path(exists=True), - help="Path to SSL client certificate file (.pem)", -) @click.option("-d", "--debug", is_flag=True, help="Set log level to DEBUG") @click.pass_context def cli( - ctx: ClickContext, api_key: str, instance_url: str, certificate: str, debug: bool + ctx: ClickContext, debug: bool ): - ctx.obj = IntelOwl(api_key, instance_url, certificate, debug) + netrc = Netrc() + host = netrc["pyintelowl"] + api_key, url, cert = host["password"], host["account"], host["login"] + if not api_key or not url: + click.echo("Hint: Use `config set` to set config variables!") + else: + ctx.obj = IntelOwl(api_key, url, cert, debug) # Compile all groups and commands diff --git a/cli/_utils.py b/cli/_utils.py index fa2fa16..9dd5ad1 100644 --- a/cli/_utils.py +++ b/cli/_utils.py @@ -1,5 +1,6 @@ import click import json +from tinynetrc import Netrc from rich.emoji import Emoji from rich.text import Text from rich.syntax import Syntax @@ -53,3 +54,9 @@ def _add_options(func): return func return _add_options + + +def get_netrc_obj(): + netrc = Netrc() + host = netrc["pyintelowl"] + return netrc, host diff --git a/cli/analyse.py b/cli/analyse.py index 9c7521e..53d6999 100644 --- a/cli/analyse.py +++ b/cli/analyse.py @@ -87,6 +87,7 @@ def observable( err=True, color="RED", ) + raise click.Abort() ans, errs = ctx.obj.send_observable_analysis_request( analyzers_requested=analyzers_list, observable_name=value, diff --git a/cli/config.py b/cli/config.py index 9976f4f..f407f2c 100644 --- a/cli/config.py +++ b/cli/config.py @@ -1,12 +1,62 @@ import click +from rich import print as rprint -from ._utils import ClickContext +from ._utils import get_netrc_obj -@click.group("config", invoke_without_command=True) -@click.pass_context -def config(ctx: ClickContext): +@click.group("config") +def config(): """ Set or view config variables """ pass + + +@config.command("get") +def config_get(): + """ + Pretty Print config variables + """ + _, host = get_netrc_obj() + rprint({ + "api_key": host["password"], + "instance_url": host["account"], + "certificate": host["login"], + }) + + +@config.command("set") +@click.option( + "-k", + "--api-key", + required=True, + help="API key to authenticate against a IntelOwl instance", +) +@click.option( + "-u", + "--instance-url", + required=True, + default="http://localhost:80", + show_default=True, + help="IntelOwl's instance URL", +) +@click.option( + "-c", + "--certificate", + required=False, + type=click.Path(exists=True), + help="Path to SSL client certificate file (.pem)", +) +def config_set(api_key, instance_url, certificate): + """ + Set/Edit config variables + """ + netrc, _ = get_netrc_obj() + if api_key: + netrc["pyintelowl"]["password"] = api_key + if instance_url: + netrc["pyintelowl"]["account"] = instance_url + if certificate: + netrc["pyintelowl"]["login"] = certificate + # finally save + netrc.save() diff --git a/cli/jobs.py b/cli/jobs.py index c5771ce..8140b00 100644 --- a/cli/jobs.py +++ b/cli/jobs.py @@ -28,7 +28,7 @@ def jobs(ctx: ClickContext, id: int, all: bool): if not errs: _display_single_job(ans) else: - print("Use -h or --help for help.") + rprint(ctx.get_usage()) if errs: rprint(errs) diff --git a/setup.py b/setup.py index 718a0dc..4975b5f 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ "click==7.1.2", "rich==9.2.0", "click-spinner==0.1.10", + "tinynetrc==1.3.0", ] # This call to setup() does all the work From 3043cab1607699192c6035af5da36ee79984c93d Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Tue, 17 Nov 2020 13:37:37 +0530 Subject: [PATCH 06/38] fix jobs with file_name --- cli/jobs.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cli/jobs.py b/cli/jobs.py index 8140b00..c1c4f96 100644 --- a/cli/jobs.py +++ b/cli/jobs.py @@ -120,15 +120,17 @@ def _display_all_jobs(data): ) table.add_column(header="Status", header_style=header_style) try: - for element in data: + for el in data: table.add_row( - str(element["id"]), - element["observable_name"], - element["observable_classification"], - ", ".join([t["label"] for t in element["tags"]]), - element["no_of_analyzers_executed"], - str(element["process_time"]), - get_status_text(element["status"]), + str(el["id"]), + el["observable_name"] if el["observable_name"] else el["file_name"], + el["observable_classification"] + if el["observable_classification"] + else el["file_mimetype"], + ", ".join([t["label"] for t in el["tags"]]), + el["no_of_analyzers_executed"], + str(el["process_time"]), + get_status_text(el["status"]), ) console.print(table, justify="center") except Exception as e: From 28662fb90a92ce5a9f8a404e3be338ed8af21d1b Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Tue, 17 Nov 2020 17:06:26 +0530 Subject: [PATCH 07/38] refactor code, output fmt for jobs and tags --- cli/__init__.py | 4 --- cli/config.py | 6 ++-- cli/jobs.py | 78 ++++++++++++++++++++++++------------------------- cli/tags.py | 39 +++++++++++++------------ 4 files changed, 63 insertions(+), 64 deletions(-) diff --git a/cli/__init__.py b/cli/__init__.py index 85268d3..4c9c6dc 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -3,7 +3,6 @@ from .tags import tags from .config import config from .commands import get_analyzer_config -from ._utils import ClickContext groups = [ analyse, @@ -13,6 +12,3 @@ cmds = [get_analyzer_config, tags] - - -__all__ = [ClickContext, groups, cmds] diff --git a/cli/config.py b/cli/config.py index f407f2c..12ecfbb 100644 --- a/cli/config.py +++ b/cli/config.py @@ -1,7 +1,7 @@ import click from rich import print as rprint -from ._utils import get_netrc_obj +from ._utils import get_netrc_obj, ClickContext @click.group("config") @@ -47,7 +47,8 @@ def config_get(): type=click.Path(exists=True), help="Path to SSL client certificate file (.pem)", ) -def config_set(api_key, instance_url, certificate): +@click.pass_context +def config_set(ctx: ClickContext, api_key, instance_url, certificate): """ Set/Edit config variables """ @@ -60,3 +61,4 @@ def config_set(api_key, instance_url, certificate): netrc["pyintelowl"]["login"] = certificate # finally save netrc.save() + ctx.obj.logger.info("Succesfully saved config variables!") diff --git a/cli/jobs.py b/cli/jobs.py index c1c4f96..4f5726a 100644 --- a/cli/jobs.py +++ b/cli/jobs.py @@ -2,10 +2,12 @@ import click_spinner from rich.console import Console from rich.table import Table -from rich.text import Text +from rich.panel import Panel +from rich.console import RenderGroup from rich import box, print as rprint -from ._utils import ClickContext, get_status_text, get_success_text, get_json_syntax +from pyintelowl.exceptions import IntelOwlAPIException +from ._utils import ClickContext, get_status_text, get_success_text, get_json_syntax, get_tags_str @click.group("jobs", short_help="Manage Jobs", invoke_without_command=True) @@ -16,21 +18,21 @@ def jobs(ctx: ClickContext, id: int, all: bool): """ Manage Jobs """ - errs = None - if all: - with click_spinner.spinner(): - ans, errs = ctx.obj.get_all_jobs() - if not errs: + try: + if all: + ctx.obj.logger.info("Requesting list of jobs..") + with click_spinner.spinner(): + ans = ctx.obj.get_all_jobs() _display_all_jobs(ans) - elif id: - with click_spinner.spinner(): - ans, errs = ctx.obj.get_job_by_id(id) - if not errs: + elif id: + ctx.obj.logger.info(f"Requesting Job [underline blue]#{id}[/]..") + with click_spinner.spinner(): + ans = ctx.obj.get_job_by_id(id) _display_single_job(ans) - else: - rprint(ctx.get_usage()) - if errs: - rprint(errs) + else: + click.echo(ctx.get_usage()) + except IntelOwlAPIException as e: + ctx.obj.logger.fatal(str(e)) @jobs.command("poll", short_help="HTTP poll a currently running job's details") @@ -57,30 +59,28 @@ def poll(ctx: ClickContext, max_tries: int, interval: int): def _display_single_job(data): console = Console() - style = "bold #31DDCF" + style = "[bold #31DDCF]" headers = ["Name", "Status", "Report", "Errors"] with console.pager(styles=True): # print job attributes - console.print(Text("Id: ", style=style, end=""), Text(str(data["id"]))) - tags = ", ".join([t["label"] for t in data["tags"]]) - console.print(Text("Tags: ", style=style, end=""), Text(tags)) - console.print(Text("User: ", style=style, end=""), Text(data["source"])) - console.print(Text("MD5: ", style=style, end=""), Text(data["md5"])) - console.print( - Text("Name: ", style=style, end=""), - (data["observable_name"] if data["observable_name"] else data["file_name"]), - ) - console.print( - Text("Classification: ", style=style, end=""), - ( - data["observable_classification"] - if data["observable_classification"] - else data["file_mimetype"] - ), + tags = get_tags_str(data["tags"]) + status = get_status_text(data["status"]) + name = data["observable_name"] if data["observable_name"] else data["file_name"] + clsfn = ( + data["observable_classification"] + if data["observable_classification"] + else data["file_mimetype"] ) - console.print( - Text("Status: ", style=style, end=""), get_status_text(data["status"]) + r = RenderGroup( + f"{style}Job ID:[/] {str(data['id'])}", + f"{style}User:[/] {data['source']}", + f"{style}MD5:[/] {data['md5']}", + f"{style}Name:[/] {name}", + f"{style}Classification:[/] {clsfn}", + f"{style}Tags:[/] {tags}", + f"{style}Status:[/] {status}", ) + console.print(Panel(r, title="Job attributes")) # construct job analysis table @@ -94,12 +94,12 @@ def _display_single_job(data): for h in headers: table.add_column(h, header_style="bold blue") # add rows - for element in data["analysis_reports"]: + for el in data["analysis_reports"]: table.add_row( - element["name"], - get_success_text((element["success"])), - get_json_syntax(element["report"]) if element["report"] else None, - get_json_syntax(element["errors"]) if element["errors"] else None, + el["name"], + get_success_text((el["success"])), + get_json_syntax(el["report"]) if el["report"] else None, + get_json_syntax(el["errors"]) if el["errors"] else None, ) console.print(table) diff --git a/cli/tags.py b/cli/tags.py index a32815f..760429e 100644 --- a/cli/tags.py +++ b/cli/tags.py @@ -2,7 +2,9 @@ from rich.console import Console from rich.table import Table from rich.text import Text -from rich import box, print as rprint +from rich import box + +from pyintelowl.exceptions import IntelOwlAPIException from ._utils import ClickContext @@ -15,31 +17,30 @@ def tags(ctx: ClickContext, id: int, all: bool): """ Manage tags """ - if all: - ans, errs = ctx.obj.get_all_tags() - elif id: - ans, errs = ctx.obj.get_tag_by_id(id) - ans = [ans] - if errs: - rprint(errs) - else: - _print_tags_table(ans) + try: + if all: + ans = ctx.obj.get_all_tags() + elif id: + ans = ctx.obj.get_tag_by_id(id) + ans = [ans] + _print_tags_table(ctx, ans) + except IntelOwlAPIException as e: + ctx.obj.logger.fatal(str(e)) -def _print_tags_table(data): +def _print_tags_table(ctx, rows): console = Console() table = Table(show_header=True, title="List of tags", box=box.DOUBLE_EDGE) - table.add_column("Id", no_wrap=True, header_style="bold blue") - table.add_column("Label", no_wrap=True, header_style="bold blue") - table.add_column("Color", no_wrap=True, header_style="bold blue") + for h in ["Id", "Label", "Color"]: + table.add_column(h, no_wrap=True, header_style="bold blue") try: - for elem in data: - color = str(elem["color"]).lower() + for row in rows: + color = str(row["color"]).lower() table.add_row( - str(elem["id"]), - str(elem["label"]), + str(row["id"]), + str(row["label"]), Text(color, style=f"on {color}") ) console.print(table, justify="center") except Exception as e: - rprint(e) + ctx.obj.logger.fatal(str(e)) From d2e2d5d43bfca6763e7c33a35df383b2a452d932 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Tue, 17 Nov 2020 17:06:55 +0530 Subject: [PATCH 08/38] add support for rich logger handler --- cli.py | 8 ++++---- cli/_utils.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/cli.py b/cli.py index 4cbfb31..d453059 100644 --- a/cli.py +++ b/cli.py @@ -1,7 +1,7 @@ import click -from tinynetrc import Netrc from pyintelowl.pyintelowl import IntelOwl -from cli import groups, cmds, ClickContext +from cli import groups, cmds +from cli._utils import get_logger, ClickContext, get_netrc_obj @click.group(context_settings=dict(help_option_names=["-h", "--help"])) @@ -10,13 +10,13 @@ def cli( ctx: ClickContext, debug: bool ): - netrc = Netrc() - host = netrc["pyintelowl"] + netrc, host = get_netrc_obj() api_key, url, cert = host["password"], host["account"], host["login"] if not api_key or not url: click.echo("Hint: Use `config set` to set config variables!") else: ctx.obj = IntelOwl(api_key, url, cert, debug) + ctx.obj.logger = get_logger() # Compile all groups and commands diff --git a/cli/_utils.py b/cli/_utils.py index 9dd5ad1..1025877 100644 --- a/cli/_utils.py +++ b/cli/_utils.py @@ -1,9 +1,11 @@ import click +import logging import json from tinynetrc import Netrc from rich.emoji import Emoji from rich.text import Text from rich.syntax import Syntax +from rich.logging import RichHandler from pyintelowl.pyintelowl import IntelOwl @@ -60,3 +62,22 @@ def get_netrc_obj(): netrc = Netrc() host = netrc["pyintelowl"] return netrc, host + + +def get_tags_str(tags): + tags_str = ", ".join( + [ + "[on {0}]{1}[/]".format(str(t["color"]).lower(), t["label"]) + for t in tags + ] + ) + return tags_str + + +def get_logger(level: str = "INFO"): + fmt = "%(message)s" + logging.basicConfig( + level=level, format=fmt, datefmt="[%X]", handlers=[RichHandler(markup=True)] + ) + logger = logging.getLogger("rich") + return logger From c44c9104bbb41c8e503236679be3d599c2d76822 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Tue, 17 Nov 2020 17:32:50 +0530 Subject: [PATCH 09/38] refactor IntelOwl error handling, make some modfs. to analyse --- cli/analyse.py | 51 +++++++++++++------ pyintelowl/exceptions.py | 4 ++ pyintelowl/pyintelowl.py | 105 ++++++++++++++++++++++++++------------- 3 files changed, 110 insertions(+), 50 deletions(-) diff --git a/cli/analyse.py b/cli/analyse.py index 53d6999..15a91b2 100644 --- a/cli/analyse.py +++ b/cli/analyse.py @@ -1,5 +1,6 @@ import click from ._utils import add_options, ClickContext +from .jobs import _display_single_job __analyse_options = [ click.option( @@ -16,6 +17,7 @@ click.option( "-aa", "--run-all-available-analyzers", + "run_all", is_flag=True, help=""" Run all available and compatible analyzers. Should not be used with @@ -45,6 +47,7 @@ "--check", type=click.Choice(["reported", "running", "force-new"], case_sensitive=False), default="reported", + show_choices=True, show_default=True, help="""\n 1. 'reported': analysis won't be repeated if already exists as running or failed.\n @@ -71,30 +74,48 @@ def observable( ctx: ClickContext, value, analyzers_list, - run_all_available_analyzers, + run_all, force_privacy, private_job, disable_external_analyzers, check, ): - if not analyzers_list: - if not run_all_available_analyzers: - click.echo( - """ - One of --analyzers-list, - --run-all-available-analyzers should be specified - """, - err=True, - color="RED", - ) - raise click.Abort() - ans, errs = ctx.obj.send_observable_analysis_request( + if analyzers_list and run_all: + logger.warn( + """ + Can't use -al and -aa options together. See usage with -h. + """ + ) + ctx.exit(-1) + if not (analyzers_list or run_all): + logger.warn( + """ + Either one of -al, -aa must be specified. See usage with -h. + """, + ) + ctx.exit(-1) + analyzers = analyzers_list if analyzers_list else "all available analyzers" + ctx.obj.logger.info( + f"""Requesting analysis.. + observable: [bold blue underline]{value}[/] + analyzers: [italic green]{analyzers}[/] + """, + ) + # first step: ask analysis availability + ans = ctx.obj.send_observable_analysis_request( analyzers_requested=analyzers_list, observable_name=value, force_privacy=force_privacy, private_job=private_job, disable_external_analyzers=disable_external_analyzers, - run_all_available_analyzers=run_all_available_analyzers, + run_all_available_analyzers=run_all, + ) + warnings = ans["warnings"] + ctx.obj.logger.info( + f"""New Job running.. + ID: {ans['job_id']} | Status: [underline pink]{ans['status']}[/]. + Got {len(warnings)} warnings: [italic red]{warnings if warnings else None}[/] + """ ) @@ -106,7 +127,7 @@ def file( ctx: ClickContext, filename, analyzers_list, - run_all_available_analyzers, + run_all, force_privacy, private_job, disable_external_analyzers, diff --git a/pyintelowl/exceptions.py b/pyintelowl/exceptions.py index 5e30296..d4a7d91 100644 --- a/pyintelowl/exceptions.py +++ b/pyintelowl/exceptions.py @@ -1,3 +1,7 @@ +class IntelOwlAPIException(Exception): + pass + + class IntelOwlClientException(Exception): pass diff --git a/pyintelowl/pyintelowl.py b/pyintelowl/pyintelowl.py index 9cc92df..aa42a26 100644 --- a/pyintelowl/pyintelowl.py +++ b/pyintelowl/pyintelowl.py @@ -1,5 +1,6 @@ import ipaddress import logging +import pathlib import re import requests import sys @@ -9,12 +10,14 @@ from json import dumps as json_dumps -from .exceptions import IntelOwlClientException +from .exceptions import IntelOwlClientException, IntelOwlAPIException logger = logging.getLogger(__name__) class IntelOwl: + logger: logging.Logger + def __init__( self, token: str, @@ -46,7 +49,7 @@ def session(self): session.headers.update( { "Authorization": f"Token {self.token}", - "User-Agent": "IntelOwlClient/2.0.0", + "User-Agent": "IntelOwlClient/3.0.0", } ) self._session = session @@ -60,8 +63,7 @@ def ask_analysis_availability( run_all_available_analyzers=False, check_reported_analysis_too=False, ): - answer = {} - errors = [] + answer = None try: params = {"md5": md5, "analyzers_needed": analyzers_needed} if run_all_available_analyzers: @@ -75,8 +77,8 @@ def ask_analysis_availability( response.raise_for_status() answer = response.json() except Exception as e: - errors.append(str(e)) - return {"errors": errors, "answer": answer} + raise IntelOwlAPIException(e) + return answer def send_file_analysis_request( self, @@ -92,8 +94,7 @@ def send_file_analysis_request( ): if runtime_configuration is None: runtime_configuration = {} - answer = {} - errors = [] + answer = None try: data = { "md5": md5, @@ -114,8 +115,8 @@ def send_file_analysis_request( response.raise_for_status() answer = response.json() except Exception as e: - errors.append(str(e)) - return {"errors": errors, "answer": answer} + raise IntelOwlAPIException(e) + return answer def send_observable_analysis_request( self, @@ -128,10 +129,9 @@ def send_observable_analysis_request( run_all_available_analyzers: bool = False, runtime_configuration: Dict = {}, ): - answer = {} - errors = [] + answer = None if not md5: - md5 = hashlib.md5(observable_name.encode("utf-8")).hexdigest() + md5 = self.get_md5(observable_name, type_="observable") try: data = { "is_sample": False, @@ -154,12 +154,11 @@ def send_observable_analysis_request( response.raise_for_status() answer = response.json() except Exception as e: - errors.append(str(e)) - return answer, errors + raise IntelOwlAPIException(e) + return answer def ask_analysis_result(self, job_id): - answer = {} - errors = [] + answer = None try: params = {"job_id": job_id} url = self.instance + "/api/ask_analysis_result" @@ -168,25 +167,23 @@ def ask_analysis_result(self, job_id): response.raise_for_status() answer = response.json() except Exception as e: - errors.append(str(e)) - return {"errors": errors, "answer": answer} + raise IntelOwlAPIException(e) + return answer def get_analyzer_configs(self): answer = None - errors = [] try: url = self.instance + "/api/get_analyzer_configs" response = self.session.get(url) - logger.debug(response.url) + logger.debug(msg=(response.url, response.status_code)) response.raise_for_status() answer = response.json() except Exception as e: - errors.append(str(e)) - return answer, errors + raise IntelOwlAPIException(e) + return answer def get_all_tags(self): answer = None - errors = [] try: url = self.instance + "/api/tags" response = self.session.get(url) @@ -194,12 +191,11 @@ def get_all_tags(self): response.raise_for_status() answer = response.json() except Exception as e: - errors.append(str(e)) - return answer, errors + raise IntelOwlAPIException(e) + return answer def get_all_jobs(self): answer = None - errors = [] try: url = self.instance + "/api/jobs" response = self.session.get(url) @@ -207,12 +203,11 @@ def get_all_jobs(self): response.raise_for_status() answer = response.json() except Exception as e: - errors.append(str(e)) - return answer, errors + raise IntelOwlAPIException(e) + return answer def get_tag_by_id(self, tag_id): answer = None - errors = [] try: url = self.instance + "/api/tags/" response = self.session.get(url + str(tag_id)) @@ -220,12 +215,11 @@ def get_tag_by_id(self, tag_id): response.raise_for_status() answer = response.json() except Exception as e: - errors.append(str(e)) - return answer, errors + raise IntelOwlAPIException(e) + return answer def get_job_by_id(self, job_id): answer = None - errors = [] try: url = self.instance + "/api/jobs/" + str(job_id) response = self.session.get(url) @@ -233,8 +227,49 @@ def get_job_by_id(self, job_id): response.raise_for_status() answer = response.json() except Exception as e: - errors.append(str(e)) - return answer, errors + raise IntelOwlAPIException(e) + return answer + + def __check_existing( + self, md5: str, analyzers_list, run_all: bool, check_reported: bool + ): + ans = self.ask_analysis_availability( + md5, + analyzers_list, + run_all, + check_reported, + ) + status = ans.get("status", None) + if not status: + raise IntelOwlClientException( + "API ask_analysis_availability gave result without status!?!?" + f" Answer: {ans}" + ) + if status != "not_available": + job_id_to_get = ans.get("job_id", None) + if job_id_to_get: + logger.info( + f"[INFO] already existing Job(#{job_id_to_get}, md5: {md5}," + f" status: {status}) with analyzers: {analyzers_list}" + ) + else: + raise IntelOwlClientException( + "API ask_analysis_availability gave result without job_id!?!?" + f" Answer: {ans}" + ) + return status != ("not_available") + + @staticmethod + def get_md5(to_hash, type_="observable"): + if type_ == "observable": + md5 = hashlib.md5(str(to_hash).lower().encode("utf-8")).hexdigest() + else: + path = pathlib.Path(to_hash) + if not path.exists(): + raise IntelOwlClientException(f"{to_hash} does not exists") + binary = path.read_bytes() + md5 = hashlib.md5(binary).hexdigest() + return md5 def get_observable_classification(value): From 9bb687e3812b58afd550bcd493a782c956d2c6fc Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Tue, 17 Nov 2020 18:11:00 +0530 Subject: [PATCH 10/38] Merge pull request #40 from CITIZENDOT/Rewrite-GAC --- cli/commands.py | 78 ++++++++++++++++++++++++++++++++++++++++++++----- cli/config.py | 12 ++++---- 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/cli/commands.py b/cli/commands.py index ea970e1..f930bc0 100644 --- a/cli/commands.py +++ b/cli/commands.py @@ -1,15 +1,77 @@ import click -from ._utils import ClickContext -from rich import print +import re +from ._utils import ClickContext, get_success_text, get_json_syntax +from rich.console import Console +from rich.table import Table +from rich import box + +from pyintelowl.pyintelowl import IntelOwlAPIException @click.command( - short_help="Get current state of `analyzer_config.json` from the IntelOwl instance" + short_help="Get current state of `analyzer_config.json` from the IntelOwl instance", +) +@click.option( + "-m", + "--re-match", + help="RegEx Pattern to filter analyzer names against", ) +@click.option("-j", "--json", is_flag=True, help="Pretty print as JSON") +@click.option("-t", "--text", is_flag=True, help="Print analyzer names as CSV") @click.pass_context -def get_analyzer_config(ctx: ClickContext): - res, err = ctx.obj.get_analyzer_configs() - if err: - print(err) +def get_analyzer_config(ctx: ClickContext, re_match: str, json: bool, text: bool): + console = Console() + ctx.obj.logger.info("Requesting [italic blue]analyzer_config.json[/]..") + try: + res = ctx.obj.get_analyzer_configs() + # filter resulset if a regex pattern was provided + if re_match: + pat = re.compile(re_match) + res = {k: v for k, v in res.items() if pat.match(k) is not None} + except IntelOwlAPIException as e: + ctx.obj.logger.fatal(str(e)) + ctx.exit(0) + if json: + with console.pager(styles=True): + console.print(res) + elif text: + click.echo(", ".join(res.keys())) else: - print(res) + # otherwise, print full table + headers = [ + "Name", + "Type", + "Description", + "Supported\nTypes", + "External\nService", + "Leaks\nInfo", + "Requires\nConfig", + "Additional\nConfig\nParams", + ] + header_style = "bold blue" + table = Table( + show_header=True, + title="Analyzer Configurations", + box=box.DOUBLE_EDGE, + show_lines=True, + ) + for h in headers: + table.add_column(h, header_style=header_style, justify="center") + for name, obj in res.items(): + table.add_row( + name, + obj["type"], + obj.get("description", ""), + get_json_syntax( + obj.get( + "observable_supported", + obj.get("supported_filetypes", []), + ) + ), + get_success_text(obj.get("external_service", False)), + get_success_text(obj.get("leaks_info", False)), + get_success_text(obj.get("requires_configuration", False)), + get_json_syntax(obj.get("additional_config_params", {})), + ) + with console.pager(styles=True): + console.print(table) diff --git a/cli/config.py b/cli/config.py index 12ecfbb..42b576c 100644 --- a/cli/config.py +++ b/cli/config.py @@ -18,11 +18,13 @@ def config_get(): Pretty Print config variables """ _, host = get_netrc_obj() - rprint({ - "api_key": host["password"], - "instance_url": host["account"], - "certificate": host["login"], - }) + rprint( + { + "api_key": host["password"], + "instance_url": host["account"], + "certificate": host["login"], + } + ) @config.command("set") From f079f518e267c32cdd15fd700496caab60ab673d Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Tue, 17 Nov 2020 18:12:07 +0530 Subject: [PATCH 11/38] add shebang line --- cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cli.py b/cli.py index d453059..c5737f9 100644 --- a/cli.py +++ b/cli.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import click from pyintelowl.pyintelowl import IntelOwl from cli import groups, cmds From 915907e1ca6f4333f16988d403643d320d47253f Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Tue, 17 Nov 2020 18:53:30 +0530 Subject: [PATCH 12/38] black format, shell-completion, refactor directory path, add entryscript for direct invocation --- cli.py | 14 ++++++++------ {cli => pyintelowl/cli}/__init__.py | 0 {cli => pyintelowl/cli}/_utils.py | 5 +---- {cli => pyintelowl/cli}/analyse.py | 0 {cli => pyintelowl/cli}/commands.py | 0 {cli => pyintelowl/cli}/config.py | 0 .../cli/domain_checkers.py | 0 {cli => pyintelowl/cli}/jobs.py | 8 +++++++- {cli => pyintelowl/cli}/tags.py | 4 +--- setup.py | 6 ++++++ 10 files changed, 23 insertions(+), 14 deletions(-) rename {cli => pyintelowl/cli}/__init__.py (100%) rename {cli => pyintelowl/cli}/_utils.py (93%) rename {cli => pyintelowl/cli}/analyse.py (100%) rename {cli => pyintelowl/cli}/commands.py (100%) rename {cli => pyintelowl/cli}/config.py (100%) rename domain_checkers.py => pyintelowl/cli/domain_checkers.py (100%) rename {cli => pyintelowl/cli}/jobs.py (97%) rename {cli => pyintelowl/cli}/tags.py (91%) diff --git a/cli.py b/cli.py index c5737f9..36e0d68 100644 --- a/cli.py +++ b/cli.py @@ -1,16 +1,19 @@ #!/usr/bin/env python3 import click +import click_completion from pyintelowl.pyintelowl import IntelOwl -from cli import groups, cmds -from cli._utils import get_logger, ClickContext, get_netrc_obj +from pyintelowl.cli import groups, cmds +from pyintelowl.cli._utils import get_logger, ClickContext, get_netrc_obj + + +# Enable auto completion +click_completion.init() @click.group(context_settings=dict(help_option_names=["-h", "--help"])) @click.option("-d", "--debug", is_flag=True, help="Set log level to DEBUG") @click.pass_context -def cli( - ctx: ClickContext, debug: bool -): +def cli(ctx: ClickContext, debug: bool): netrc, host = get_netrc_obj() api_key, url, cert = host["password"], host["account"], host["login"] if not api_key or not url: @@ -24,7 +27,6 @@ def cli( for c in groups + cmds: cli.add_command(c) - # Entrypoint/executor if __name__ == "__main__": cli() diff --git a/cli/__init__.py b/pyintelowl/cli/__init__.py similarity index 100% rename from cli/__init__.py rename to pyintelowl/cli/__init__.py diff --git a/cli/_utils.py b/pyintelowl/cli/_utils.py similarity index 93% rename from cli/_utils.py rename to pyintelowl/cli/_utils.py index 1025877..8d36a78 100644 --- a/cli/_utils.py +++ b/pyintelowl/cli/_utils.py @@ -66,10 +66,7 @@ def get_netrc_obj(): def get_tags_str(tags): tags_str = ", ".join( - [ - "[on {0}]{1}[/]".format(str(t["color"]).lower(), t["label"]) - for t in tags - ] + ["[on {0}]{1}[/]".format(str(t["color"]).lower(), t["label"]) for t in tags] ) return tags_str diff --git a/cli/analyse.py b/pyintelowl/cli/analyse.py similarity index 100% rename from cli/analyse.py rename to pyintelowl/cli/analyse.py diff --git a/cli/commands.py b/pyintelowl/cli/commands.py similarity index 100% rename from cli/commands.py rename to pyintelowl/cli/commands.py diff --git a/cli/config.py b/pyintelowl/cli/config.py similarity index 100% rename from cli/config.py rename to pyintelowl/cli/config.py diff --git a/domain_checkers.py b/pyintelowl/cli/domain_checkers.py similarity index 100% rename from domain_checkers.py rename to pyintelowl/cli/domain_checkers.py diff --git a/cli/jobs.py b/pyintelowl/cli/jobs.py similarity index 97% rename from cli/jobs.py rename to pyintelowl/cli/jobs.py index 4f5726a..6aefcb4 100644 --- a/cli/jobs.py +++ b/pyintelowl/cli/jobs.py @@ -7,7 +7,13 @@ from rich import box, print as rprint from pyintelowl.exceptions import IntelOwlAPIException -from ._utils import ClickContext, get_status_text, get_success_text, get_json_syntax, get_tags_str +from ._utils import ( + ClickContext, + get_status_text, + get_success_text, + get_json_syntax, + get_tags_str, +) @click.group("jobs", short_help="Manage Jobs", invoke_without_command=True) diff --git a/cli/tags.py b/pyintelowl/cli/tags.py similarity index 91% rename from cli/tags.py rename to pyintelowl/cli/tags.py index 760429e..de368eb 100644 --- a/cli/tags.py +++ b/pyintelowl/cli/tags.py @@ -37,9 +37,7 @@ def _print_tags_table(ctx, rows): for row in rows: color = str(row["color"]).lower() table.add_row( - str(row["id"]), - str(row["label"]), - Text(color, style=f"on {color}") + str(row["id"]), str(row["label"]), Text(color, style=f"on {color}") ) console.print(table, justify="center") except Exception as e: diff --git a/setup.py b/setup.py index 4975b5f..15b3316 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ "click==7.1.2", "rich==9.2.0", "click-spinner==0.1.10", + "click_completion", "tinynetrc==1.3.0", ] @@ -71,4 +72,9 @@ "dev": ["black==20.8b1", "flake8"] + requirements, "test": ["black==20.8b1", "flake8"] + requirements, }, + # pip install --editable . + entry_points=""" + [console_scripts] + pyintelowl=cli:cli + """, ) From 2634ba828ff65b571a044b9a91d1a86b41bf6249 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 18 Nov 2020 14:53:58 +0530 Subject: [PATCH 13/38] fix docs url --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 15b3316..a408c8e 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ include_package_data=True, install_requires=requirements, project_urls={ - "Documentation": "https://django-rest-durin.readthedocs.io/", + "Documentation": GITHUB_URL, "Funding": "https://liberapay.com/IntelOwlProject/", "Source": GITHUB_URL, "Tracker": "{}/issues".format(GITHUB_URL), From d08776b3c6429f31385390550f90bb395445e971 Mon Sep 17 00:00:00 2001 From: Appaji Chintimi Date: Wed, 18 Nov 2020 17:06:03 +0530 Subject: [PATCH 14/38] Added file listing with status --- pyintelowl/cli/jobs.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pyintelowl/cli/jobs.py b/pyintelowl/cli/jobs.py index 6aefcb4..2b5be3a 100644 --- a/pyintelowl/cli/jobs.py +++ b/pyintelowl/cli/jobs.py @@ -19,8 +19,23 @@ @click.group("jobs", short_help="Manage Jobs", invoke_without_command=True) @click.option("-a", "--all", is_flag=True, help="List all jobs") @click.option("--id", type=int, default=0, help="Retrieve Job details by ID") +@click.option( + "--status", + type=click.Choice( + [ + "pending", + "running", + "reported_without_fails", + "reported_with_fails", + "failed", + ], + case_sensitive=False, + ), + show_choices=True, + help="Only use with --all (Case Insensitive)", +) @click.pass_context -def jobs(ctx: ClickContext, id: int, all: bool): +def jobs(ctx: ClickContext, id: int, all: bool, status: str): """ Manage Jobs """ @@ -29,6 +44,8 @@ def jobs(ctx: ClickContext, id: int, all: bool): ctx.obj.logger.info("Requesting list of jobs..") with click_spinner.spinner(): ans = ctx.obj.get_all_jobs() + if status: + ans = [el for el in ans if el["status"].lower() == status.lower()] _display_all_jobs(ans) elif id: ctx.obj.logger.info(f"Requesting Job [underline blue]#{id}[/]..") From ffbbb0e0ed37e75f7107b44f082c111d41c47ef8 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 18 Nov 2020 19:37:04 +0530 Subject: [PATCH 15/38] Update pyintelowl/cli/jobs.py --- pyintelowl/cli/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyintelowl/cli/jobs.py b/pyintelowl/cli/jobs.py index 2b5be3a..08b770a 100644 --- a/pyintelowl/cli/jobs.py +++ b/pyintelowl/cli/jobs.py @@ -32,7 +32,7 @@ case_sensitive=False, ), show_choices=True, - help="Only use with --all (Case Insensitive)", + help="Filter jobs with a particular status. Should be used with --all." ) @click.pass_context def jobs(ctx: ClickContext, id: int, all: bool, status: str): From 1d99c0669e2657375f06dba50dd8bc5b44bb2e00 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Tue, 17 Nov 2020 18:53:30 +0530 Subject: [PATCH 16/38] black format, shell-completion, refactor directory path, add entryscript for direct invocation --- cli.py | 14 ++++++++------ {cli => pyintelowl/cli}/__init__.py | 0 {cli => pyintelowl/cli}/_utils.py | 5 +---- {cli => pyintelowl/cli}/analyse.py | 0 {cli => pyintelowl/cli}/commands.py | 6 +++--- {cli => pyintelowl/cli}/config.py | 2 +- .../cli/domain_checkers.py | 0 {cli => pyintelowl/cli}/jobs.py | 12 +++++++++--- {cli => pyintelowl/cli}/tags.py | 10 ++++------ setup.py | 6 ++++++ 10 files changed, 32 insertions(+), 23 deletions(-) rename {cli => pyintelowl/cli}/__init__.py (100%) rename {cli => pyintelowl/cli}/_utils.py (93%) rename {cli => pyintelowl/cli}/analyse.py (100%) rename {cli => pyintelowl/cli}/commands.py (93%) rename {cli => pyintelowl/cli}/config.py (96%) rename domain_checkers.py => pyintelowl/cli/domain_checkers.py (100%) rename {cli => pyintelowl/cli}/jobs.py (95%) rename {cli => pyintelowl/cli}/tags.py (82%) diff --git a/cli.py b/cli.py index c5737f9..36e0d68 100644 --- a/cli.py +++ b/cli.py @@ -1,16 +1,19 @@ #!/usr/bin/env python3 import click +import click_completion from pyintelowl.pyintelowl import IntelOwl -from cli import groups, cmds -from cli._utils import get_logger, ClickContext, get_netrc_obj +from pyintelowl.cli import groups, cmds +from pyintelowl.cli._utils import get_logger, ClickContext, get_netrc_obj + + +# Enable auto completion +click_completion.init() @click.group(context_settings=dict(help_option_names=["-h", "--help"])) @click.option("-d", "--debug", is_flag=True, help="Set log level to DEBUG") @click.pass_context -def cli( - ctx: ClickContext, debug: bool -): +def cli(ctx: ClickContext, debug: bool): netrc, host = get_netrc_obj() api_key, url, cert = host["password"], host["account"], host["login"] if not api_key or not url: @@ -24,7 +27,6 @@ def cli( for c in groups + cmds: cli.add_command(c) - # Entrypoint/executor if __name__ == "__main__": cli() diff --git a/cli/__init__.py b/pyintelowl/cli/__init__.py similarity index 100% rename from cli/__init__.py rename to pyintelowl/cli/__init__.py diff --git a/cli/_utils.py b/pyintelowl/cli/_utils.py similarity index 93% rename from cli/_utils.py rename to pyintelowl/cli/_utils.py index 1025877..8d36a78 100644 --- a/cli/_utils.py +++ b/pyintelowl/cli/_utils.py @@ -66,10 +66,7 @@ def get_netrc_obj(): def get_tags_str(tags): tags_str = ", ".join( - [ - "[on {0}]{1}[/]".format(str(t["color"]).lower(), t["label"]) - for t in tags - ] + ["[on {0}]{1}[/]".format(str(t["color"]).lower(), t["label"]) for t in tags] ) return tags_str diff --git a/cli/analyse.py b/pyintelowl/cli/analyse.py similarity index 100% rename from cli/analyse.py rename to pyintelowl/cli/analyse.py diff --git a/cli/commands.py b/pyintelowl/cli/commands.py similarity index 93% rename from cli/commands.py rename to pyintelowl/cli/commands.py index f930bc0..5805263 100644 --- a/cli/commands.py +++ b/pyintelowl/cli/commands.py @@ -1,11 +1,11 @@ import click import re -from ._utils import ClickContext, get_success_text, get_json_syntax +from _utils import ClickContext, get_success_text, get_json_syntax from rich.console import Console from rich.table import Table from rich import box -from pyintelowl.pyintelowl import IntelOwlAPIException +from pyintelowl.pyintelowl import IntelOwlClientException @click.command( @@ -28,7 +28,7 @@ def get_analyzer_config(ctx: ClickContext, re_match: str, json: bool, text: bool if re_match: pat = re.compile(re_match) res = {k: v for k, v in res.items() if pat.match(k) is not None} - except IntelOwlAPIException as e: + except IntelOwlClientException as e: ctx.obj.logger.fatal(str(e)) ctx.exit(0) if json: diff --git a/cli/config.py b/pyintelowl/cli/config.py similarity index 96% rename from cli/config.py rename to pyintelowl/cli/config.py index 42b576c..3212bd9 100644 --- a/cli/config.py +++ b/pyintelowl/cli/config.py @@ -1,7 +1,7 @@ import click from rich import print as rprint -from ._utils import get_netrc_obj, ClickContext +from _utils import get_netrc_obj, ClickContext @click.group("config") diff --git a/domain_checkers.py b/pyintelowl/cli/domain_checkers.py similarity index 100% rename from domain_checkers.py rename to pyintelowl/cli/domain_checkers.py diff --git a/cli/jobs.py b/pyintelowl/cli/jobs.py similarity index 95% rename from cli/jobs.py rename to pyintelowl/cli/jobs.py index 4f5726a..a11e12f 100644 --- a/cli/jobs.py +++ b/pyintelowl/cli/jobs.py @@ -6,8 +6,14 @@ from rich.console import RenderGroup from rich import box, print as rprint -from pyintelowl.exceptions import IntelOwlAPIException -from ._utils import ClickContext, get_status_text, get_success_text, get_json_syntax, get_tags_str +from pyintelowl.exceptions import IntelOwlClientException +from _utils import ( + ClickContext, + get_status_text, + get_success_text, + get_json_syntax, + get_tags_str, +) @click.group("jobs", short_help="Manage Jobs", invoke_without_command=True) @@ -31,7 +37,7 @@ def jobs(ctx: ClickContext, id: int, all: bool): _display_single_job(ans) else: click.echo(ctx.get_usage()) - except IntelOwlAPIException as e: + except IntelOwlClientException as e: ctx.obj.logger.fatal(str(e)) diff --git a/cli/tags.py b/pyintelowl/cli/tags.py similarity index 82% rename from cli/tags.py rename to pyintelowl/cli/tags.py index 760429e..7d53ab4 100644 --- a/cli/tags.py +++ b/pyintelowl/cli/tags.py @@ -4,9 +4,9 @@ from rich.text import Text from rich import box -from pyintelowl.exceptions import IntelOwlAPIException +from pyintelowl.exceptions import IntelOwlClientException -from ._utils import ClickContext +from _utils import ClickContext @click.command(short_help="Manage tags") @@ -24,7 +24,7 @@ def tags(ctx: ClickContext, id: int, all: bool): ans = ctx.obj.get_tag_by_id(id) ans = [ans] _print_tags_table(ctx, ans) - except IntelOwlAPIException as e: + except IntelOwlClientException as e: ctx.obj.logger.fatal(str(e)) @@ -37,9 +37,7 @@ def _print_tags_table(ctx, rows): for row in rows: color = str(row["color"]).lower() table.add_row( - str(row["id"]), - str(row["label"]), - Text(color, style=f"on {color}") + str(row["id"]), str(row["label"]), Text(color, style=f"on {color}") ) console.print(table, justify="center") except Exception as e: diff --git a/setup.py b/setup.py index 4975b5f..15b3316 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ "click==7.1.2", "rich==9.2.0", "click-spinner==0.1.10", + "click_completion", "tinynetrc==1.3.0", ] @@ -71,4 +72,9 @@ "dev": ["black==20.8b1", "flake8"] + requirements, "test": ["black==20.8b1", "flake8"] + requirements, }, + # pip install --editable . + entry_points=""" + [console_scripts] + pyintelowl=cli:cli + """, ) From 785bbad3033c16dd0aa8898ab1e2a76ec99f3e88 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 18 Nov 2020 14:53:58 +0530 Subject: [PATCH 17/38] fix docs url --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 15b3316..a408c8e 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ include_package_data=True, install_requires=requirements, project_urls={ - "Documentation": "https://django-rest-durin.readthedocs.io/", + "Documentation": GITHUB_URL, "Funding": "https://liberapay.com/IntelOwlProject/", "Source": GITHUB_URL, "Tracker": "{}/issues".format(GITHUB_URL), From 5cd0306e7a24b21f851813068db99a5130a94347 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 18 Nov 2020 20:10:08 +0530 Subject: [PATCH 18/38] revamp logging, err handling in IntelOwl cls | various improvements --- cli.py | 4 +- pyintelowl/exceptions.py | 4 - pyintelowl/pyintelowl.py | 283 +++++++++++++++++++++++---------------- 3 files changed, 173 insertions(+), 118 deletions(-) diff --git a/cli.py b/cli.py index 36e0d68..4f4d51e 100644 --- a/cli.py +++ b/cli.py @@ -19,8 +19,8 @@ def cli(ctx: ClickContext, debug: bool): if not api_key or not url: click.echo("Hint: Use `config set` to set config variables!") else: - ctx.obj = IntelOwl(api_key, url, cert, debug) - ctx.obj.logger = get_logger() + logger = get_logger("DEBUG" if debug else "INFO") + ctx.obj = IntelOwl(api_key, url, cert, logger) # Compile all groups and commands diff --git a/pyintelowl/exceptions.py b/pyintelowl/exceptions.py index d4a7d91..5e30296 100644 --- a/pyintelowl/exceptions.py +++ b/pyintelowl/exceptions.py @@ -1,7 +1,3 @@ -class IntelOwlAPIException(Exception): - pass - - class IntelOwlClientException(Exception): pass diff --git a/pyintelowl/pyintelowl.py b/pyintelowl/pyintelowl.py index aa42a26..72a2c1a 100644 --- a/pyintelowl/pyintelowl.py +++ b/pyintelowl/pyintelowl.py @@ -3,16 +3,13 @@ import pathlib import re import requests -import sys import hashlib from typing import List, Dict from json import dumps as json_dumps -from .exceptions import IntelOwlClientException, IntelOwlAPIException - -logger = logging.getLogger(__name__) +from .exceptions import IntelOwlClientException class IntelOwl: @@ -23,22 +20,15 @@ def __init__( token: str, instance_url: str, certificate: str = None, - debug: bool = False, + logger: logging.Logger = None, ): self.token = token self.instance = instance_url self.certificate = certificate - self.__debug__hndlr = logging.StreamHandler(sys.stdout) - self.debug(debug) - - def debug(self, on: bool) -> None: - if on: - # if debug add stdout logging - logger.setLevel(logging.DEBUG) - logger.addHandler(self.__debug__hndlr) + if logger: + self.logger = logger else: - logger.setLevel(logging.INFO) - logger.removeHandler(self.__debug__hndlr) + self.logger = logging.getLogger(__name__) @property def session(self): @@ -58,10 +48,10 @@ def session(self): def ask_analysis_availability( self, - md5, - analyzers_needed, - run_all_available_analyzers=False, - check_reported_analysis_too=False, + md5: str, + analyzers_needed: List[str], + run_all_available_analyzers: bool = False, + check_reported_analysis_too: bool = False, ): answer = None try: @@ -72,38 +62,46 @@ def ask_analysis_availability( params["running_only"] = True url = self.instance + "/api/ask_analysis_availability" response = self.session.get(url, params=params) - logger.debug(response.url) - logger.debug(response.headers) + self.logger.debug(msg=(response.url, response.status_code)) response.raise_for_status() answer = response.json() + status, job_id = answer.get("status", None), answer.get("job_id", None) + # check sanity cases + if not status: + raise IntelOwlClientException( + "API ask_analysis_availability gave result without status ?" + f" Response: {answer}" + ) + if status != "not_available" and not job_id: + raise IntelOwlClientException( + "API ask_analysis_availability gave result without job_id ?" + f" Response: {answer}" + ) except Exception as e: - raise IntelOwlAPIException(e) + raise IntelOwlClientException(e) return answer def send_file_analysis_request( self, - md5, - analyzers_requested, - filename, - binary, - force_privacy=False, - private_job=False, - disable_external_analyzers=False, - run_all_available_analyzers=False, - runtime_configuration=None, + analyzers_requested: List[str], + filename: str, + binary: bytes, + force_privacy: bool = False, + private_job: bool = False, + disable_external_analyzers: bool = False, + run_all_available_analyzers: bool = False, + runtime_configuration: Dict = {}, ): - if runtime_configuration is None: - runtime_configuration = {} answer = None try: data = { - "md5": md5, + "is_sample": True, + "md5": self.get_md5(binary, type_="binary"), "analyzers_requested": analyzers_requested, "run_all_available_analyzers": run_all_available_analyzers, "force_privacy": force_privacy, "private": private_job, "disable_external_analyzers": disable_external_analyzers, - "is_sample": True, "file_name": filename, } if runtime_configuration: @@ -111,18 +109,25 @@ def send_file_analysis_request( files = {"file": (filename, binary)} url = self.instance + "/api/send_analysis_request" response = self.session.post(url, data=data, files=files) - logger.debug(response.url) + self.logger.debug(msg=(response.url, response.status_code)) response.raise_for_status() answer = response.json() + warnings = answer["warnings"] + self.logger.info( + f"""New Job running.. + ID: {answer['job_id']} | Status: [underline pink]{answer['status']}[/]. + Got {len(warnings)} warnings: + [italic red]{warnings if warnings else None}[/] + """ + ) except Exception as e: - raise IntelOwlAPIException(e) + raise IntelOwlClientException(e) return answer def send_observable_analysis_request( self, analyzers_requested: List[str], observable_name: str, - md5: str = None, force_privacy: bool = False, private_job: bool = False, disable_external_analyzers: bool = False, @@ -130,19 +135,17 @@ def send_observable_analysis_request( runtime_configuration: Dict = {}, ): answer = None - if not md5: - md5 = self.get_md5(observable_name, type_="observable") try: data = { "is_sample": False, - "md5": md5, + "md5": self.get_md5(observable_name, type_="observable"), "analyzers_requested": analyzers_requested, "run_all_available_analyzers": run_all_available_analyzers, "force_privacy": force_privacy, "private": private_job, "disable_external_analyzers": disable_external_analyzers, "observable_name": observable_name, - "observable_classification": get_observable_classification( + "observable_classification": self._get_observable_classification( observable_name ), } @@ -150,11 +153,19 @@ def send_observable_analysis_request( data["runtime_configuration"] = json_dumps(runtime_configuration) url = self.instance + "/api/send_analysis_request" response = self.session.post(url, data=data) - logger.debug(response.url) + self.logger.debug(msg=(response.url, response.status_code)) response.raise_for_status() answer = response.json() + warnings = answer["warnings"] + self.logger.info( + f"""New Job running.. + ID: {answer['job_id']} | Status: [underline pink]{answer['status']}[/]. + Got {len(warnings)} warnings: + [italic red]{warnings if warnings else None}[/] + """ + ) except Exception as e: - raise IntelOwlAPIException(e) + raise IntelOwlClientException(e) return answer def ask_analysis_result(self, job_id): @@ -163,11 +174,11 @@ def ask_analysis_result(self, job_id): params = {"job_id": job_id} url = self.instance + "/api/ask_analysis_result" response = self.session.get(url, params=params) - logger.debug(response.url) + self.logger.debug(response.url) response.raise_for_status() answer = response.json() except Exception as e: - raise IntelOwlAPIException(e) + raise IntelOwlClientException(e) return answer def get_analyzer_configs(self): @@ -175,11 +186,11 @@ def get_analyzer_configs(self): try: url = self.instance + "/api/get_analyzer_configs" response = self.session.get(url) - logger.debug(msg=(response.url, response.status_code)) + self.logger.debug(msg=(response.url, response.status_code)) response.raise_for_status() answer = response.json() except Exception as e: - raise IntelOwlAPIException(e) + raise IntelOwlClientException(e) return answer def get_all_tags(self): @@ -187,11 +198,11 @@ def get_all_tags(self): try: url = self.instance + "/api/tags" response = self.session.get(url) - logger.debug(response.url) + self.logger.debug(msg=(response.url, response.status_code)) response.raise_for_status() answer = response.json() except Exception as e: - raise IntelOwlAPIException(e) + raise IntelOwlClientException(e) return answer def get_all_jobs(self): @@ -199,11 +210,11 @@ def get_all_jobs(self): try: url = self.instance + "/api/jobs" response = self.session.get(url) - logger.debug(msg=(response.url, response.status_code)) + self.logger.debug(msg=(response.url, response.status_code)) response.raise_for_status() answer = response.json() except Exception as e: - raise IntelOwlAPIException(e) + raise IntelOwlClientException(e) return answer def get_tag_by_id(self, tag_id): @@ -211,11 +222,11 @@ def get_tag_by_id(self, tag_id): try: url = self.instance + "/api/tags/" response = self.session.get(url + str(tag_id)) - logger.debug(msg=(response.url, response.status_code)) + self.logger.debug(msg=(response.url, response.status_code)) response.raise_for_status() answer = response.json() except Exception as e: - raise IntelOwlAPIException(e) + raise IntelOwlClientException(e) return answer def get_job_by_id(self, job_id): @@ -223,47 +234,21 @@ def get_job_by_id(self, job_id): try: url = self.instance + "/api/jobs/" + str(job_id) response = self.session.get(url) - logger.debug(msg=(response.url, response.status_code)) + self.logger.debug(msg=(response.url, response.status_code)) response.raise_for_status() answer = response.json() except Exception as e: - raise IntelOwlAPIException(e) + raise IntelOwlClientException(e) return answer - def __check_existing( - self, md5: str, analyzers_list, run_all: bool, check_reported: bool - ): - ans = self.ask_analysis_availability( - md5, - analyzers_list, - run_all, - check_reported, - ) - status = ans.get("status", None) - if not status: - raise IntelOwlClientException( - "API ask_analysis_availability gave result without status!?!?" - f" Answer: {ans}" - ) - if status != "not_available": - job_id_to_get = ans.get("job_id", None) - if job_id_to_get: - logger.info( - f"[INFO] already existing Job(#{job_id_to_get}, md5: {md5}," - f" status: {status}) with analyzers: {analyzers_list}" - ) - else: - raise IntelOwlClientException( - "API ask_analysis_availability gave result without job_id!?!?" - f" Answer: {ans}" - ) - return status != ("not_available") - @staticmethod def get_md5(to_hash, type_="observable"): + md5 = None if type_ == "observable": md5 = hashlib.md5(str(to_hash).lower().encode("utf-8")).hexdigest() - else: + elif type_ == "binary": + md5 = hashlib.md5(to_hash).hexdigest() + elif type_ == "file": path = pathlib.Path(to_hash) if not path.exists(): raise IntelOwlClientException(f"{to_hash} does not exists") @@ -271,33 +256,107 @@ def get_md5(to_hash, type_="observable"): md5 = hashlib.md5(binary).hexdigest() return md5 + def _new_analysis_cli( + self, + obj: str, + type_: str, + analyzers_list: List[str], + run_all: bool, + force_privacy, + private_job, + disable_external_analyzers, + check, + ): + # CLI sanity checks + if analyzers_list and run_all: + self.logger.warn( + """ + Can't use -al and -aa options together. See usage with -h. + """ + ) + return + if not (analyzers_list or run_all): + self.logger.warn( + """ + Either one of -al, -aa must be specified. See usage with -h. + """, + ) + return + analyzers = analyzers_list if analyzers_list else "all available analyzers" + self.logger.info( + f"""Requesting analysis.. + {type_}: [bold blue underline]{val}[/] + analyzers: [italic green]{analyzers}[/] + """ + ) + # 1st step: ask analysis availability + md5 = self.get_md5(obj, type_=type_) + resp = self.ask_analysis_availability( + md5, analyzers_list, run_all, True if check == "reported" else False + ) + status, job_id = resp.get("status", None), resp.get("job_id", None) + if status != "not_available": + self.logger( + f"""Found existing analysis! + Job: #{job_id} + status: [underlined pink]{status}[/] -def get_observable_classification(value): - # only following types are supported: - # ip - domain - url - hash (md5, sha1, sha256) - try: - ipaddress.ip_address(value) - except ValueError: - if re.match( - "^(?:ht|f)tps?://[a-z\d-]{1,63}(?:\.[a-z\d-]{1,63})+" - "(?:/[a-z\d-]{1,63})*(?:\.\w+)?", - value, - ): - classification = "url" - elif re.match("^(\.)?[a-z\d-]{1,63}(\.[a-z\d-]{1,63})+$", value): - classification = "domain" - elif ( - re.match("^[a-f\d]{32}$", value) - or re.match("^[a-f\d]{40}$", value) - or re.match("^[a-f\d]{64}$", value) - ): - classification = "hash" + [i]Hint: use --check force-new to perform new scan anyway[/] + """ + ) + return + # 2nd step: send new analysis request + if type_ == "observable": + _ = self.send_observable_analysis_request( + analyzers_requested=analyzers, + observable_name=obj, + force_privacy=force_privacy, + private_job=private_job, + disable_external_analyzers=disable_external_analyzers, + run_all_available_analyzers=run_all, + ) else: - raise IntelOwlClientException( - f"{value} is neither a domain nor a URL nor a IP not a hash" + fname = obj.name + binary = pathlib.Path(obj).read_bytes() + _ = self.send_file_analysis_request( + analyzers_requested=analyzers, + filename=fname, + binary=binary, + force_privacy=force_privacy, + private_job=private_job, + disable_external_analyzers=disable_external_analyzers, + run_all_available_analyzers=run_all, ) - else: - # its a simple IP - classification = "ip" + # 3rd step: poll for result + # todo + + @staticmethod + def _get_observable_classification(value): + # only following types are supported: + # ip - domain - url - hash (md5, sha1, sha256) + try: + ipaddress.ip_address(value) + except ValueError: + if re.match( + "^(?:ht|f)tps?://[a-z\d-]{1,63}(?:\.[a-z\d-]{1,63})+" + "(?:/[a-z\d-]{1,63})*(?:\.\w+)?", + value, + ): + classification = "url" + elif re.match("^(\.)?[a-z\d-]{1,63}(\.[a-z\d-]{1,63})+$", value): + classification = "domain" + elif ( + re.match("^[a-f\d]{32}$", value) + or re.match("^[a-f\d]{40}$", value) + or re.match("^[a-f\d]{64}$", value) + ): + classification = "hash" + else: + raise IntelOwlClientException( + f"{value} is neither a domain nor a URL nor a IP not a hash" + ) + else: + # its a simple IP + classification = "ip" - return classification + return classification From ed44e40ac80ecc3274ea5460a48aa791919e16c1 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 18 Nov 2020 21:08:03 +0530 Subject: [PATCH 19/38] rewrite analyse.py with latest changes in IntelOwl cls --- pyintelowl/cli/analyse.py | 75 ++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 45 deletions(-) diff --git a/pyintelowl/cli/analyse.py b/pyintelowl/cli/analyse.py index 15a91b2..f572af6 100644 --- a/pyintelowl/cli/analyse.py +++ b/pyintelowl/cli/analyse.py @@ -1,11 +1,12 @@ import click -from ._utils import add_options, ClickContext -from .jobs import _display_single_job + +from _utils import add_options, ClickContext + __analyse_options = [ click.option( - "-al", - "--analyzers-list", + "-a", + "--analyzers", multiple=True, type=str, default=(), @@ -73,64 +74,48 @@ def analyse(): def observable( ctx: ClickContext, value, - analyzers_list, + analyzers, run_all, force_privacy, private_job, disable_external_analyzers, check, ): - if analyzers_list and run_all: - logger.warn( - """ - Can't use -al and -aa options together. See usage with -h. - """ - ) - ctx.exit(-1) - if not (analyzers_list or run_all): - logger.warn( - """ - Either one of -al, -aa must be specified. See usage with -h. - """, - ) - ctx.exit(-1) - analyzers = analyzers_list if analyzers_list else "all available analyzers" - ctx.obj.logger.info( - f"""Requesting analysis.. - observable: [bold blue underline]{value}[/] - analyzers: [italic green]{analyzers}[/] - """, - ) - # first step: ask analysis availability - ans = ctx.obj.send_observable_analysis_request( - analyzers_requested=analyzers_list, - observable_name=value, - force_privacy=force_privacy, - private_job=private_job, - disable_external_analyzers=disable_external_analyzers, - run_all_available_analyzers=run_all, - ) - warnings = ans["warnings"] - ctx.obj.logger.info( - f"""New Job running.. - ID: {ans['job_id']} | Status: [underline pink]{ans['status']}[/]. - Got {len(warnings)} warnings: [italic red]{warnings if warnings else None}[/] - """ + ctx.obj._new_analysis_cli( + ctx, + value, + "observable", + analyzers, + run_all, + force_privacy, + private_job, + disable_external_analyzers, + check, ) @analyse.command(short_help="Send analysis request for a file") -@click.argument("filename", type=click.Path(exists=True)) +@click.argument("filepath", type=click.Path(exists=True)) @add_options(__analyse_options) @click.pass_context def file( ctx: ClickContext, - filename, - analyzers_list, + filepath: click.Path, + analyzers, run_all, force_privacy, private_job, disable_external_analyzers, check, ): - pass + ctx.obj._new_analysis_cli( + ctx, + filepath, + "observable", + analyzers, + run_all, + force_privacy, + private_job, + disable_external_analyzers, + check, + ) From c61427a2c491928bbecb013b9f61bac3cb56a7b5 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 18 Nov 2020 21:17:21 +0530 Subject: [PATCH 20/38] fix relative imports, black format, replace short_help with help --- pyintelowl/cli/analyse.py | 6 +++--- pyintelowl/cli/commands.py | 4 ++-- pyintelowl/cli/config.py | 2 +- pyintelowl/cli/jobs.py | 8 ++++---- pyintelowl/cli/tags.py | 4 ++-- setup.py | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pyintelowl/cli/analyse.py b/pyintelowl/cli/analyse.py index f572af6..69e6fe7 100644 --- a/pyintelowl/cli/analyse.py +++ b/pyintelowl/cli/analyse.py @@ -1,6 +1,6 @@ import click -from _utils import add_options, ClickContext +from ..cli._utils import add_options, ClickContext __analyse_options = [ @@ -67,7 +67,7 @@ def analyse(): pass -@analyse.command(short_help="Send analysis request for an observable") +@analyse.command(help="Send analysis request for an observable") @click.argument("value") @add_options(__analyse_options) @click.pass_context @@ -94,7 +94,7 @@ def observable( ) -@analyse.command(short_help="Send analysis request for a file") +@analyse.command(help="Send analysis request for a file") @click.argument("filepath", type=click.Path(exists=True)) @add_options(__analyse_options) @click.pass_context diff --git a/pyintelowl/cli/commands.py b/pyintelowl/cli/commands.py index 5805263..a58a983 100644 --- a/pyintelowl/cli/commands.py +++ b/pyintelowl/cli/commands.py @@ -1,15 +1,15 @@ import click import re -from _utils import ClickContext, get_success_text, get_json_syntax from rich.console import Console from rich.table import Table from rich import box +from ..cli._utils import ClickContext, get_success_text, get_json_syntax from pyintelowl.pyintelowl import IntelOwlClientException @click.command( - short_help="Get current state of `analyzer_config.json` from the IntelOwl instance", + help="Get current state of `analyzer_config.json` from the IntelOwl instance", ) @click.option( "-m", diff --git a/pyintelowl/cli/config.py b/pyintelowl/cli/config.py index 3212bd9..9ce22f6 100644 --- a/pyintelowl/cli/config.py +++ b/pyintelowl/cli/config.py @@ -1,7 +1,7 @@ import click from rich import print as rprint -from _utils import get_netrc_obj, ClickContext +from ..cli._utils import get_netrc_obj, ClickContext @click.group("config") diff --git a/pyintelowl/cli/jobs.py b/pyintelowl/cli/jobs.py index 99fcf94..521d6c0 100644 --- a/pyintelowl/cli/jobs.py +++ b/pyintelowl/cli/jobs.py @@ -7,7 +7,7 @@ from rich import box, print as rprint from pyintelowl.exceptions import IntelOwlClientException -from _utils import ( +from ..cli._utils import ( ClickContext, get_status_text, get_success_text, @@ -16,7 +16,7 @@ ) -@click.group("jobs", short_help="Manage Jobs", invoke_without_command=True) +@click.group("jobs", help="Manage Jobs", invoke_without_command=True) @click.option("-a", "--all", is_flag=True, help="List all jobs") @click.option("--id", type=int, default=0, help="Retrieve Job details by ID") @click.option( @@ -32,7 +32,7 @@ case_sensitive=False, ), show_choices=True, - help="Filter jobs with a particular status. Should be used with --all." + help="Filter jobs with a particular status. Should be used with --all.", ) @click.pass_context def jobs(ctx: ClickContext, id: int, all: bool, status: str): @@ -58,7 +58,7 @@ def jobs(ctx: ClickContext, id: int, all: bool, status: str): ctx.obj.logger.fatal(str(e)) -@jobs.command("poll", short_help="HTTP poll a currently running job's details") +@jobs.command("poll", help="HTTP poll a currently running job's details") @click.option( "-t", "--max-tries", diff --git a/pyintelowl/cli/tags.py b/pyintelowl/cli/tags.py index 7d53ab4..338fc93 100644 --- a/pyintelowl/cli/tags.py +++ b/pyintelowl/cli/tags.py @@ -6,10 +6,10 @@ from pyintelowl.exceptions import IntelOwlClientException -from _utils import ClickContext +from ..cli._utils import ClickContext -@click.command(short_help="Manage tags") +@click.command(help="Manage tags") @click.option("-a", "--all", is_flag=True, help="List all tags") @click.option("--id", type=int, default=0, help="Retrieve tag details by ID") @click.pass_context diff --git a/setup.py b/setup.py index a408c8e..90ae11a 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ "dev": ["black==20.8b1", "flake8"] + requirements, "test": ["black==20.8b1", "flake8"] + requirements, }, - # pip install --editable . + # pip install --editable . entry_points=""" [console_scripts] pyintelowl=cli:cli From b1d2ba5e6ad6296357ba6e98e7c9ca2d18ce0966 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 18 Nov 2020 21:26:37 +0530 Subject: [PATCH 21/38] fixup! rewrite analyse.py --- pyintelowl/cli/analyse.py | 53 +++++++++++++++++++++------------------ pyintelowl/pyintelowl.py | 1 + 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/pyintelowl/cli/analyse.py b/pyintelowl/cli/analyse.py index 69e6fe7..71ed403 100644 --- a/pyintelowl/cli/analyse.py +++ b/pyintelowl/cli/analyse.py @@ -1,5 +1,6 @@ import click +from pyintelowl.exceptions import IntelOwlClientException from ..cli._utils import add_options, ClickContext @@ -81,26 +82,28 @@ def observable( disable_external_analyzers, check, ): - ctx.obj._new_analysis_cli( - ctx, - value, - "observable", - analyzers, - run_all, - force_privacy, - private_job, - disable_external_analyzers, - check, - ) + try: + ctx.obj._new_analysis_cli( + value, + "observable", + analyzers, + run_all, + force_privacy, + private_job, + disable_external_analyzers, + check, + ) + except IntelOwlClientException as e: + ctx.obj.logger.fatal(str(e)) @analyse.command(help="Send analysis request for a file") -@click.argument("filepath", type=click.Path(exists=True)) +@click.argument("filepath", type=click.Path(exists=True, resolve_path=True)) @add_options(__analyse_options) @click.pass_context def file( ctx: ClickContext, - filepath: click.Path, + filepath: str, analyzers, run_all, force_privacy, @@ -108,14 +111,16 @@ def file( disable_external_analyzers, check, ): - ctx.obj._new_analysis_cli( - ctx, - filepath, - "observable", - analyzers, - run_all, - force_privacy, - private_job, - disable_external_analyzers, - check, - ) + try: + ctx.obj._new_analysis_cli( + filepath, + "file", + analyzers, + run_all, + force_privacy, + private_job, + disable_external_analyzers, + check, + ) + except IntelOwlClientException as e: + ctx.obj.logger.fatal(str(e)) diff --git a/pyintelowl/pyintelowl.py b/pyintelowl/pyintelowl.py index 72a2c1a..727881d 100644 --- a/pyintelowl/pyintelowl.py +++ b/pyintelowl/pyintelowl.py @@ -283,6 +283,7 @@ def _new_analysis_cli( ) return analyzers = analyzers_list if analyzers_list else "all available analyzers" + val = obj if type_ == "observable" else obj.name self.logger.info( f"""Requesting analysis.. {type_}: [bold blue underline]{val}[/] From 60bb297785b992ea928a3bcfb3858f6f9399d537 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 18 Nov 2020 21:49:22 +0530 Subject: [PATCH 22/38] pretty/fix log msgs, fixes in IntelOwl cls --- pyintelowl/pyintelowl.py | 86 +++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/pyintelowl/pyintelowl.py b/pyintelowl/pyintelowl.py index 727881d..76e66be 100644 --- a/pyintelowl/pyintelowl.py +++ b/pyintelowl/pyintelowl.py @@ -5,7 +5,7 @@ import requests import hashlib -from typing import List, Dict +from typing import List, Dict, Any from json import dumps as json_dumps @@ -107,19 +107,7 @@ def send_file_analysis_request( if runtime_configuration: data["runtime_configuration"] = json_dumps(runtime_configuration) files = {"file": (filename, binary)} - url = self.instance + "/api/send_analysis_request" - response = self.session.post(url, data=data, files=files) - self.logger.debug(msg=(response.url, response.status_code)) - response.raise_for_status() - answer = response.json() - warnings = answer["warnings"] - self.logger.info( - f"""New Job running.. - ID: {answer['job_id']} | Status: [underline pink]{answer['status']}[/]. - Got {len(warnings)} warnings: - [italic red]{warnings if warnings else None}[/] - """ - ) + answer = self.__send_analysis_request(data=data, files=files) except Exception as e: raise IntelOwlClientException(e) return answer @@ -151,23 +139,41 @@ def send_observable_analysis_request( } if runtime_configuration: data["runtime_configuration"] = json_dumps(runtime_configuration) - url = self.instance + "/api/send_analysis_request" - response = self.session.post(url, data=data) - self.logger.debug(msg=(response.url, response.status_code)) - response.raise_for_status() - answer = response.json() - warnings = answer["warnings"] - self.logger.info( - f"""New Job running.. - ID: {answer['job_id']} | Status: [underline pink]{answer['status']}[/]. - Got {len(warnings)} warnings: - [italic red]{warnings if warnings else None}[/] - """ - ) + answer = self.__send_analysis_request(data=data, files=None) except Exception as e: raise IntelOwlClientException(e) return answer + def __send_analysis_request(self, data=None, files=None): + url = self.instance + "/api/send_analysis_request" + response = self.session.post(url, data=data, files=files) + self.logger.debug( + msg={ + "url": response.url, + "code": response.status_code, + "headers": response.headers, + "body": response.json(), + } + ) + answer = response.json() + if answer.get("error", "") == "814": + err = """ + Request failed.. + Error: [i yellow]After the filter, no analyzers can be run. + Try with other analyzers.[/] + """ + raise IntelOwlClientException(err) + warnings = answer.get("warnings", []) + self.logger.info( + f"""New Job running.. + ID: {answer['job_id']} | Status: [u blue]{answer['status']}[/]. + Got {len(warnings)} warnings: + [i yellow]{warnings if warnings else None}[/] + """ + ) + response.raise_for_status() + return answer + def ask_analysis_result(self, job_id): answer = None try: @@ -242,7 +248,7 @@ def get_job_by_id(self, job_id): return answer @staticmethod - def get_md5(to_hash, type_="observable"): + def get_md5(to_hash: Any, type_="observable"): md5 = None if type_ == "observable": md5 = hashlib.md5(str(to_hash).lower().encode("utf-8")).hexdigest() @@ -283,11 +289,10 @@ def _new_analysis_cli( ) return analyzers = analyzers_list if analyzers_list else "all available analyzers" - val = obj if type_ == "observable" else obj.name self.logger.info( f"""Requesting analysis.. - {type_}: [bold blue underline]{val}[/] - analyzers: [italic green]{analyzers}[/] + {type_}: [blue]{obj}[/] + analyzers: [i green]{analyzers}[/] """ ) # 1st step: ask analysis availability @@ -297,19 +302,19 @@ def _new_analysis_cli( ) status, job_id = resp.get("status", None), resp.get("job_id", None) if status != "not_available": - self.logger( + self.logger.info( f"""Found existing analysis! - Job: #{job_id} - status: [underlined pink]{status}[/] + Job: #{job_id} + status: [u blue]{status}[/] - [i]Hint: use --check force-new to perform new scan anyway[/] + [i]Hint: use [#854442]--check force-new[/] to perform new scan anyway[/] """ ) return # 2nd step: send new analysis request if type_ == "observable": _ = self.send_observable_analysis_request( - analyzers_requested=analyzers, + analyzers_requested=analyzers_list, observable_name=obj, force_privacy=force_privacy, private_job=private_job, @@ -317,12 +322,11 @@ def _new_analysis_cli( run_all_available_analyzers=run_all, ) else: - fname = obj.name - binary = pathlib.Path(obj).read_bytes() + path = pathlib.Path(obj) _ = self.send_file_analysis_request( - analyzers_requested=analyzers, - filename=fname, - binary=binary, + analyzers_requested=analyzers_list, + filename=path.name, + binary=path.read_bytes(), force_privacy=force_privacy, private_job=private_job, disable_external_analyzers=disable_external_analyzers, From 380e8aac7e1626c28d7ed4476860c8bd301766d5 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 18 Nov 2020 23:17:37 +0530 Subject: [PATCH 23/38] change analyzers list from multiple option to comma seperated string/list --- pyintelowl/cli/analyse.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pyintelowl/cli/analyse.py b/pyintelowl/cli/analyse.py index 71ed403..e39a02a 100644 --- a/pyintelowl/cli/analyse.py +++ b/pyintelowl/cli/analyse.py @@ -6,13 +6,12 @@ __analyse_options = [ click.option( - "-a", - "--analyzers", - multiple=True, + "-al", + "--analyzers-list", type=str, default=(), help=""" - List of analyzer names to invoke. Should not be used with + Comma seperated list of analyzer names to invoke. Should not be used with --run-all-available-analyzers """, ), @@ -75,18 +74,19 @@ def analyse(): def observable( ctx: ClickContext, value, - analyzers, + analyzers_list: str, run_all, force_privacy, private_job, disable_external_analyzers, check, ): + analyzers_list = analyzers_list.split(",") try: ctx.obj._new_analysis_cli( value, "observable", - analyzers, + analyzers_list, run_all, force_privacy, private_job, @@ -104,18 +104,19 @@ def observable( def file( ctx: ClickContext, filepath: str, - analyzers, + analyzers_list: str, run_all, force_privacy, private_job, disable_external_analyzers, check, ): + analyzers_list = analyzers_list.split(",") try: ctx.obj._new_analysis_cli( filepath, "file", - analyzers, + analyzers_list, run_all, force_privacy, private_job, From e9742d1b352ad585e9737759eec00601b95823e0 Mon Sep 17 00:00:00 2001 From: Appaji Chintimi Date: Thu, 19 Nov 2020 16:58:11 +0530 Subject: [PATCH 24/38] Added runtime_config Update pyintelowl/cli/_utils.py Co-authored-by: Eshaan Bansal --- pyintelowl/cli/_utils.py | 7 +++++++ pyintelowl/cli/analyse.py | 26 +++++++++++++++++++++++--- pyintelowl/pyintelowl.py | 3 +++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/pyintelowl/cli/_utils.py b/pyintelowl/cli/_utils.py index 8d36a78..f5234c2 100644 --- a/pyintelowl/cli/_utils.py +++ b/pyintelowl/cli/_utils.py @@ -78,3 +78,10 @@ def get_logger(level: str = "INFO"): ) logger = logging.getLogger("rich") return logger + + +def get_json_data(filepath): + obj = None + with open(filepath) as fp: + obj = json.load(fp) + return obj diff --git a/pyintelowl/cli/analyse.py b/pyintelowl/cli/analyse.py index e39a02a..db54304 100644 --- a/pyintelowl/cli/analyse.py +++ b/pyintelowl/cli/analyse.py @@ -1,7 +1,7 @@ import click from pyintelowl.exceptions import IntelOwlClientException -from ..cli._utils import add_options, ClickContext +from ..cli._utils import add_options, ClickContext, get_json_data __analyse_options = [ @@ -56,6 +56,12 @@ 3. 'force_new': force new analysis """, ), + click.option( + "-r", + "--runtime-config", + help="Path to JSON file which contains runtime_configuration.", + type=click.Path(exists=True, resolve_path=True), + ), ] @@ -80,8 +86,14 @@ def observable( private_job, disable_external_analyzers, check, + runtime_config, ): - analyzers_list = analyzers_list.split(",") + if not run_all: + analyzers_list = analyzers_list.split(",") + if runtime_config: + runtime_config = get_json_data(runtime_config) + else: + runtime_config = {} try: ctx.obj._new_analysis_cli( value, @@ -92,6 +104,7 @@ def observable( private_job, disable_external_analyzers, check, + runtime_config, ) except IntelOwlClientException as e: ctx.obj.logger.fatal(str(e)) @@ -110,8 +123,14 @@ def file( private_job, disable_external_analyzers, check, + runtime_config, ): - analyzers_list = analyzers_list.split(",") + if not run_all: + analyzers_list = analyzers_list.split(",") + if runtime_config: + runtime_config = get_json_data(runtime_config) + else: + runtime_config = {} try: ctx.obj._new_analysis_cli( filepath, @@ -122,6 +141,7 @@ def file( private_job, disable_external_analyzers, check, + runtime_config, ) except IntelOwlClientException as e: ctx.obj.logger.fatal(str(e)) diff --git a/pyintelowl/pyintelowl.py b/pyintelowl/pyintelowl.py index 76e66be..f3ae148 100644 --- a/pyintelowl/pyintelowl.py +++ b/pyintelowl/pyintelowl.py @@ -272,6 +272,7 @@ def _new_analysis_cli( private_job, disable_external_analyzers, check, + runtime_configuration: Dict = {}, ): # CLI sanity checks if analyzers_list and run_all: @@ -320,6 +321,7 @@ def _new_analysis_cli( private_job=private_job, disable_external_analyzers=disable_external_analyzers, run_all_available_analyzers=run_all, + runtime_configuration=runtime_configuration, ) else: path = pathlib.Path(obj) @@ -331,6 +333,7 @@ def _new_analysis_cli( private_job=private_job, disable_external_analyzers=disable_external_analyzers, run_all_available_analyzers=run_all, + runtime_configuration=runtime_configuration, ) # 3rd step: poll for result # todo From 0bbffa280b24cd32a33d5f969339fff6432f207e Mon Sep 17 00:00:00 2001 From: Appaji Chintimi Date: Sun, 22 Nov 2020 16:10:12 +0530 Subject: [PATCH 25/38] Added Batch Analysis Solves Issue #37 Added Runtime Config for Json, Csv --- pyintelowl/cli/_utils.py | 12 ++++++++++-- pyintelowl/cli/analyse.py | 22 ++++++++++++++++++++++ pyintelowl/pyintelowl.py | 31 +++++++++++++++++++++++++++---- 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/pyintelowl/cli/_utils.py b/pyintelowl/cli/_utils.py index f5234c2..cb1c259 100644 --- a/pyintelowl/cli/_utils.py +++ b/pyintelowl/cli/_utils.py @@ -1,5 +1,6 @@ import click import logging +import csv import json from tinynetrc import Netrc from rich.emoji import Emoji @@ -82,6 +83,13 @@ def get_logger(level: str = "INFO"): def get_json_data(filepath): obj = None - with open(filepath) as fp: - obj = json.load(fp) + with open(filepath) as _tmp: + line = _tmp.readline() + if line[0] in "[{": + with open(filepath) as fp: + obj = json.load(fp) + else: + with open(filepath) as fp: + reader = csv.DictReader(fp) + obj = [dict(row) for row in reader] return obj diff --git a/pyintelowl/cli/analyse.py b/pyintelowl/cli/analyse.py index db54304..fe9088e 100644 --- a/pyintelowl/cli/analyse.py +++ b/pyintelowl/cli/analyse.py @@ -145,3 +145,25 @@ def file( ) except IntelOwlClientException as e: ctx.obj.logger.fatal(str(e)) + + +@analyse.command( + help="Send multiple analysis requests. Reads file (csv or json) for inputs." +) +@click.argument("filepath", type=click.Path(exists=True, resolve_path=True)) +@click.pass_context +def batch( + ctx: ClickContext, + filepath: str, +): + rows = get_json_data(filepath) + flags = ["run_all", "force_privacy", "private_job", "disable_external_analyzers"] + for row in rows: + for flag in flags: + row[flag] = (row.get(flag, False).lower() == "true") | ( + row.get(flag, False) == True + ) + try: + ctx.obj.send_analysis_batch(rows) + except IntelOwlClientException as e: + ctx.obj.logger.fatal(str(e)) diff --git a/pyintelowl/pyintelowl.py b/pyintelowl/pyintelowl.py index f3ae148..bec00b4 100644 --- a/pyintelowl/pyintelowl.py +++ b/pyintelowl/pyintelowl.py @@ -1,14 +1,13 @@ import ipaddress import logging import pathlib +import json import re import requests import hashlib from typing import List, Dict, Any -from json import dumps as json_dumps - from .exceptions import IntelOwlClientException @@ -105,7 +104,7 @@ def send_file_analysis_request( "file_name": filename, } if runtime_configuration: - data["runtime_configuration"] = json_dumps(runtime_configuration) + data["runtime_configuration"] = json.dumps(runtime_configuration) files = {"file": (filename, binary)} answer = self.__send_analysis_request(data=data, files=files) except Exception as e: @@ -138,12 +137,36 @@ def send_observable_analysis_request( ), } if runtime_configuration: - data["runtime_configuration"] = json_dumps(runtime_configuration) + data["runtime_configuration"] = json.dumps(runtime_configuration) answer = self.__send_analysis_request(data=data, files=None) except Exception as e: raise IntelOwlClientException(e) return answer + def send_analysis_batch(self, rows: List = []): + for obj in rows: + try: + if obj.get("runtime_config", None): + with open(obj["runtime_config"]) as fp: + obj["runtime_config"] = json.load(fp) + + if not (obj.get("run_all", False)): + obj["analyzers_list"] = obj["analyzers_list"].split(",") + + self._new_analysis_cli( + obj["value"], + obj["type"], + obj.get("analyzers_list", None), + obj.get("run_all", False), + obj.get("force_privacy", False), + obj.get("private_job", False), + obj.get("disable_external_analyzers", False), + obj.get("check", None), + obj.get("runtime_config", {}), + ) + except IntelOwlClientException as e: + self.logger.fatal(str(e)) + def __send_analysis_request(self, data=None, files=None): url = self.instance + "/api/send_analysis_request" response = self.session.post(url, data=data, files=files) From 4fbab4707d90c6012d131d6ff225dff78bfd0c69 Mon Sep 17 00:00:00 2001 From: Ramnath Kumar Date: Fri, 20 Nov 2020 20:32:57 +0530 Subject: [PATCH 26/38] Rewrite job polls Error handling Fixed linting issues Added poll_for_job Fixed linting issues Added console.clear() after every iteration Edit exception handling message add poll command, refactor job display fn into smaller fns Write output to file Write to file, json --- pyintelowl/cli/_jobs_utils.py | 70 ++++++++++++++++++++++ pyintelowl/cli/_utils.py | 5 +- pyintelowl/cli/jobs.py | 110 ++++++++++++++++------------------ 3 files changed, 125 insertions(+), 60 deletions(-) create mode 100644 pyintelowl/cli/_jobs_utils.py diff --git a/pyintelowl/cli/_jobs_utils.py b/pyintelowl/cli/_jobs_utils.py new file mode 100644 index 0000000..b74e5b7 --- /dev/null +++ b/pyintelowl/cli/_jobs_utils.py @@ -0,0 +1,70 @@ +from rich import box +from rich.panel import Panel +from rich.table import Table +from rich.console import RenderGroup, Console +from ..cli._utils import ( + get_status_text, + get_success_text, + get_tags_str, + get_json_syntax, +) + + +def _display_single_job(data): + console = Console() + with console.pager(styles=True): + # print job attributes + attrs = _render_job_attributes(data) + console.print(attrs) + # construct job analysis table + table = _render_job_analysis_table(data["analysis_reports"], verbose=True) + console.print(table, justify="center") + + +def _render_job_analysis_table(rows, verbose=False): + if verbose: + headers = ["Name", "Status", "Report", "Errors"] + else: + headers = ["Name", "Status"] + table = Table( + show_header=True, + title="Analysis Data", + box=box.DOUBLE_EDGE, + show_lines=True, + ) + # add headers + for h in headers: + table.add_column(h, header_style="bold blue") + # add rows + for el in rows: + cols = [ + el["name"], + get_success_text((el["success"])), + ] + if verbose: + cols.append(get_json_syntax(el["report"]) if el["report"] else None) + cols.append(get_json_syntax(el["errors"]) if el["errors"] else None) + table.add_row(*cols) + return table + + +def _render_job_attributes(data): + style = "[bold #31DDCF]" + tags = get_tags_str(data["tags"]) + name = data["observable_name"] if data["observable_name"] else data["file_name"] + clsfn = ( + data["observable_classification"] + if data["observable_classification"] + else data["file_mimetype"] + ) + status: str = get_status_text(data["status"], as_text=False) + r = RenderGroup( + f"{style}Job ID:[/] {str(data['id'])}", + f"{style}User:[/] {data['source']}", + f"{style}MD5:[/] {data['md5']}", + f"{style}Name:[/] {name}", + f"{style}Classification:[/] {clsfn}", + f"{style}Tags:[/] {tags}", + f"{style}Status:[/] {status}", + ) + return Panel(r, title="Job attributes") diff --git a/pyintelowl/cli/_utils.py b/pyintelowl/cli/_utils.py index 8d36a78..2816581 100644 --- a/pyintelowl/cli/_utils.py +++ b/pyintelowl/cli/_utils.py @@ -17,7 +17,7 @@ class ClickContext(click.Context): """ -def get_status_text(status: str): +def get_status_text(status: str, as_text=True): styles = { "pending": ("#CE5C00", str(Emoji("gear"))), "running": ("#CE5C00", str(Emoji("gear"))), @@ -26,7 +26,8 @@ def get_status_text(status: str): "failed": ("#CC0000", str(Emoji("cross_mark"))), } color, emoji = styles[status] - return Text(status + " " + emoji, style=color) + s = f"[{color}]{status} {emoji}[/]" + return Text(status + " " + emoji, style=color) if as_text else s def get_success_text(success): diff --git a/pyintelowl/cli/jobs.py b/pyintelowl/cli/jobs.py index 521d6c0..f66efe5 100644 --- a/pyintelowl/cli/jobs.py +++ b/pyintelowl/cli/jobs.py @@ -1,18 +1,17 @@ +import time import click +import json import click_spinner from rich.console import Console from rich.table import Table -from rich.panel import Panel -from rich.console import RenderGroup from rich import box, print as rprint - +from rich.progress import track from pyintelowl.exceptions import IntelOwlClientException -from ..cli._utils import ( - ClickContext, - get_status_text, - get_success_text, - get_json_syntax, - get_tags_str, +from ..cli._utils import ClickContext, get_status_text +from ..cli._jobs_utils import ( + _display_single_job, + _render_job_attributes, + _render_job_analysis_table, ) @@ -53,19 +52,28 @@ def jobs(ctx: ClickContext, id: int, all: bool, status: str): ans = ctx.obj.get_job_by_id(id) _display_single_job(ans) else: - click.echo(ctx.get_usage()) + if not ctx.invoked_subcommand: + click.echo(ctx.get_usage()) except IntelOwlClientException as e: ctx.obj.logger.fatal(str(e)) @jobs.command("poll", help="HTTP poll a currently running job's details") +@click.option("--id", type=int, required=True, help="HTTP poll a job for live updates") +@click.option( + "-o", + "--output", + type=click.Path(exists=False, resolve_path=True), + required=True, + help="File to save results", +) @click.option( "-t", "--max-tries", type=int, default=0, show_default=True, - help="maximum number of tries (in sec)", + help="maximum number of tries", ) @click.option( "-i", @@ -73,58 +81,44 @@ def jobs(ctx: ClickContext, id: int, all: bool, status: str): type=int, default=5, show_default=True, - help="sleep interval before subsequent requests (in sec)", + help="sleep interval between subsequent requests (in sec)", ) @click.pass_context -def poll(ctx: ClickContext, max_tries: int, interval: int): - pass - - -def _display_single_job(data): +def poll(ctx: ClickContext, id: int, max_tries: int, interval: int, output: str): console = Console() - style = "[bold #31DDCF]" - headers = ["Name", "Status", "Report", "Errors"] - with console.pager(styles=True): - # print job attributes - tags = get_tags_str(data["tags"]) - status = get_status_text(data["status"]) - name = data["observable_name"] if data["observable_name"] else data["file_name"] - clsfn = ( - data["observable_classification"] - if data["observable_classification"] - else data["file_mimetype"] - ) - r = RenderGroup( - f"{style}Job ID:[/] {str(data['id'])}", - f"{style}User:[/] {data['source']}", - f"{style}MD5:[/] {data['md5']}", - f"{style}Name:[/] {name}", - f"{style}Classification:[/] {clsfn}", - f"{style}Tags:[/] {tags}", - f"{style}Status:[/] {status}", - ) - console.print(Panel(r, title="Job attributes")) - - # construct job analysis table + poll_result = {} + try: + for i in track( + range(max_tries), + description=f"Polling Job [underline blue]#{id}[/]..", + console=console, + ): + if i != 0: + console.print(f"sleeping for {interval} seconds before next request..") + time.sleep(interval) + ans = ctx.obj.get_job_by_id(id) + poll_result[f"Try {i+1}"] = ans + status = ans["status"].lower() + if i == 0: + console.print(_render_job_attributes(ans)) - table = Table( - show_header=True, - title="Analysis Data", - box=box.DOUBLE_EDGE, - show_lines=True, - ) - # add headers - for h in headers: - table.add_column(h, header_style="bold blue") - # add rows - for el in data["analysis_reports"]: - table.add_row( - el["name"], - get_success_text((el["success"])), - get_json_syntax(el["report"]) if el["report"] else None, - get_json_syntax(el["errors"]) if el["errors"] else None, + if status not in ["running", "pending"]: + console.print( + "\nPolling stopped because job has finished with status: ", + get_status_text(status), + end="", + ) + break + console.clear() + console.print( + _render_job_analysis_table(ans["analysis_reports"], verbose=False), + justify="center", ) - console.print(table) + with open(output, "w") as outfile: + json.dump(poll_result, outfile, indent=4) + + except Exception as e: + ctx.obj.logger.error(f"Error in retrieving job: {str(e)}") def _display_all_jobs(data): From 5bc246596886bd2fabe332c8b7f47a19b376ec3a Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Thu, 3 Dec 2020 00:19:01 +0530 Subject: [PATCH 27/38] add --json option and change command/groups --- pyintelowl/cli/__init__.py | 3 +- pyintelowl/cli/commands.py | 23 +++-- pyintelowl/cli/jobs.py | 181 ++++++++++++++++--------------------- pyintelowl/cli/tags.py | 49 +++++++--- 4 files changed, 129 insertions(+), 127 deletions(-) diff --git a/pyintelowl/cli/__init__.py b/pyintelowl/cli/__init__.py index 4c9c6dc..38474fc 100644 --- a/pyintelowl/cli/__init__.py +++ b/pyintelowl/cli/__init__.py @@ -8,7 +8,8 @@ analyse, config, jobs, + tags, ] -cmds = [get_analyzer_config, tags] +cmds = [get_analyzer_config] diff --git a/pyintelowl/cli/commands.py b/pyintelowl/cli/commands.py index a58a983..337a0f0 100644 --- a/pyintelowl/cli/commands.py +++ b/pyintelowl/cli/commands.py @@ -1,10 +1,17 @@ import click +import json import re from rich.console import Console from rich.table import Table from rich import box -from ..cli._utils import ClickContext, get_success_text, get_json_syntax +from ..cli._utils import ( + ClickContext, + get_success_text, + get_json_syntax, + add_options, + json_flag_option, +) from pyintelowl.pyintelowl import IntelOwlClientException @@ -16,10 +23,12 @@ "--re-match", help="RegEx Pattern to filter analyzer names against", ) -@click.option("-j", "--json", is_flag=True, help="Pretty print as JSON") -@click.option("-t", "--text", is_flag=True, help="Print analyzer names as CSV") +@add_options(json_flag_option) +@click.option( + "-t", "--text", "as_text", is_flag=True, help="Print analyzer names as CSV" +) @click.pass_context -def get_analyzer_config(ctx: ClickContext, re_match: str, json: bool, text: bool): +def get_analyzer_config(ctx: ClickContext, re_match: str, as_json: bool, as_text: bool): console = Console() ctx.obj.logger.info("Requesting [italic blue]analyzer_config.json[/]..") try: @@ -31,10 +40,10 @@ def get_analyzer_config(ctx: ClickContext, re_match: str, json: bool, text: bool except IntelOwlClientException as e: ctx.obj.logger.fatal(str(e)) ctx.exit(0) - if json: + if as_json: with console.pager(styles=True): - console.print(res) - elif text: + console.print(json.dumps(res, indent=4)) + elif as_text: click.echo(", ".join(res.keys())) else: # otherwise, print full table diff --git a/pyintelowl/cli/jobs.py b/pyintelowl/cli/jobs.py index f66efe5..1c1d368 100644 --- a/pyintelowl/cli/jobs.py +++ b/pyintelowl/cli/jobs.py @@ -1,23 +1,23 @@ -import time import click import json -import click_spinner +from rich import print as rprint from rich.console import Console -from rich.table import Table -from rich import box, print as rprint -from rich.progress import track from pyintelowl.exceptions import IntelOwlClientException -from ..cli._utils import ClickContext, get_status_text +from ..cli._utils import ClickContext, add_options, json_flag_option from ..cli._jobs_utils import ( + _display_all_jobs, _display_single_job, - _render_job_attributes, - _render_job_analysis_table, + _result_filter_and_tabular_print, + _poll_for_job_cli, ) -@click.group("jobs", help="Manage Jobs", invoke_without_command=True) -@click.option("-a", "--all", is_flag=True, help="List all jobs") -@click.option("--id", type=int, default=0, help="Retrieve Job details by ID") +@click.group("jobs", help="Manage Jobs") +def jobs(): + pass + + +@jobs.command(help="List all jobs") @click.option( "--status", type=click.Choice( @@ -31,47 +31,72 @@ case_sensitive=False, ), show_choices=True, - help="Filter jobs with a particular status. Should be used with --all.", + help="Only show jobs having a particular status", ) +@add_options(json_flag_option) @click.pass_context -def jobs(ctx: ClickContext, id: int, all: bool, status: str): - """ - Manage Jobs - """ +def ls(ctx: ClickContext, status: str, as_json: bool): + # ctx.obj.logger.info("Requesting list of jobs..") try: - if all: - ctx.obj.logger.info("Requesting list of jobs..") - with click_spinner.spinner(): - ans = ctx.obj.get_all_jobs() - if status: - ans = [el for el in ans if el["status"].lower() == status.lower()] - _display_all_jobs(ans) - elif id: - ctx.obj.logger.info(f"Requesting Job [underline blue]#{id}[/]..") - with click_spinner.spinner(): - ans = ctx.obj.get_job_by_id(id) - _display_single_job(ans) + ans = ctx.obj.get_all_jobs() + if status: + ans = [el for el in ans if el["status"].lower() == status.lower()] + if as_json: + rprint(json.dumps(ans, indent=4)) else: - if not ctx.invoked_subcommand: - click.echo(ctx.get_usage()) + _display_all_jobs(ctx.obj.logger, ans) except IntelOwlClientException as e: ctx.obj.logger.fatal(str(e)) -@jobs.command("poll", help="HTTP poll a currently running job's details") -@click.option("--id", type=int, required=True, help="HTTP poll a job for live updates") +@jobs.command(help="Tabular print job attributes and results for a job ID") +@click.argument("id", type=int) @click.option( - "-o", - "--output", - type=click.Path(exists=False, resolve_path=True), - required=True, - help="File to save results", + "-c", + "--categorize", + is_flag=True, + help=""" + Categorize results according to type and analyzer. + Only works for observable analysis. + """, +) +@add_options(json_flag_option) +@click.pass_context +def view(ctx: ClickContext, id: int, categorize: bool, as_json: bool): + # ctx.obj.logger.info(f"Requesting Job [underline blue]#{id}[/]..") + if as_json and categorize: + raise click.Abort("Cannot use the -c and -j options together") + try: + ans = ctx.obj.get_job_by_id(id) + if as_json: + rprint(json.dumps(ans, indent=4)) + elif categorize: + if ans["is_sample"]: + raise click.Abort("Cannot use the -c option for a file analysis") + _result_filter_and_tabular_print( + ans["analysis_reports"], + ans["observable_name"], + ans["observable_classification"], + ) + else: + _display_single_job(ans) + except IntelOwlClientException as e: + ctx.obj.logger.fatal(str(e)) + + +@jobs.command( + "poll", + help=""" + HTTP poll a running job's details until + it finishes and save result into a file. + """, ) +@click.argument("id", type=int) @click.option( "-t", "--max-tries", type=int, - default=0, + default=5, show_default=True, help="maximum number of tries", ) @@ -83,72 +108,20 @@ def jobs(ctx: ClickContext, id: int, all: bool, status: str): show_default=True, help="sleep interval between subsequent requests (in sec)", ) +@click.option( + "-o", + "--output-file", + type=click.Path(exists=False, resolve_path=True), + required=True, + help="Path to JSON file to save result to", +) @click.pass_context -def poll(ctx: ClickContext, id: int, max_tries: int, interval: int, output: str): - console = Console() - poll_result = {} +def poll(ctx: ClickContext, id: int, max_tries: int, interval: int, output_file: str): try: - for i in track( - range(max_tries), - description=f"Polling Job [underline blue]#{id}[/]..", - console=console, - ): - if i != 0: - console.print(f"sleeping for {interval} seconds before next request..") - time.sleep(interval) - ans = ctx.obj.get_job_by_id(id) - poll_result[f"Try {i+1}"] = ans - status = ans["status"].lower() - if i == 0: - console.print(_render_job_attributes(ans)) - - if status not in ["running", "pending"]: - console.print( - "\nPolling stopped because job has finished with status: ", - get_status_text(status), - end="", - ) - break - console.clear() - console.print( - _render_job_analysis_table(ans["analysis_reports"], verbose=False), - justify="center", - ) - with open(output, "w") as outfile: - json.dump(poll_result, outfile, indent=4) - - except Exception as e: - ctx.obj.logger.error(f"Error in retrieving job: {str(e)}") - - -def _display_all_jobs(data): - console = Console() - table = Table(show_header=True, title="List of Jobs", box=box.DOUBLE_EDGE) - header_style = "bold blue" - table.add_column(header="Id", header_style=header_style) - table.add_column(header="Name", header_style=header_style) - table.add_column(header="Type", header_style=header_style) - table.add_column(header="Tags", header_style=header_style) - table.add_column( - header="Analyzers\nCalled", justify="center", header_style=header_style - ) - table.add_column( - header="Process\nTime(s)", justify="center", header_style=header_style - ) - table.add_column(header="Status", header_style=header_style) - try: - for el in data: - table.add_row( - str(el["id"]), - el["observable_name"] if el["observable_name"] else el["file_name"], - el["observable_classification"] - if el["observable_classification"] - else el["file_mimetype"], - ", ".join([t["label"] for t in el["tags"]]), - el["no_of_analyzers_executed"], - str(el["process_time"]), - get_status_text(el["status"]), - ) - console.print(table, justify="center") + ans = _poll_for_job_cli(id, max_tries, interval) + if ans: + with open(output_file, "w") as fp: + json.dump(ans, fp, indent=4) + Console().print(f"Result saved into [u red]{output_file}[/]") except Exception as e: - rprint(e) + ctx.obj.logger.fatal(f"Error in retrieving job: {str(e)}") diff --git a/pyintelowl/cli/tags.py b/pyintelowl/cli/tags.py index 338fc93..40f1f99 100644 --- a/pyintelowl/cli/tags.py +++ b/pyintelowl/cli/tags.py @@ -1,34 +1,53 @@ import click +import json from rich.console import Console from rich.table import Table from rich.text import Text from rich import box +from rich import print as rprint from pyintelowl.exceptions import IntelOwlClientException -from ..cli._utils import ClickContext +from ..cli._utils import ClickContext, add_options, json_flag_option -@click.command(help="Manage tags") -@click.option("-a", "--all", is_flag=True, help="List all tags") -@click.option("--id", type=int, default=0, help="Retrieve tag details by ID") +@click.group(help="Manage tags") +def tags(): + pass + + +@tags.command(help="List all tags") +@add_options(json_flag_option) @click.pass_context -def tags(ctx: ClickContext, id: int, all: bool): - """ - Manage tags - """ +def ls(ctx: ClickContext, as_json: bool): try: - if all: - ans = ctx.obj.get_all_tags() - elif id: - ans = ctx.obj.get_tag_by_id(id) + ans = ctx.obj.get_all_tags() + if as_json: + rprint(json.dumps(ans, indent=4)) + else: + _print_tags_table(ctx.obj.logger, ans) + except IntelOwlClientException as e: + ctx.obj.logger.fatal(str(e)) + + +@tags.command(help="Retrieve tag details by ID") +@click.argument("id", type=int) +@add_options(json_flag_option) +@click.pass_context +def view(ctx: ClickContext, id: int, as_json: bool): + try: + ans = ctx.obj.get_tag_by_id(id) + if as_json: + rprint(json.dumps(ans, indent=4)) + else: ans = [ans] - _print_tags_table(ctx, ans) + _print_tags_table(ctx.obj.logger, ans) except IntelOwlClientException as e: + print(e.response.status_code) ctx.obj.logger.fatal(str(e)) -def _print_tags_table(ctx, rows): +def _print_tags_table(logger, rows): console = Console() table = Table(show_header=True, title="List of tags", box=box.DOUBLE_EDGE) for h in ["Id", "Label", "Color"]: @@ -41,4 +60,4 @@ def _print_tags_table(ctx, rows): ) console.print(table, justify="center") except Exception as e: - ctx.obj.logger.fatal(str(e)) + logger.fatal(str(e)) From 61ef42709fe3c6b8af77369bc116a5ce88c3b437 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Thu, 3 Dec 2020 00:19:31 +0530 Subject: [PATCH 28/38] make it work with rich console and pager --- pyintelowl/cli/_utils.py | 10 ++++++++++ pyintelowl/cli/domain_checkers.py | 15 ++++++++++++++- pyintelowl/cli/jobs.py | 16 +++++++++------- pyintelowl/cli/tags.py | 6 +++--- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/pyintelowl/cli/_utils.py b/pyintelowl/cli/_utils.py index 43770d3..0a8a380 100644 --- a/pyintelowl/cli/_utils.py +++ b/pyintelowl/cli/_utils.py @@ -10,6 +10,16 @@ from pyintelowl.pyintelowl import IntelOwl +json_flag_option = [ + click.option( + "-j", + "--json", + "as_json", + is_flag=True, + help="output as raw JSON", + ), +] + class ClickContext(click.Context): obj: IntelOwl diff --git a/pyintelowl/cli/domain_checkers.py b/pyintelowl/cli/domain_checkers.py index d20f92c..8a661c8 100644 --- a/pyintelowl/cli/domain_checkers.py +++ b/pyintelowl/cli/domain_checkers.py @@ -1,6 +1,10 @@ import geocoder -from datetime import datetime import os +from datetime import datetime +from rich.console import Console + +console = Console() +print = console.print class MyColors: @@ -41,6 +45,15 @@ def __init__(self, results, value): self.results = results self.value = value + @property + def func_map(self): + return { + "domain": self.check_domain, + "hash": self.check_hash, + "url": self.check_url, + "ip": self.check_ip, + } + def check_url(self): for result in self.results: if "name" in result: diff --git a/pyintelowl/cli/jobs.py b/pyintelowl/cli/jobs.py index 1c1d368..1b3486a 100644 --- a/pyintelowl/cli/jobs.py +++ b/pyintelowl/cli/jobs.py @@ -50,7 +50,7 @@ def ls(ctx: ClickContext, status: str, as_json: bool): @jobs.command(help="Tabular print job attributes and results for a job ID") -@click.argument("id", type=int) +@click.argument("job_id", type=int) @click.option( "-c", "--categorize", @@ -62,12 +62,12 @@ def ls(ctx: ClickContext, status: str, as_json: bool): ) @add_options(json_flag_option) @click.pass_context -def view(ctx: ClickContext, id: int, categorize: bool, as_json: bool): - # ctx.obj.logger.info(f"Requesting Job [underline blue]#{id}[/]..") +def view(ctx: ClickContext, job_id: int, categorize: bool, as_json: bool): + # ctx.obj.logger.info(f"Requesting Job [underline blue]#{job_id}[/]..") if as_json and categorize: raise click.Abort("Cannot use the -c and -j options together") try: - ans = ctx.obj.get_job_by_id(id) + ans = ctx.obj.get_job_by_id(job_id) if as_json: rprint(json.dumps(ans, indent=4)) elif categorize: @@ -91,7 +91,7 @@ def view(ctx: ClickContext, id: int, categorize: bool, as_json: bool): it finishes and save result into a file. """, ) -@click.argument("id", type=int) +@click.argument("job_id", type=int) @click.option( "-t", "--max-tries", @@ -116,9 +116,11 @@ def view(ctx: ClickContext, id: int, categorize: bool, as_json: bool): help="Path to JSON file to save result to", ) @click.pass_context -def poll(ctx: ClickContext, id: int, max_tries: int, interval: int, output_file: str): +def poll( + ctx: ClickContext, job_id: int, max_tries: int, interval: int, output_file: str +): try: - ans = _poll_for_job_cli(id, max_tries, interval) + ans = _poll_for_job_cli(job_id, max_tries, interval) if ans: with open(output_file, "w") as fp: json.dump(ans, fp, indent=4) diff --git a/pyintelowl/cli/tags.py b/pyintelowl/cli/tags.py index 40f1f99..ecb82f1 100644 --- a/pyintelowl/cli/tags.py +++ b/pyintelowl/cli/tags.py @@ -31,12 +31,12 @@ def ls(ctx: ClickContext, as_json: bool): @tags.command(help="Retrieve tag details by ID") -@click.argument("id", type=int) +@click.argument("tag_id", type=int) @add_options(json_flag_option) @click.pass_context -def view(ctx: ClickContext, id: int, as_json: bool): +def view(ctx: ClickContext, tag_id: int, as_json: bool): try: - ans = ctx.obj.get_tag_by_id(id) + ans = ctx.obj.get_tag_by_id(tag_id) if as_json: rprint(json.dumps(ans, indent=4)) else: From 3b5c02f1315c5bfe8720b5ced70df98d2b7679eb Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Thu, 3 Dec 2020 00:20:47 +0530 Subject: [PATCH 29/38] refactor job polling --- pyintelowl/cli/_jobs_utils.py | 82 +++++++++++++++++++++++++++++++++++ pyintelowl/cli/jobs.py | 2 +- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/pyintelowl/cli/_jobs_utils.py b/pyintelowl/cli/_jobs_utils.py index b74e5b7..a2c26c7 100644 --- a/pyintelowl/cli/_jobs_utils.py +++ b/pyintelowl/cli/_jobs_utils.py @@ -1,6 +1,8 @@ +import time from rich import box from rich.panel import Panel from rich.table import Table +from rich.progress import track from rich.console import RenderGroup, Console from ..cli._utils import ( get_status_text, @@ -8,6 +10,8 @@ get_tags_str, get_json_syntax, ) +from pyintelowl.exceptions import IntelOwlClientException +from ..cli.domain_checkers import Checkers, console as checkers_console def _display_single_job(data): @@ -68,3 +72,81 @@ def _render_job_attributes(data): f"{style}Status:[/] {status}", ) return Panel(r, title="Job attributes") + + +def _display_all_jobs(logger, rows): + console = Console() + table = Table(show_header=True, title="List of Jobs", box=box.DOUBLE_EDGE) + header_style = "bold blue" + table.add_column(header="Id", header_style=header_style) + table.add_column(header="Name", header_style=header_style) + table.add_column(header="Type", header_style=header_style) + table.add_column(header="Tags", header_style=header_style) + table.add_column( + header="Analyzers\nCalled", justify="center", header_style=header_style + ) + table.add_column( + header="Process\nTime(s)", justify="center", header_style=header_style + ) + table.add_column(header="Status", header_style=header_style) + try: + for el in rows: + table.add_row( + str(el["id"]), + el["observable_name"] if el["observable_name"] else el["file_name"], + el["observable_classification"] + if el["observable_classification"] + else el["file_mimetype"], + ", ".join([t["label"] for t in el["tags"]]), + el["no_of_analyzers_executed"], + str(el["process_time"]), + get_status_text(el["status"]), + ) + console.print(table, justify="center") + except Exception as e: + logger.fatal(e) + + +def _result_filter_and_tabular_print(result, observable: str, obs_clsfn: str): + checkers = Checkers(result, observable) + func = checkers.func_map.get(obs_clsfn, None) + if not func: + raise IntelOwlClientException(f"Not supported observable type: {obs_clsfn}") + with checkers_console.pager(): + func() + + +def _poll_for_job_cli( + obj, + job_id: int, + max_tries=5, + interval=5, +): + console = Console() + ans = None + for i in track( + range(max_tries), + description=f"Polling Job [underline blue]#{job_id}[/]..", + console=console, + transient=True, + update_period=interval, + ): + if i != 0: + # console.print(f"sleeping for {interval} seconds before next request..") + time.sleep(interval) + ans = obj.get_job_by_id(job_id) + status = ans["status"].lower() + if i == 0: + console.print(_render_job_attributes(ans)) + console.print( + _render_job_analysis_table(ans["analysis_reports"], verbose=False), + justify="center", + ) + if status not in ["running", "pending"]: + console.print( + "\nPolling stopped because job has finished with status: ", + get_status_text(status), + end="", + ) + break + return ans diff --git a/pyintelowl/cli/jobs.py b/pyintelowl/cli/jobs.py index 1b3486a..7c50d3c 100644 --- a/pyintelowl/cli/jobs.py +++ b/pyintelowl/cli/jobs.py @@ -120,7 +120,7 @@ def poll( ctx: ClickContext, job_id: int, max_tries: int, interval: int, output_file: str ): try: - ans = _poll_for_job_cli(job_id, max_tries, interval) + ans = _poll_for_job_cli(ctx.obj, job_id, max_tries, interval) if ans: with open(output_file, "w") as fp: json.dump(ans, fp, indent=4) From 80a82c1d69a25a3fc018f24a8a9b48eb7aff2f1b Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Thu, 3 Dec 2020 01:00:10 +0530 Subject: [PATCH 30/38] add analyze --poll + more type hintings --- pyintelowl/cli/analyse.py | 11 +++++- pyintelowl/pyintelowl.py | 81 ++++++++++++++++++++++++--------------- 2 files changed, 60 insertions(+), 32 deletions(-) diff --git a/pyintelowl/cli/analyse.py b/pyintelowl/cli/analyse.py index fe9088e..8c0e744 100644 --- a/pyintelowl/cli/analyse.py +++ b/pyintelowl/cli/analyse.py @@ -3,7 +3,6 @@ from pyintelowl.exceptions import IntelOwlClientException from ..cli._utils import add_options, ClickContext, get_json_data - __analyse_options = [ click.option( "-al", @@ -62,6 +61,12 @@ help="Path to JSON file which contains runtime_configuration.", type=click.Path(exists=True, resolve_path=True), ), + click.option( + "--poll", + "should_poll", + is_flag=True, + help="HTTP poll for the job result and notify when it's finished", + ), ] @@ -87,6 +92,7 @@ def observable( disable_external_analyzers, check, runtime_config, + should_poll: bool, ): if not run_all: analyzers_list = analyzers_list.split(",") @@ -105,6 +111,7 @@ def observable( disable_external_analyzers, check, runtime_config, + should_poll, ) except IntelOwlClientException as e: ctx.obj.logger.fatal(str(e)) @@ -124,6 +131,7 @@ def file( disable_external_analyzers, check, runtime_config, + should_poll: bool, ): if not run_all: analyzers_list = analyzers_list.split(",") @@ -142,6 +150,7 @@ def file( disable_external_analyzers, check, runtime_config, + should_poll, ) except IntelOwlClientException as e: ctx.obj.logger.fatal(str(e)) diff --git a/pyintelowl/pyintelowl.py b/pyintelowl/pyintelowl.py index bec00b4..4f7a5d5 100644 --- a/pyintelowl/pyintelowl.py +++ b/pyintelowl/pyintelowl.py @@ -30,7 +30,7 @@ def __init__( self.logger = logging.getLogger(__name__) @property - def session(self): + def session(self) -> requests.Session: if not hasattr(self, "_session"): session = requests.Session() if self.certificate: @@ -51,7 +51,7 @@ def ask_analysis_availability( analyzers_needed: List[str], run_all_available_analyzers: bool = False, check_reported_analysis_too: bool = False, - ): + ) -> Dict: answer = None try: params = {"md5": md5, "analyzers_needed": analyzers_needed} @@ -90,7 +90,7 @@ def send_file_analysis_request( disable_external_analyzers: bool = False, run_all_available_analyzers: bool = False, runtime_configuration: Dict = {}, - ): + ) -> Dict: answer = None try: data = { @@ -120,7 +120,7 @@ def send_observable_analysis_request( disable_external_analyzers: bool = False, run_all_available_analyzers: bool = False, runtime_configuration: Dict = {}, - ): + ) -> Dict: answer = None try: data = { @@ -143,12 +143,16 @@ def send_observable_analysis_request( raise IntelOwlClientException(e) return answer - def send_analysis_batch(self, rows: List = []): + def send_analysis_batch(self, rows: List): + """ + Send multiple analysis requests. + """ for obj in rows: try: - if obj.get("runtime_config", None): - with open(obj["runtime_config"]) as fp: - obj["runtime_config"] = json.load(fp) + runtime_config = obj.get("runtime_config", {}) + if runtime_config: + with open(runtime_config) as fp: + runtime_config = json.load(fp) if not (obj.get("run_all", False)): obj["analyzers_list"] = obj["analyzers_list"].split(",") @@ -162,7 +166,7 @@ def send_analysis_batch(self, rows: List = []): obj.get("private_job", False), obj.get("disable_external_analyzers", False), obj.get("check", None), - obj.get("runtime_config", {}), + runtime_config, ) except IntelOwlClientException as e: self.logger.fatal(str(e)) @@ -246,7 +250,7 @@ def get_all_jobs(self): raise IntelOwlClientException(e) return answer - def get_tag_by_id(self, tag_id): + def get_tag_by_id(self, tag_id) -> Dict: answer = None try: url = self.instance + "/api/tags/" @@ -258,7 +262,7 @@ def get_tag_by_id(self, tag_id): raise IntelOwlClientException(e) return answer - def get_job_by_id(self, job_id): + def get_job_by_id(self, job_id) -> Dict: answer = None try: url = self.instance + "/api/jobs/" + str(job_id) @@ -271,8 +275,8 @@ def get_job_by_id(self, job_id): return answer @staticmethod - def get_md5(to_hash: Any, type_="observable"): - md5 = None + def get_md5(to_hash: Any, type_="observable") -> str: + md5 = "" if type_ == "observable": md5 = hashlib.md5(str(to_hash).lower().encode("utf-8")).hexdigest() elif type_ == "binary": @@ -296,6 +300,7 @@ def _new_analysis_cli( disable_external_analyzers, check, runtime_configuration: Dict = {}, + should_poll: bool = False, ): # CLI sanity checks if analyzers_list and run_all: @@ -320,24 +325,25 @@ def _new_analysis_cli( """ ) # 1st step: ask analysis availability - md5 = self.get_md5(obj, type_=type_) - resp = self.ask_analysis_availability( - md5, analyzers_list, run_all, True if check == "reported" else False - ) - status, job_id = resp.get("status", None), resp.get("job_id", None) - if status != "not_available": - self.logger.info( - f"""Found existing analysis! - Job: #{job_id} - status: [u blue]{status}[/] - - [i]Hint: use [#854442]--check force-new[/] to perform new scan anyway[/] - """ + if check != "force-new": + md5 = self.get_md5(obj, type_=type_) + resp = self.ask_analysis_availability( + md5, analyzers_list, run_all, True if check == "reported" else False ) - return + status, job_id = resp.get("status", None), resp.get("job_id", None) + if status != "not_available": + self.logger.info( + f"""Found existing analysis! + Job: #{job_id} + status: [u blue]{status}[/] + + [i]Hint: use [#854442]--check force-new[/] to perform new scan anyway[/] + """ + ) + return # 2nd step: send new analysis request if type_ == "observable": - _ = self.send_observable_analysis_request( + resp2 = self.send_observable_analysis_request( analyzers_requested=analyzers_list, observable_name=obj, force_privacy=force_privacy, @@ -348,7 +354,7 @@ def _new_analysis_cli( ) else: path = pathlib.Path(obj) - _ = self.send_file_analysis_request( + resp2 = self.send_file_analysis_request( analyzers_requested=analyzers_list, filename=path.name, binary=path.read_bytes(), @@ -359,10 +365,23 @@ def _new_analysis_cli( runtime_configuration=runtime_configuration, ) # 3rd step: poll for result - # todo + if should_poll: + if resp2["status"] != "accepted": + self.logger.fatal("Can't poll a failed job") + # import poll function + from .cli._jobs_utils import _poll_for_job_cli + + job_id = resp2["job_id"] + _ = _poll_for_job_cli(self, job_id) + self.logger.info( + f""" + Polling finished. + Execute [i blue]pyintelowl jobs view {job_id}[/] to view the result + """ + ) @staticmethod - def _get_observable_classification(value): + def _get_observable_classification(value) -> str: # only following types are supported: # ip - domain - url - hash (md5, sha1, sha256) try: From 0852ac051ffa3b915736e64c83b72e6b3361b4c5 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Thu, 3 Dec 2020 01:05:44 +0530 Subject: [PATCH 31/38] add log stmts --- pyintelowl/cli/jobs.py | 4 ++-- pyintelowl/cli/tags.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyintelowl/cli/jobs.py b/pyintelowl/cli/jobs.py index 7c50d3c..f9a320a 100644 --- a/pyintelowl/cli/jobs.py +++ b/pyintelowl/cli/jobs.py @@ -36,7 +36,7 @@ def jobs(): @add_options(json_flag_option) @click.pass_context def ls(ctx: ClickContext, status: str, as_json: bool): - # ctx.obj.logger.info("Requesting list of jobs..") + ctx.obj.logger.info("Requesting list of jobs..") try: ans = ctx.obj.get_all_jobs() if status: @@ -63,7 +63,7 @@ def ls(ctx: ClickContext, status: str, as_json: bool): @add_options(json_flag_option) @click.pass_context def view(ctx: ClickContext, job_id: int, categorize: bool, as_json: bool): - # ctx.obj.logger.info(f"Requesting Job [underline blue]#{job_id}[/]..") + ctx.obj.logger.info(f"Requesting Job [underline blue]#{job_id}[/]..") if as_json and categorize: raise click.Abort("Cannot use the -c and -j options together") try: diff --git a/pyintelowl/cli/tags.py b/pyintelowl/cli/tags.py index ecb82f1..5e85985 100644 --- a/pyintelowl/cli/tags.py +++ b/pyintelowl/cli/tags.py @@ -20,6 +20,7 @@ def tags(): @add_options(json_flag_option) @click.pass_context def ls(ctx: ClickContext, as_json: bool): + ctx.obj.logger.info("Requesting list of tags..") try: ans = ctx.obj.get_all_tags() if as_json: @@ -35,6 +36,7 @@ def ls(ctx: ClickContext, as_json: bool): @add_options(json_flag_option) @click.pass_context def view(ctx: ClickContext, tag_id: int, as_json: bool): + ctx.obj.logger.info(f"Requesting Tag [underline blue]#{tag_id}[/]..") try: ans = ctx.obj.get_tag_by_id(tag_id) if as_json: From 63f58b1285558e26995de74d71e5ceaaeee3b016 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Thu, 3 Dec 2020 01:07:18 +0530 Subject: [PATCH 32/38] remove click-spinner dependency --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 90ae11a..40e0074 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,6 @@ "geocoder==1.38.1", "click==7.1.2", "rich==9.2.0", - "click-spinner==0.1.10", "click_completion", "tinynetrc==1.3.0", ] From eb9c3d3216f55aff20fb72091c78fa98a56aa6f5 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Thu, 3 Dec 2020 01:16:28 +0530 Subject: [PATCH 33/38] remove old client --- intel_owl_client.py | 354 -------------------------------------------- 1 file changed, 354 deletions(-) delete mode 100644 intel_owl_client.py diff --git a/intel_owl_client.py b/intel_owl_client.py deleted file mode 100644 index 4d30be2..0000000 --- a/intel_owl_client.py +++ /dev/null @@ -1,354 +0,0 @@ -import argparse -import hashlib -import logging -import os -import requests -import time -from domain_checkers import Checkers -from pprint import pprint - -from pyintelowl.pyintelowl import ( - IntelOwl, - IntelOwlClientException, - get_observable_classification, -) - - -def intel_owl_client(): - - parser = argparse.ArgumentParser(description="Intel Owl classic client") - parser.add_argument( - "-sc", - "--show-colors", - action="store_true", - default=False, - help="Show colorful and more user-friendly results." - " By default JSON raw results are shown", - ) - parser.add_argument( - "-k", - "--api_key", - help="API key to authenticate against a IntelOwl instance", - ) - parser.add_argument( - "-c", "--certificate", default=False, help="path to Intel Owl certificate" - ) - parser.add_argument("-i", "--instance", required=True, help="your instance URL") - parser.add_argument( - "-d", "--debug", action="store_true", default=False, help="debug mode" - ) - parser.add_argument("-l", "--log-to-file", help="log to specified file") - parser.add_argument( - "-gc", - "--get-configuration", - action="store_true", - default=False, - help="get analyzers configuration only", - ) - parser.add_argument( - "-a", "--analyzers-list", action="append", help="list of analyzers to launch" - ) - parser.add_argument( - "-aa", - "--run-all-available-analyzers", - action="store_true", - default=False, - help="run all available and compatible analyzers", - ) - parser.add_argument( - "-fp", - "--force-privacy", - action="store_true", - default=False, - help="disable analyzers that could impact privacy", - ) - parser.add_argument( - "-p", - "--private-job", - action="store_true", - default=False, - help="Limit view permissions to my group", - ) - parser.add_argument( - "-e", - "--disable-external-analyzers", - action="store_true", - default=False, - help="disable analyzers that use external services", - ) - parser.add_argument( - "-r", - "--check-reported-analysis-too", - action="store_true", - default=False, - help="check reported analysis too, not only 'running' ones", - ) - parser.add_argument( - "-s", - "--skip-check-analysis-availability", - action="store_true", - default=False, - help="skip check analysis availability", - ) - - subparsers = parser.add_subparsers(help="choose type of analysis", dest="command") - parser_sample = subparsers.add_parser("file", help="File analysis") - parser_observable = subparsers.add_parser("observable", help="Observables analysis") - - parser_sample.add_argument("-f", "--file", required=True, help="file to analyze") - parser_observable.add_argument( - "-v", "--value", required=True, help="observable to analyze" - ) - - args = parser.parse_args() - - if not args.get_configuration: - if not args.analyzers_list and not args.run_all_available_analyzers: - print( - "you must specify at least an analyzer (-a) or" - " every available analyzer (--aa)" - ) - exit(2) - if args.analyzers_list and args.run_all_available_analyzers: - print("you must specify either --aa or -a, not both") - exit(3) - - logger = get_logger(args.debug, args.log_to_file) - - _pyintelowl_logic(args, logger) - - -def _pyintelowl_logic(args, logger): - md5 = None - results = [] - elapsed_time = None - get_configuration_only = False - try: - filename = None - binary = None - if args.command == "file": - if not os.path.exists(args.file): - raise IntelOwlClientException(f"{args.file} does not exists") - with open(args.file, "rb") as f: - binary = f.read() - filename = os.path.basename(f.name) - md5 = hashlib.md5(binary).hexdigest() - elif args.command == "observable": - args.value = args.value.lower() - md5 = hashlib.md5(args.value.encode("utf-8")).hexdigest() - elif args.get_configuration: - get_configuration_only = True - else: - raise IntelOwlClientException( - "you must specify the type of the analysis: [observable, file]" - ) - - pyintelowl_client = IntelOwl( - args.api_key, - args.certificate, - args.instance, - args.debug, - ) - - if get_configuration_only: - api_request_result = pyintelowl_client.get_analyzer_configs() - errors = api_request_result.get("errors", []) - if errors: - logger.error(f"API get_analyzer_configs failed. Errors: {errors}") - analyzers_config = api_request_result.get("answer", {}) - logger.info(f"extracted analyzer_configuration: {analyzers_config}") - pprint(analyzers_config) - exit(0) - - analysis_available = False - if not args.skip_check_analysis_availability: - job_id_to_get = None - # first step: ask analysis availability - logger.info( - f"[STARTED] request ask_analysis_availability for md5: {md5}," - f" analyzers: {args.analyzers_list}" - ) - - api_request_result = pyintelowl_client.ask_analysis_availability( - md5, - args.analyzers_list, - args.run_all_available_analyzers, - args.check_reported_analysis_too, - ) - errors = api_request_result.get("errors", []) - if errors: - raise IntelOwlClientException( - f"API ask_analysis_availability failed. Errors: {errors}" - ) - answer = api_request_result.get("answer", {}) - status = answer.get("status", None) - if not status: - raise IntelOwlClientException( - "API ask_analysis_availability gave result without status!?!?" - f" Answer: {answer}" - ) - if status != "not_available": - analysis_available = True - job_id_to_get = answer.get("job_id", "") - if job_id_to_get: - logger.info( - f"[INFO] already existing Job(#{job_id_to_get}, md5: {md5}," - f" status: {status}) with analyzers: {args.analyzers_list}" - ) - else: - raise IntelOwlClientException( - "API ask_analysis_availability gave result without job_id!?!?" - f" Answer:{answer}" - ) - - # second step: in case there are no analysis available, start a new analysis - if not analysis_available: - - if args.command == "file": - api_request_result = pyintelowl_client.send_file_analysis_request( - md5, - args.analyzers_list, - filename, - binary, - args.force_privacy, - args.private_job, - args.disable_external_analyzers, - args.run_all_available_analyzers, - ) - elif args.command == "observable": - api_request_result = pyintelowl_client.send_observable_analysis_request( - md5, - args.analyzers_list, - args.value, - args.force_privacy, - args.private_job, - args.disable_external_analyzers, - args.run_all_available_analyzers, - ) - else: - raise NotImplementedError() - - # both cases share the same logic for the management of - # the result retrieved from the API - errors = api_request_result.get("errors", []) - if errors: - raise IntelOwlClientException( - f"API send_analysis_request failed. Errors: {errors}" - ) - answer = api_request_result.get("answer", {}) - logger.info(f"[INFO] md5: {md5} received response from intel_owl: {answer}") - status = answer.get("status", None) - if not status: - raise IntelOwlClientException( - "API send_analysis_request gave result without status!?" - f" Answer:{answer}" - ) - if status != "accepted": - raise IntelOwlClientException( - f"API send_analysis_request gave unexpected result status: {status}" - ) - job_id_to_get = answer.get("job_id", None) - analyzers_running = answer.get("analyzers_running", None) - warnings = answer.get("warnings", []) - if job_id_to_get: - logger.info( - f"[STARTED] Job(#{job_id_to_get}, md5: {md5}, status: {status})" - f" -> analyzers: {analyzers_running}. Warnings: {warnings}" - ) - else: - raise IntelOwlClientException( - f"API send_analysis_request gave result without job_id!?!?" - f"answer:{answer}" - ) - - # third step: at this moment we must have a job_id to check for results - polling_max_tries = 60 * 20 - polling_interval = 1 - logger.info("[STARTED] polling...") - for chance in range(polling_max_tries): - time.sleep(polling_interval) - api_request_result = pyintelowl_client.ask_analysis_result(job_id_to_get) - errors = api_request_result.get("errors", []) - if errors: - raise IntelOwlClientException( - f"API ask_analysis_result failed. Errors: {errors}" - ) - answer = api_request_result.get("answer", {}) - status = answer.get("status", None) - if not status: - raise IntelOwlClientException( - f"API ask_analysis_result gave result without status!?!?" - f" job_id:{job_id_to_get} answer:{answer}" - ) - if status in ["invalid_id", "not_available"]: - raise IntelOwlClientException( - f"API send_analysis_request gave status {status}" - ) - if status == "running": - continue - if status == "pending": - logger.warning( - f"API ask_analysis_result check job in status 'pending'." - f" Maybe it is stuck" - f"job_id:{job_id_to_get} md5:{md5}" - f" analyzer_list:{args.analyzers_list}" - ) - elif status in ["reported_without_fails", "reported_with_fails", "failed"]: - logger.info( - f"[FINISHED] Job(#{job_id_to_get}, md5: {md5}, status: {status})" - f" -> analyzer_list:{args.analyzers_list}" - ) - results = answer.get("results", []) - elapsed_time = answer.get("elapsed_time_in_seconds", []) - break - if not results: - raise IntelOwlClientException( - f"[ENDED] Reached polling timeout without results." - f" Job_id: {job_id_to_get}" - ) - - except IntelOwlClientException as e: - logger.error(f"Error: {e} md5: {md5}") - except requests.exceptions.HTTPError as e: - logger.exception(e) - except Exception as e: - logger.exception(e) - - logger.info(f"Elapsed time: {elapsed_time}") - logger.info("Results:") - if args.show_colors: - checkers = Checkers(results, args.value) - observable = get_observable_classification(args.value) - if "domain" in observable: - checkers.check_domain() - elif "hash" in observable: - checkers.check_hash() - elif "url" in observable: - checkers.check_url() - else: - checkers.check_ip() - else: - pprint(results) - - -def get_logger(debug_mode, log_to_file): - if debug_mode: - log_level = logging.DEBUG - else: - log_level = logging.INFO - if log_to_file: - handler = logging.FileHandler(log_to_file) - else: - handler = logging.StreamHandler() - formatter = logging.Formatter( - "[%(asctime)s - %(levelname)s] %(message)s", "%Y-%m-%d %H:%M:%S" - ) - handler.setFormatter(formatter) - logger = logging.getLogger(__name__) - logger.setLevel(log_level) - logger.addHandler(handler) - return logger - - -if __name__ == "__main__": - intel_owl_client() From 30b54104b064dc36931efc580b1d6de0c86cd2fe Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Sat, 5 Dec 2020 21:03:17 +0530 Subject: [PATCH 34/38] fix .netrc not present error --- pyintelowl/cli/_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyintelowl/cli/_utils.py b/pyintelowl/cli/_utils.py index 0a8a380..42ec076 100644 --- a/pyintelowl/cli/_utils.py +++ b/pyintelowl/cli/_utils.py @@ -2,6 +2,7 @@ import logging import csv import json +import pathlib from tinynetrc import Netrc from rich.emoji import Emoji from rich.text import Text @@ -71,7 +72,9 @@ def _add_options(func): def get_netrc_obj(): - netrc = Netrc() + filepath = pathlib.Path().home().joinpath(".netrc") + filepath.touch(exist_ok=True) + netrc = Netrc(str(filepath)) host = netrc["pyintelowl"] return netrc, host From 8d0295bf2f8f411999fadc15ee0d6147907b6fcf Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Sat, 5 Dec 2020 21:32:00 +0530 Subject: [PATCH 35/38] version opt, version no to file, remove click_completion, test requirements to setup --- cli.py | 13 +++++++------ pyintelowl/cli/_utils.py | 5 +++++ setup.py | 13 ++++++++----- test-requirements.txt | 3 --- version.txt | 1 + 5 files changed, 21 insertions(+), 14 deletions(-) delete mode 100644 test-requirements.txt create mode 100644 version.txt diff --git a/cli.py b/cli.py index 4f4d51e..f6d468a 100644 --- a/cli.py +++ b/cli.py @@ -1,17 +1,18 @@ #!/usr/bin/env python3 import click -import click_completion from pyintelowl.pyintelowl import IntelOwl from pyintelowl.cli import groups, cmds -from pyintelowl.cli._utils import get_logger, ClickContext, get_netrc_obj - - -# Enable auto completion -click_completion.init() +from pyintelowl.cli._utils import ( + get_logger, + ClickContext, + get_netrc_obj, + get_version_number, +) @click.group(context_settings=dict(help_option_names=["-h", "--help"])) @click.option("-d", "--debug", is_flag=True, help="Set log level to DEBUG") +@click.version_option(version=get_version_number()) @click.pass_context def cli(ctx: ClickContext, debug: bool): netrc, host = get_netrc_obj() diff --git a/pyintelowl/cli/_utils.py b/pyintelowl/cli/_utils.py index 42ec076..aca92ec 100644 --- a/pyintelowl/cli/_utils.py +++ b/pyintelowl/cli/_utils.py @@ -107,3 +107,8 @@ def get_json_data(filepath): reader = csv.DictReader(fp) obj = [dict(row) for row in reader] return obj + + +def get_version_number() -> str: + f = pathlib.Path(__file__).joinpath("../../../version.txt").resolve() + return f.read_text() diff --git a/setup.py b/setup.py index 40e0074..2ce9c97 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,8 @@ # The text of the README file README = (HERE / "README.md").read_text() +VERSION = (HERE / "version.txt").read_text() + GITHUB_URL = "https://github.com/intelowlproject/pyintelowl" requirements = [ @@ -20,14 +22,15 @@ "geocoder==1.38.1", "click==7.1.2", "rich==9.2.0", - "click_completion", "tinynetrc==1.3.0", ] +requirements_test = ["black==20.8b1", "flake8==3.7.9", "pre-commit==2.7.1"] + # This call to setup() does all the work setup( name="pyintelowl", - version="3.0.0", + version=VERSION, description="Robust Python SDK and CLI for IntelOwl's API", long_description=README, long_description_content_type="text/markdown", @@ -53,7 +56,7 @@ "Topic :: Software Development :: Libraries :: Python Modules", ], packages=["pyintelowl"], - python_requires="~=3.6", + python_requires=">=3.6", include_package_data=True, install_requires=requirements, project_urls={ @@ -68,8 +71,8 @@ # for example: # $ pip install -e .[dev,test] extras_require={ - "dev": ["black==20.8b1", "flake8"] + requirements, - "test": ["black==20.8b1", "flake8"] + requirements, + "dev": requirements_test + requirements, + "test": requirements_test + requirements, }, # pip install --editable . entry_points=""" diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 820e5cf..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -flake8==3.7.9 -black==20.8b1 -pre-commit==2.7.1 \ No newline at end of file diff --git a/version.txt b/version.txt new file mode 100644 index 0000000..56fea8a --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +3.0.0 \ No newline at end of file From 1dce6242f90f465a94184690b6f74c0600869e64 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Sat, 5 Dec 2020 23:57:27 +0530 Subject: [PATCH 36/38] complete typehinting and docs for IntelOwl cls --- pyintelowl/pyintelowl.py | 193 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 181 insertions(+), 12 deletions(-) diff --git a/pyintelowl/pyintelowl.py b/pyintelowl/pyintelowl.py index 4f7a5d5..2c97bad 100644 --- a/pyintelowl/pyintelowl.py +++ b/pyintelowl/pyintelowl.py @@ -5,8 +5,7 @@ import re import requests import hashlib - -from typing import List, Dict, Any +from typing import List, Dict, Any, Union, AnyStr from .exceptions import IntelOwlClientException @@ -31,6 +30,9 @@ def __init__( @property def session(self) -> requests.Session: + """ + Internal use only. + """ if not hasattr(self, "_session"): session = requests.Session() if self.certificate: @@ -52,6 +54,23 @@ def ask_analysis_availability( run_all_available_analyzers: bool = False, check_reported_analysis_too: bool = False, ) -> Dict: + """Search for already available analysis.\n + Endpoint: ``/api/ask_analysis_availability`` + + Args: + md5 (str): md5sum of the observable or file + analyzers_needed (List[str]): list of analyzers to invoke + run_all_available_analyzers (bool, optional): + If True, runs all compatible analyzers. Defaults to ``False``. + check_reported_analysis_too (bool, optional): + Check against all existing jobs. Defaults to ``False``. + + Raises: + IntelOwlClientException: on client/HTTP error + + Returns: + Dict: JSON body + """ answer = None try: params = {"md5": md5, "analyzers_needed": analyzers_needed} @@ -91,6 +110,33 @@ def send_file_analysis_request( run_all_available_analyzers: bool = False, runtime_configuration: Dict = {}, ) -> Dict: + """Send analysis request for a file.\n + Endpoint: ``/api/send_analysis_request`` + + Args: + analyzers_requested (List[str]): + List of analyzers to invoke + filename (str): + Filename + binary (bytes): + File contents as bytes + force_privacy (bool, optional): + Disable analyzers that can leak info. Defaults to ``False``. + private_job (bool, optional): + Limit view permissions to your groups . Defaults to ``False``. + disable_external_analyzers (bool, optional): + Disable analyzers that use external services. Defaults to ``False``. + run_all_available_analyzers (bool, optional): + If True, runs all compatible analyzers. Defaults to ``False``. + runtime_configuration (Dict, optional): + Overwrite configuration for analyzers. Defaults to ``{}``. + + Raises: + IntelOwlClientException: on client/HTTP error + + Returns: + Dict: JSON body + """ answer = None try: data = { @@ -121,6 +167,31 @@ def send_observable_analysis_request( run_all_available_analyzers: bool = False, runtime_configuration: Dict = {}, ) -> Dict: + """Send analysis request for an observable.\n + Endpoint: ``/api/send_analysis_request`` + + Args: + analyzers_requested (List[str]): + List of analyzers to invoke + observable_name (str): + Observable value + force_privacy (bool, optional): + Disable analyzers that can leak info. Defaults to ``False``. + private_job (bool, optional): + Limit view permissions to your groups . Defaults to ``False``. + disable_external_analyzers (bool, optional): + Disable analyzers that use external services. Defaults to ``False``. + run_all_available_analyzers (bool, optional): + If True, runs all compatible analyzers. Defaults to ``False``. + runtime_configuration (Dict, optional): + Overwrite configuration for analyzers. Defaults to ``{}``. + + Raises: + IntelOwlClientException: on client/HTTP error + + Returns: + Dict: JSON body + """ answer = None try: data = { @@ -143,9 +214,19 @@ def send_observable_analysis_request( raise IntelOwlClientException(e) return answer - def send_analysis_batch(self, rows: List): + def send_analysis_batch(self, rows: List[Dict]): """ Send multiple analysis requests. + Can be mix of observable or file analysis requests. + + Used by the pyintelowl CLI. + + Args: + rows (List[Dict]): + Each row should be a dictionary with keys, + `value`, `type`, `analyzers_list`, `run_all` + `force_privacy`, `private_job`, `disable_external_analyzers`, + `check`. """ for obj in rows: try: @@ -172,6 +253,9 @@ def send_analysis_batch(self, rows: List): self.logger.fatal(str(e)) def __send_analysis_request(self, data=None, files=None): + """ + Internal use only. + """ url = self.instance + "/api/send_analysis_request" response = self.session.post(url, data=data, files=files) self.logger.debug( @@ -202,6 +286,10 @@ def __send_analysis_request(self, data=None, files=None): return answer def ask_analysis_result(self, job_id): + """ + will be deprecated soon. Use `get_job_by_id` function instead.\n + Endpoint: ``/api/ask_analysis_result`` + """ answer = None try: params = {"job_id": job_id} @@ -215,6 +303,10 @@ def ask_analysis_result(self, job_id): return answer def get_analyzer_configs(self): + """ + Get current state of `analyzer_config.json` from the IntelOwl instance.\n + Endpoint: ``/api/get_analyzer_configs`` + """ answer = None try: url = self.instance + "/api/get_analyzer_configs" @@ -226,7 +318,17 @@ def get_analyzer_configs(self): raise IntelOwlClientException(e) return answer - def get_all_tags(self): + def get_all_tags(self) -> List[Dict[str, str]]: + """ + Fetch list of all tags.\n + Endpoint: ``/api/tags`` + + Raises: + IntelOwlClientException: on client/HTTP error + + Returns: + List[Dict[str, str]]: List of tags + """ answer = None try: url = self.instance + "/api/tags" @@ -238,7 +340,17 @@ def get_all_tags(self): raise IntelOwlClientException(e) return answer - def get_all_jobs(self): + def get_all_jobs(self) -> List[Dict[str, Any]]: + """ + Fetch list of all jobs.\n + Endpoint: ``/api/jobs`` + + Raises: + IntelOwlClientException: on client/HTTP error + + Returns: + List[Dict[str, Any]]: List of jobs + """ answer = None try: url = self.instance + "/api/jobs" @@ -250,7 +362,19 @@ def get_all_jobs(self): raise IntelOwlClientException(e) return answer - def get_tag_by_id(self, tag_id) -> Dict: + def get_tag_by_id(self, tag_id: Union[int, str]) -> Dict[str, str]: + """Fetch tag info by ID.\n + Endpoint: ``/api/tag/{tag_id}`` + + Args: + tag_id (Union[int, str]): Tag ID + + Raises: + IntelOwlClientException: on client/HTTP error + + Returns: + Dict[str, str]: Dict with 3 keys: `id`, `label` and `color`. + """ answer = None try: url = self.instance + "/api/tags/" @@ -262,7 +386,19 @@ def get_tag_by_id(self, tag_id) -> Dict: raise IntelOwlClientException(e) return answer - def get_job_by_id(self, job_id) -> Dict: + def get_job_by_id(self, job_id: Union[int, str]) -> Dict[str, Any]: + """Fetch job info by ID. + Endpoint: ``/api/job/{job_id}`` + + Args: + job_id (Union[int, str]): Job ID + + Raises: + IntelOwlClientException: on client/HTTP error + + Returns: + Dict[str, Any]: JSON body. + """ answer = None try: url = self.instance + "/api/jobs/" + str(job_id) @@ -275,7 +411,24 @@ def get_job_by_id(self, job_id) -> Dict: return answer @staticmethod - def get_md5(to_hash: Any, type_="observable") -> str: + def get_md5( + to_hash: AnyStr, + type_: Union["observable", "binary", "file"] = "observable", + ) -> str: + """Returns md5sum of given observable or file object. + + Args: + to_hash (AnyStr): + either an observable string, file contents as bytes or path to a file + type_ (Union["observable", "binary", "file"], optional): + `observable`, `binary`, `file`. Defaults to "observable". + + Raises: + IntelOwlClientException: on client/HTTP error + + Returns: + str: md5sum + """ md5 = "" if type_ == "observable": md5 = hashlib.md5(str(to_hash).lower().encode("utf-8")).hexdigest() @@ -301,7 +454,10 @@ def _new_analysis_cli( check, runtime_configuration: Dict = {}, should_poll: bool = False, - ): + ) -> None: + """ + For internal use by the pyintelowl CLI. + """ # CLI sanity checks if analyzers_list and run_all: self.logger.warn( @@ -381,9 +537,22 @@ def _new_analysis_cli( ) @staticmethod - def _get_observable_classification(value) -> str: - # only following types are supported: - # ip - domain - url - hash (md5, sha1, sha256) + def _get_observable_classification(value: str) -> str: + """Returns observable classification for the given value.\n + Only following types are supported: + ip, domain, url, hash (md5, sha1, sha256) + + Args: + value (str): + observable value + + Raises: + IntelOwlClientException: + if value type is not recognized + + Returns: + str: one of `ip`, `url`, `domain` or `hash`. + """ try: ipaddress.ip_address(value) except ValueError: From 7ce487ffc4b5c54f9893a9e0bca0b9d9cadad2ad Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Sun, 6 Dec 2020 00:09:44 +0530 Subject: [PATCH 37/38] lots of docs, CHANGELOG.md --- .github/CHANGELOG.md | 88 +++++++++++++++++++++++++++++++++++++ .gitignore | 3 +- README.md | 60 +++++++++++++------------ docs/Makefile | 20 +++++++++ docs/conf.py | 84 +++++++++++++++++++++++++++++++++++ docs/index.rst | 100 ++++++++++++++++++++++++++++++++++++++++++ docs/make.bat | 35 +++++++++++++++ docs/pyintelowl.rst | 18 ++++++++ docs/requirements.txt | 4 ++ 9 files changed, 382 insertions(+), 30 deletions(-) create mode 100644 .github/CHANGELOG.md create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/make.bat create mode 100644 docs/pyintelowl.rst create mode 100644 docs/requirements.txt diff --git a/.github/CHANGELOG.md b/.github/CHANGELOG.md new file mode 100644 index 0000000..fdbffa7 --- /dev/null +++ b/.github/CHANGELOG.md @@ -0,0 +1,88 @@ +# Changelog + +## 3.0.0 (https://github.com/intelowlproject/pyintelowl/releases/tag/3.0.0) + +*Note: Incompatible with previous versions* + +This version brings a complete rewrite of the pyintelowl library as well as command line client. We very much recommend you to update to the latest version to enjoy all new features. + +- The new CLI is written with [pallets/click](https://github.com/pallets/click) and supports all IntelOwl API endpoints. The CLI is well-documented and will help you navigate different commands; you can use it to request new analysis, view an old analysis, view `analyzer_config.json`, view list of tags, list of jobs, etc. +- Complete type-hinting and sphinx docs for the `pyintelowl.IntelOwl` class with helper member functions for each IntelOwl API endpoint. + + +## 2.0.0 (https://github.com/intelowlproject/pyintelowl/releases/tag/2.0.0) +**This version supports only IntelOwl versions >=1.8.0 (about to be released). To interact with previous IntelOwl versions programmatically please refer to pyintelowl version 1.3.5** + +* we forced [black](https://github.com/psf/black) style, added linters and precommit configuration. In this way pyintelowl is aligned to IntelOwl. +* we have updated the authentication method from a JWT Token to a simple Token. In this way, it is easier to use pyintelowl for integrations with other products and there are no more concurrency problems on multiple simultaneous requests. + +If you were using pyintelowl and IntelOwl before this version, you have to: +* update IntelOwl to version>=1.8.0 +* retrieve a new API token from the Django Admin Interface for your user: you have to go in the *Durin* section (click on `Auth tokens`) and generate a key there. This token is valid until manually deleted. + + +## 1.3.5 (https://github.com/intelowlproject/pyintelowl/releases/tag/1.3.5) +Now optional parameter "runtime_configuration" properly works + +Please use this version of pyintelowl with version >= 1.5.x of IntelOwl + + +## 1.3.4 (https://github.com/intelowlproject/pyintelowl/releases/tag/1.3.4) +see [1.3.3](https://github.com/intelowlproject/pyintelowl/releases/tag/1.3.3) for details + + +## 1.3.3 (https://github.com/intelowlproject/pyintelowl/releases/tag/1.3.3) +Some fixes: + +* pyintelowl did not work correctly against HTTPS-enabled IntelOwl Servers +* fixed parameter name in send_observable_analysis_request + +Please use this version of pyintelowl with v1.5.x of IntelOwl + + +## 1.3.2 (https://github.com/intelowlproject/pyintelowl/releases/tag/1.3.2) +Patch Release after [1.3.0](https://github.com/intelowlproject/pyintelowl/releases/tag/1.3.0). + +- renamed `additional_configuration` to `runtime_configuration`. +- Formatting with psf/black formatter. + +**Please use this version of pyintelowl with v1.5.x of IntelOwl.** + + +## 1.3.1 (https://github.com/intelowlproject/pyintelowl/releases/tag/1.3.1) +Fixes and improvements to "--show-colors" option + + +## 1.3.0 (https://github.com/intelowlproject/pyintelowl/releases/tag/1.3.0) +reformatted some code + added support for new parameter "additional_configuration" + + +## 1.2.0 (https://github.com/intelowlproject/pyintelowl/releases/tag/1.2.0) +PR #16 for details. + + +## 1.1.0 (https://github.com/intelowlproject/pyintelowl/releases/tag/1.1.0) +Added an option when executing pyintelowl as CLI: `-sc` will show the results in a colorful and organized way that helps the user in looking for useful information. By default, the results are still shown in the JSON format. Thanks to tsale to his idea and contribution. + +**Example:** + +``` +python3 intel_owl_client.py -i -sc -a VirusTotal_v2_Get_Observable -a HybridAnalysis_Get_Observable -a OTXQuery observable -v www.google.com +``` + + +## 1.0.0 (https://github.com/intelowlproject/pyintelowl/releases/tag/1.0.0) +For all the details, check the official blog post: + +https://www.honeynet.org/2020/07/05/intel-owl-release-v1-0-0/ + +This version is compatible only with the related (1.x) IntelOwl release. + +## 0.2.1 + +## 0.2.0 + +## 0.1.2 + +## 0.1.1 + diff --git a/.gitignore b/.gitignore index 34d23e0..f8f4547 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ test_files/ *.exe __pycache__ .vscode -api_token_value.txt \ No newline at end of file +api_token_value.txt +docs/_build/ \ No newline at end of file diff --git a/README.md b/README.md index c4aff43..e3d48cf 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,15 @@ Robust Python **SDK** and **Command Line Client** for interacting with [IntelOwl](https://github.com/intelowlproject/IntelOwl)'s API. -You can select which analyzers you want to run for every analysis you perform. - -For additional help, we suggest to check the ["How to use pyintelowl" Youtube Video](https://www.youtube.com/watch?v=fpd6Kt9EZdI) by [Kostas](https://github.com/tsale). +## Features +- Easy one-time configuration with self documented help and hints along the way. +- Request new analysis for observables and files. + - Select which analyzers you want to run for every analysis you perform. + - Choose whether you want to HTTP poll for the analysis to finish or not. +- List all jobs or view one job in a prettified tabular form. +- List all tags or view one tag in a prettified tabular form. +- Tabular view of the `analyzer_config.json` from IntelOwl with RegEx matching capabilities. ## Installation @@ -19,54 +24,51 @@ $ pip3 install pyintelowl For development/testing, `pip3 install pyintelowl[dev]` -## Usage +## Quickstart ### As Command Line Client +On successful installation, The `pyintelowl` entryscript should be directly invokable. For example, + ```bash -$ python3 cli.py -h -Usage: cli.py [OPTIONS] COMMAND [ARGS]... +$ pyintelowl +Usage: pyintelowl [OPTIONS] COMMAND [ARGS]... Options: - -k, --api-key TEXT API key to authenticate against a IntelOwl instance - [required] - - -u, --instance-url TEXT IntelOwl instance URL [required] - -c, --certificate PATH Path to SSL client certificate file (.pem) - --debug / --no-debug Set log level to DEBUG - -h, --help Show this message and exit. + -d, --debug Set log level to DEBUG + --version Show the version and exit. + -h, --help Show this message and exit. Commands: analyse Send new analysis request config Set or view config variables - get-analyzer-config Get current state of analyzer_config.json from the IntelOwl instance - jobs List jobs - tags List tags + get-analyzer-config Get current state of `analyzer_config.json` from the... + jobs Manage Jobs + tags Manage tags ``` -## As a library / SDK - -`from pyintelowl.pyintelowl import IntelOwl` +### As a library / SDK -#### Endpoints -`ask_analysis_availability` -> search for already available analysis - -`send_file_analysis_request` -> send a file to be analyzed - -`send_observable_analysis_request` -> send an observable to be analyzed +```python +from pyintelowl import IntelOwl +obj = IntelOwl("", "", "optional") +``` -`ask_analysis_result` -> request analysis result by job ID +For more comprehensive documentation, please see https://pyintelowl.readthedocs.io/. -`get_analyzer_configs` -> get the analyzers configuration +## Changelog +View [CHANGELOG.md](https://github.com/intelowlproject/pyintelowl/blob/master/.github/CHANGELOG.md). ## FAQ -### Generate API key +#### Generate API key You need a valid API key to interact with the IntelOwl server. Keys should be created from the admin interface of [IntelOwl](https://github.com/intelowlproject/intelowl): you have to go in the *Durin* section (click on `Auth tokens`) and generate a key there. -You can use the with the parameter `-k ` from CLI +#### Incompatibility after version 3.0 + +We did a complete rewrite of the PyIntelOwl client and CLI both for the version `3.0.0`. We very much recommend you to update to the latest version to enjoy all new features. #### (old auth method) JWT Token Authentication > this auth was available in IntelOwl versions <1.8.0 and pyintelowl versions <2.0.0 diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..8fa6da8 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,84 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +import pathlib + + +sys.path.append(os.path.abspath("../")) + + +# -- Project information ----------------------------------------------------- + +project = "PyIntelOwl" +copyright = "2020, Matteo Lodi" +author = "Matteo Lodi" + + +VERSION = (pathlib.Path("../version.txt")).read_text() + +GITHUB_URL = "https://github.com/intelowlproject/pyintelowl" + +# -- General configuration --------------------------------------------------- + +html_title = f"{project} Documentation ({VERSION})" + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx_rtd_theme", + "sphinxcontrib.napoleon", + "sphinx.ext.autodoc", + "sphinx.ext.viewcode", + "sphinx.ext.todo", + "sphinxcontrib.asciinema", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "sphinx_rtd_theme" +html_theme_path = [ + "_themes", +] + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..6eee8d4 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,100 @@ +.. pyintelowl documentation master file, created by + sphinx-quickstart on Sat Dec 5 23:00:08 2020. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to PyIntelOwl's documentation! +====================================== + +Robust Python **SDK** and **Command Line Client** for interacting with `IntelOwl `__ API. + +Installation +-------------------------- + +.. code-block:: bash + + $ pip install pyintelowl + + +Usage as CLI +-------------------------- + +On successful installation, The ``pyintelowl`` entryscript should be directly invokable. For example, + +.. code-block:: bash + :emphasize-lines: 1 + + $ pyintelowl + Usage: pyintelowl [OPTIONS] COMMAND [ARGS]... + + Options: + -d, --debug Set log level to DEBUG + --version Show the version and exit. + -h, --help Show this message and exit. + + Commands: + analyse Send new analysis request + config Set or view config variables + get-analyzer-config Get current state of `analyzer_config.json` from the... + jobs Manage Jobs + tags Manage tags + +Configuration +^^^^^^^^^^^^^ + +You can use ``set`` to set the config variables and ``get`` to view them. + +.. code-block:: bash + :caption: `View on asciinema `__ + + $ pyintelowl config set -k 4bf03f20add626e7138f4023e4cf52b8 -u "http://localhost:80" + $ pyintelowl config get + +.. Hint:: + The CLI would is well-documented which will help you navigate various commands easily. + Invoke ``pyintelowl -h`` or ``pyintelowl -h`` to get help. + + +Usage as SDK/library +-------------------------- + +.. code-block:: python + :linenos: + + from pyintelowl import IntelOwl, IntelOwlClientException + obj = IntelOwl( + "4bf03f20add626e7138f4023e4cf52b8", + "http://localhost:80", + None, + ) + """ + obj = IntelOwl( + "", + "", + "optional" + ) + """ + + try: + ans = obj.get_analyzer_configs(1) + print(ans) + except IntelOwlClientException as e: + print("Oh no! Error: ", e) + +.. Tip:: We very much **recommend** going through the :class:`pyintelowl.pyintelowl.IntelOwl` docs. + + +API Client Docs +================== + +.. toctree:: + :maxdepth: 3 + + Index + pyintelowl + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..2119f51 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/pyintelowl.rst b/docs/pyintelowl.rst new file mode 100644 index 0000000..a80ae27 --- /dev/null +++ b/docs/pyintelowl.rst @@ -0,0 +1,18 @@ +pyintelowl modules +============================ + +pyintelowl.pyintelowl module +---------------------------- + +.. automodule:: pyintelowl.pyintelowl + :members: + :undoc-members: + :show-inheritance: + +pyintelowl.exceptions module +---------------------------- + +.. automodule:: pyintelowl.exceptions + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..8adf84c --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +Sphinx==3.2.1 +sphinx-rtd-theme +sphinxcontrib.asciinema +sphinxcontrib-napoleon \ No newline at end of file From ca642be47f02f0a48154b7ccb262545ca55ece8e Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Sun, 6 Dec 2020 19:08:31 +0530 Subject: [PATCH 38/38] fix flake8 error, change imports --- cli.py | 2 +- pyintelowl/__init__.py | 3 +++ pyintelowl/cli/_utils.py | 2 +- pyintelowl/cli/analyse.py | 15 +++++++++------ 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/cli.py b/cli.py index f6d468a..265776b 100644 --- a/cli.py +++ b/cli.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import click -from pyintelowl.pyintelowl import IntelOwl +from pyintelowl import IntelOwl from pyintelowl.cli import groups, cmds from pyintelowl.cli._utils import ( get_logger, diff --git a/pyintelowl/__init__.py b/pyintelowl/__init__.py index e69de29..c4e80e8 100644 --- a/pyintelowl/__init__.py +++ b/pyintelowl/__init__.py @@ -0,0 +1,3 @@ +# flake8: noqa +from .pyintelowl import IntelOwl +from .exceptions import IntelOwlClientException diff --git a/pyintelowl/cli/_utils.py b/pyintelowl/cli/_utils.py index aca92ec..516c61b 100644 --- a/pyintelowl/cli/_utils.py +++ b/pyintelowl/cli/_utils.py @@ -9,7 +9,7 @@ from rich.syntax import Syntax from rich.logging import RichHandler -from pyintelowl.pyintelowl import IntelOwl +from pyintelowl import IntelOwl json_flag_option = [ click.option( diff --git a/pyintelowl/cli/analyse.py b/pyintelowl/cli/analyse.py index 8c0e744..4a2b49f 100644 --- a/pyintelowl/cli/analyse.py +++ b/pyintelowl/cli/analyse.py @@ -75,7 +75,6 @@ def analyse(): """ Send new analysis request """ - pass @analyse.command(help="Send analysis request for an observable") @@ -166,12 +165,16 @@ def batch( filepath: str, ): rows = get_json_data(filepath) - flags = ["run_all", "force_privacy", "private_job", "disable_external_analyzers"] + # parse boolean columns + bool_flags = [ + "run_all", + "force_privacy", + "private_job", + "disable_external_analyzers", + ] for row in rows: - for flag in flags: - row[flag] = (row.get(flag, False).lower() == "true") | ( - row.get(flag, False) == True - ) + for flag in bool_flags: + row[flag] = row.get(flag, False) in ["true", True] try: ctx.obj.send_analysis_batch(rows) except IntelOwlClientException as e: