diff --git a/.gitignore b/.gitignore index 616aa923..6471dfcb 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,6 @@ fabric.properties logs/* */logs/* *.log.* + +# MacOS ds files +.DS_Store diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f7ecdb74..07e253b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ variables: VAULT_ROLE: 'python-sast-readonly' VAULT_ADDR: 'https://tluav-lb.faradaysec.com' VAULT_SECRET_PATH: 'gitlab/SAST' + TYPING_BRANCH: 'dev' workflow: rules: diff --git a/.gitlab/ci/testing/.testing-gitlab-ci.yml b/.gitlab/ci/testing/.testing-gitlab-ci.yml index 9c5dc294..7d44db4e 100644 --- a/.gitlab/ci/testing/.testing-gitlab-ci.yml +++ b/.gitlab/ci/testing/.testing-gitlab-ci.yml @@ -11,7 +11,8 @@ - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/faradaysec/cloud/faraday_agent_parameters_types.git typing - cd typing - if [ -z "$TYPING_BRANCH" ]; then export TYPING_BRANCH="master"; fi - - echo $TYPING_BRANCH + - echo "Using faraday_agent_parameters_type branch -> $TYPING_BRANCH" + - git fetch - git checkout $TYPING_BRANCH - pip install . - cd .. diff --git a/CHANGELOG/3.4.0/200.md b/CHANGELOG/3.4.0/200.md new file mode 100644 index 00000000..0cf65535 --- /dev/null +++ b/CHANGELOG/3.4.0/200.md @@ -0,0 +1 @@ +[FIX] Fix agent websocket token not changing on registration token update #200 diff --git a/CHANGELOG/3.4.0/214.md b/CHANGELOG/3.4.0/214.md new file mode 100644 index 00000000..644242d0 --- /dev/null +++ b/CHANGELOG/3.4.0/214.md @@ -0,0 +1 @@ +[FIX] Resolved the issue where users were unable to execute user-defined templates and pre-defined TenableIO templates. Additionally, fixed the functionality to retrieve completed scan results and relaunch previously created scans. #214 diff --git a/CHANGELOG/3.4.0/216.md b/CHANGELOG/3.4.0/216.md new file mode 100644 index 00000000..4952c8b7 --- /dev/null +++ b/CHANGELOG/3.4.0/216.md @@ -0,0 +1 @@ +[ADD] Introducing the new Tenable SC agent, now available for integration. This initial version focuses on supporting scan imports. #216 diff --git a/CHANGELOG/3.4.0/217.md b/CHANGELOG/3.4.0/217.md new file mode 100644 index 00000000..c1de6d1f --- /dev/null +++ b/CHANGELOG/3.4.0/217.md @@ -0,0 +1 @@ +[ADD] Added new agent for Cisco Cyber Vision. Also added severity calc utility. #217 diff --git a/CHANGELOG/3.4.0/date.md b/CHANGELOG/3.4.0/date.md new file mode 100644 index 00000000..d7f736d5 --- /dev/null +++ b/CHANGELOG/3.4.0/date.md @@ -0,0 +1 @@ +May 22th, 2024 diff --git a/RELEASE.md b/RELEASE.md index 430b5a45..d5a45f9c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,10 @@ +3.4.0 [May 22th, 2024]: +--- + * [ADD] Introducing the new Tenable SC agent, now available for integration. This initial version focuses on supporting scan imports. #216 + * [ADD] Added new agent for Cisco Cyber Vision. Also added severity calc utility. #217 + * [FIX] Fix agent websocket token not changing on registration token update #200 + * [FIX] Resolved the issue where users were unable to execute user-defined templates and pre-defined TenableIO templates. Additionally, fixed the functionality to retrieve completed scan results and relaunch previously created scans. #214 + 3.3.0 [Mar 12th, 2024]: --- * [ADD] Add hotspots option to SonarQube. #197 diff --git a/faraday_agent_dispatcher/__init__.py b/faraday_agent_dispatcher/__init__.py index c7a1437b..fcd8d1ee 100644 --- a/faraday_agent_dispatcher/__init__.py +++ b/faraday_agent_dispatcher/__init__.py @@ -20,4 +20,4 @@ __author__ = """Faraday Development Team""" __email__ = "devel@infobytesec.com" -__version__ = "3.3.0" +__version__ = "3.4.0" diff --git a/faraday_agent_dispatcher/dispatcher_io.py b/faraday_agent_dispatcher/dispatcher_io.py index 67ae0a11..bec4e7d4 100644 --- a/faraday_agent_dispatcher/dispatcher_io.py +++ b/faraday_agent_dispatcher/dispatcher_io.py @@ -156,7 +156,7 @@ async def register(self, registration_token=None): if not await self.check_connection(): exit(1) - if self.agent_token is None: + if self.agent_token is None or registration_token is not None: try: control_registration_token("token", registration_token) except ValueError as ex: @@ -515,7 +515,11 @@ async def create_process(executor: Executor, args: dict, plugin_args: dict): # Executor Variables if isinstance(args, dict): for k in args: - env[f"EXECUTOR_CONFIG_{k.upper()}"] = str(args[k]) + # This should allow to correctly get lists from environment with json.loads + if isinstance(args[k], list): + env[f"EXECUTOR_CONFIG_{k.upper()}"] = json.dumps(args[k]) + else: + env[f"EXECUTOR_CONFIG_{k.upper()}"] = str(args[k]) else: logger.error("Args from data received has a not supported type") raise ValueError("Args from data received has a not supported type") diff --git a/faraday_agent_dispatcher/static/executors/official/cisco_cybervision.py b/faraday_agent_dispatcher/static/executors/official/cisco_cybervision.py new file mode 100644 index 00000000..96efb237 --- /dev/null +++ b/faraday_agent_dispatcher/static/executors/official/cisco_cybervision.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +import os +import sys +import json +import datetime +import requests +from urllib3.exceptions import InsecureRequestWarning +from faraday_agent_dispatcher.utils.severity_utils import severity_from_score + +API_BASE = "/api/3.0/" + + +def log(msg, end="\n"): + print(msg, file=sys.stderr, flush=True, end=end) + + +def cybervision_report_composer(url, token, preset_list, asset_tags, vuln_tags): + req_headers = {"accept": "application/json", "x-token-id": token} + requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) + presets_queue = [] + presets_id = {} + + # STAGE 1 - get preset list + req_url = f"{url}{API_BASE}presets" + try: + resp = requests.get(req_url, headers=req_headers, timeout=20, verify=False).json() + except TimeoutError: + log("Can't reach Cyber Vision: connection timed out") + sys.exit(1) + if "error" in resp: + log(f"API Error: {resp['error']}") + sys.exit(1) + for req_preset in preset_list: + for preset in resp: + if preset["label"] == req_preset: + presets_id[preset["label"]] = preset["id"] + presets_queue.append(preset["id"]) + + # STAGE 2 - get all vulns per preset + presets_vuln_collection = {} + step_c = 1 + step_s = 100 + for _id in presets_queue: + step_c = 1 + presets_vuln_collection[_id] = [] + while True: + req_url = f"{url}{API_BASE}presets/{_id}/visualisations/" f"vulnerability-list?page={step_c}&size={step_s}" + try: + resp = requests.get(req_url, headers=req_headers, timeout=20, verify=False) + if resp.content.decode("UTF-8") == "Service unavailable: data is not available yet": + raise ValueError(resp.content.decode("UTF-8")) + resp = resp.json() + except TimeoutError: + log("Can't reach Cyber Vision: connection timed out") + sys.exit(1) + except ValueError as ve: + log(f"{str(ve)} at preset {_id}") + break + if "error" in resp: + log(f"API Error: {resp['error']}") + break + if len(resp) < 1: + break + presets_vuln_collection[_id].append(resp) + step_c += 1 + + # STAGE 3 - processing vulns + hosts = {} + for pres_data in presets_id.items(): + for vuln_pack in presets_vuln_collection[pres_data[1]]: + for vuln in vuln_pack: + if not vuln["device"]["label"] in hosts: + hosts[vuln["device"]["label"]] = { + "ip": vuln["device"]["label"], + "description": f"Device ID {vuln['device']['id']}", + "mac": "" if not vuln["device"]["mac"] else vuln["device"]["mac"], + "vulnerabilities": [], + "tags": [pres_data[0]] + asset_tags, + } + if not vuln["title"] in [x["name"] for x in hosts[vuln["device"]["label"]]["vulnerabilities"]]: + try: + vuln["cvss"] = float(vuln["cvss"]) + except ValueError: + vuln["cvss"] = -1.0 + hosts[vuln["device"]["label"]]["vulnerabilities"].append( + { + "name": vuln["title"], + "desc": vuln["summary"], + "severity": severity_from_score(vuln["cvss"], 10.0), + "refs": [{"name": ref["link"], "type": "other"} for ref in vuln["links"]], + "external_id": vuln["id"], + "type": "Vulnerability", + "resolution": vuln["solution"], + "data": vuln["fullDescription"], + "status": "open", + "cve": [x["cve"] for i, x in enumerate(vuln_pack) if x["title"] == vuln["title"]], + "run_date": datetime.datetime.strptime( + vuln["publishTime"], "%Y-%m-%dT%H:%M:%SZ" + ).timestamp(), + "tags": vuln_tags, + } + ) + data = {"hosts": [x[1] for x in hosts.items()]} + print(json.dumps(data)) + + +def main(): + params_cybervision_token = os.getenv("CYBERVISION_TOKEN") + params_cybervision_url = os.getenv("CYBERVISION_HTTPS_URL") + params_cybervision_presets_env = os.getenv("EXECUTOR_CONFIG_CYBERVISION_PRESETS") + + params_cybervision_presets = [] + if params_cybervision_presets_env: + params_cybervision_presets = json.loads(params_cybervision_presets_env) + + if not params_cybervision_url.startswith("https://"): + log("Cyber Vision URL must be HTTPS") + sys.exit(1) + + params_vulnerability_tags = os.getenv("AGENT_CONFIG_VULN_TAG") or [] + if params_vulnerability_tags: + params_vulnerability_tags = params_vulnerability_tags.split(",") + params_asset_tags = os.getenv("AGENT_CONFIG_HOSTNAME_TAG") or [] + if params_asset_tags: + params_asset_tags = params_asset_tags.split(",") + + cybervision_report_composer( + params_cybervision_url, + params_cybervision_token, + params_cybervision_presets, + params_asset_tags, + params_vulnerability_tags, + ) + + +if __name__ == "__main__": + main() diff --git a/faraday_agent_dispatcher/static/executors/official/tenableio.py b/faraday_agent_dispatcher/static/executors/official/tenableio.py index 609c0a60..698627f2 100644 --- a/faraday_agent_dispatcher/static/executors/official/tenableio.py +++ b/faraday_agent_dispatcher/static/executors/official/tenableio.py @@ -2,19 +2,78 @@ import sys import time import re +import json from tenable.io import TenableIO from faraday_plugins.plugins.repo.nessus.plugin import NessusPlugin -from faraday_agent_dispatcher.utils.url_utils import resolve_hostname HTTP_REGEX = re.compile("^(http|https)://") +TEMPLATE_NAMES = [ + "asv", + "discovery", + "wannacry", + "intelamt", + "basic", + "patch_audit", + "webapp", + "malware", + "mobile", + "mdm", + "compliance", + "pci", + "offline", + "cloud_audit", + "scap", + "custom", + "ghost", + "spectre_meltdown", + "advanced", + "agent_advanced", + "agent_basic", + "agent_compliance", + "agent_scap", + "agent_malware", + "agent_custom", + "ripple-treck", + "zerologon", + "solorigate", + "hafnium", + "printnightmare", + "active_directory", + "log4shell", + "log4shell_dc", + "agent_log4shell", + "log4shell_vulnerable_ecosystem", + "eoy22", + "agent_inventory_collection", + "cisa_alert_aa22011a", + "contileaks", + "ransomware_ecosystem_2022", + "active_directory_identity", +] def log(msg): print(msg, file=sys.stderr) +def search_policy_id_by_template_name(tio, template_name): + policies = tio.policies.list() + for policy in policies: + if policy["name"] == template_name: + return policy["id"] + return None + + +def get_agent_group_uuid_by_agent_group_name(tio, agent_group_name): + agent_groups = tio.agent_groups.list() + for agent_group in agent_groups: + if agent_group["name"] == agent_group_name: + return agent_group["uuid"] + return None + + def search_scan_id(tio, TENABLE_SCAN_ID): scans = tio.scans.list() scans_id = "" @@ -33,6 +92,20 @@ def search_scan_id(tio, TENABLE_SCAN_ID): return scan +def parse_targets(tenable_scan_targets): + """ + Given a string of ips separated by ',' it takes out the http or https + and retrieves only the domain name or ip address""" + parsed_targets = [] + for ip in tenable_scan_targets: + if HTTP_REGEX.match(ip): + target = re.sub(HTTP_REGEX, "", ip).strip() + else: + target = ip.strip() + parsed_targets.append(target) + return parsed_targets + + def main(): ignore_info = os.getenv("AGENT_CONFIG_IGNORE_INFO", "False").lower() == "true" hostname_resolution = os.getenv("AGENT_CONFIG_RESOLVE_HOSTNAME", "True").lower() == "true" @@ -45,19 +118,17 @@ def main(): host_tag = os.getenv("AGENT_CONFIG_HOSTNAME_TAG", None) if host_tag: host_tag = host_tag.split(",") - - TENABLE_SCAN_NAME = os.getenv("EXECUTOR_CONFIG_TENABLE_SCAN_NAME", "faraday-scan") - TENABLE_SCANNER_NAME = os.getenv("EXECUTOR_CONFIG_TENABLE_SCANNER_NAME") - TENABLE_SCAN_ID = os.getenv("EXECUTOR_CONFIG_TENABLE_SCAN_ID") + TENABLE_SCAN_NAME = os.getenv("EXECUTOR_CONFIG_SCAN_NAME", "faraday-scan") + TENABLE_SCAN_ID = os.getenv("EXECUTOR_CONFIG_SCAN_ID") TENABLE_RELAUNCH_SCAN = os.getenv("EXECUTOR_CONFIG_RELAUNCH_SCAN", "False").lower() == "true" - TENABLE_SCAN_TARGET = os.getenv("EXECUTOR_CONFIG_TENABLE_SCAN_TARGET") - TENABLE_SCAN_TEMPLATE = os.getenv( - "EXECUTOR_CONFIG_TENABLE_SCAN_TEMPLATE", - "basic", - ) + TENABLE_SCAN_TARGETS = os.getenv("EXECUTOR_CONFIG_SCAN_TARGETS") # solo para user-defined template + TENABLE_TEMPLATE_NAME = os.getenv("EXECUTOR_CONFIG_TEMPLATE_NAME", "agent_basic") TENABLE_PULL_INTERVAL = os.getenv("TENABLE_PULL_INTERVAL", 30) TENABLE_ACCESS_KEY = os.getenv("TENABLE_ACCESS_KEY") TENABLE_SECRET_KEY = os.getenv("TENABLE_SECRET_KEY") + TENABLE_IS_USER_DEFINED = os.getenv("EXECUTOR_CONFIG_USE_USER_DEFINED_TEMPLATE", "False").lower() == "true" + TENABLE_AGENT_GROUP_NAME = os.getenv("EXECUTOR_CONFIG_AGENT_GROUP_NAME") + if not (TENABLE_ACCESS_KEY and TENABLE_SECRET_KEY): log("TenableIo access_key and secret_key were not provided") exit(1) @@ -77,26 +148,30 @@ def main(): return if TENABLE_SCAN_ID: scan = search_scan_id(tio, TENABLE_SCAN_ID) + elif TENABLE_IS_USER_DEFINED and TENABLE_SCAN_NAME and TENABLE_SCAN_TARGETS and TENABLE_TEMPLATE_NAME: + policy_id = search_policy_id_by_template_name( + tio, TENABLE_TEMPLATE_NAME + ) # User defined template policy id is searched by template_name + if not policy_id: + log("The provided template name does not exist.") # policy_id refers to a user_defined_template + exit(1) + targets = json.loads(TENABLE_SCAN_TARGETS) + parsed_targets = parse_targets(targets) + log(f"The targets are {parsed_targets}") + scan = tio.scans.create(name=TENABLE_SCAN_NAME, targets=parsed_targets, policy_id=policy_id) else: - if HTTP_REGEX.match(TENABLE_SCAN_TARGET): - target = re.sub(HTTP_REGEX, "", TENABLE_SCAN_TARGET) - else: - target = TENABLE_SCAN_TARGET - target_ip = resolve_hostname(target) - log(f"The target ip is {target_ip}") - if TENABLE_SCANNER_NAME: - scan = tio.scans.create( - name=TENABLE_SCAN_NAME, - targets=[target_ip], - template=TENABLE_SCAN_TEMPLATE, - scanner=TENABLE_SCANNER_NAME, - ) - else: - scan = tio.scans.create( - name=TENABLE_SCAN_NAME, - targets=[target_ip], - template=TENABLE_SCAN_TEMPLATE, - ) + if TENABLE_TEMPLATE_NAME not in TEMPLATE_NAMES: + log("The provided template name does not exist.") + exit(1) + agent_group_uuid = get_agent_group_uuid_by_agent_group_name(tio, TENABLE_AGENT_GROUP_NAME) + if not agent_group_uuid: + log("The provided agent group does not exist.") + exit(1) + scan = tio.scans.create( + name=TENABLE_SCAN_NAME, + template=TENABLE_TEMPLATE_NAME, + agent_group_id=[agent_group_uuid], # Must be a list, otherwise returns 500 + ) tio.scans.launch(scan["id"]) status = "pending" while status[-2:] != "ed": @@ -105,7 +180,9 @@ def main(): if status != "completed": log(f"Scanner ended with status {status}") exit(1) - report = tio.scans.export(scan["id"]) + report = tio.scans.export( + scan["id"] + ) # Valid report is assumed. If report isn't valid, executor will crash but dispatcher won't. plugin = NessusPlugin( ignore_info=ignore_info, hostname_resolution=hostname_resolution, diff --git a/faraday_agent_dispatcher/static/executors/official/tenablesc.py b/faraday_agent_dispatcher/static/executors/official/tenablesc.py new file mode 100644 index 00000000..01cb0fc7 --- /dev/null +++ b/faraday_agent_dispatcher/static/executors/official/tenablesc.py @@ -0,0 +1,112 @@ +import json +import os +import io +import sys +import zipfile as zp +from tenable.sc import TenableSC +from faraday_plugins.plugins.repo.nessus.plugin import NessusPlugin + + +def log(msg): + print(msg, file=sys.stderr) + + +def get_only_usable_ids(tsc, scan_ids): + tenable_scans = tsc.scan_instances.list() + usable_tenable_scans = [str(scan["id"]) for scan in tenable_scans["usable"]] + log(usable_tenable_scans) + return [_id for _id in scan_ids if str(_id) in usable_tenable_scans] + + +def process_scan( + tsc, scan_id, ignore_info=False, hostname_resolution=False, host_tag=False, service_tag=False, vuln_tag=False +): + log(f"Processing scan id {scan_id}") + try: + report = tsc.scan_instances.export_scan(scan_id) + except Exception as e: + log(e) + return {} + with zp.ZipFile(io.BytesIO(report.read()), "r") as zip_ref: + with zip_ref.open(zip_ref.namelist()[0]) as file: + plugin = NessusPlugin( + ignore_info=ignore_info, + hostname_resolution=hostname_resolution, + host_tag=host_tag, + service_tag=service_tag, + vuln_tag=vuln_tag, + ) + plugin.parseOutputString(file.read()) + return plugin.get_json() + return {} + + +def main(): + ignore_info = os.getenv("AGENT_CONFIG_IGNORE_INFO", "False").lower() == "true" + hostname_resolution = os.getenv("AGENT_CONFIG_RESOLVE_HOSTNAME", "True").lower() == "true" + vuln_tag = os.getenv("AGENT_CONFIG_VULN_TAG", None) + if vuln_tag: + vuln_tag = vuln_tag.split(",") + service_tag = os.getenv("AGENT_CONFIG_SERVICE_TAG", None) + if service_tag: + service_tag = service_tag.split(",") + host_tag = os.getenv("AGENT_CONFIG_HOSTNAME_TAG", None) + if host_tag: + host_tag = host_tag.split(",") + + tenable_scan_ids = os.getenv("EXECUTOR_CONFIG_TENABLE_SCAN_ID") + TENABLE_ACCESS_KEY = os.getenv("TENABLE_ACCESS_KEY") + TENABLE_SECRET_KEY = os.getenv("TENABLE_SECRET_KEY") + TENABLE_HOST = os.getenv("TENABLE_HOST") + + if not (TENABLE_ACCESS_KEY and TENABLE_SECRET_KEY): + log("TenableSC access_key and secret_key were not provided") + exit(1) + + if not TENABLE_HOST: + log("TenableSC Host not provided") + exit(1) + + if not tenable_scan_ids: + log("TenableSC Scan ID not provided") + exit(1) + + # it should be a list but it is save as a str in the environment + try: + tenable_scan_ids_list = json.loads(tenable_scan_ids) + except Exception as e: + log(f"TenableSC Scan IDs could not be parsed {e}") + exit(1) + + tsc = TenableSC(host=TENABLE_HOST, access_key=TENABLE_ACCESS_KEY, secret_key=TENABLE_SECRET_KEY) + usable_scan_ids = get_only_usable_ids(tsc, tenable_scan_ids_list) + + log(usable_scan_ids) + + responses = [] + for scan_id in usable_scan_ids: + processed_scan = process_scan( + tsc, + scan_id, + ignore_info=ignore_info, + hostname_resolution=hostname_resolution, + host_tag=host_tag, + service_tag=service_tag, + vuln_tag=vuln_tag, + ) + if processed_scan: + responses.append(processed_scan) + if responses: + final_response = json.loads(responses.pop(0)) + for response in responses: + json_response = json.loads(response) + for host in json_response["hosts"]: + final_response["hosts"].append(host) + print(json.dumps(final_response)) + + +if __name__ == "__main__": + try: + main() + except Exception as e: + log(f"Agent execution failed. {e}") diff --git a/faraday_agent_dispatcher/utils/severity_utils.py b/faraday_agent_dispatcher/utils/severity_utils.py new file mode 100644 index 00000000..b14a6d6e --- /dev/null +++ b/faraday_agent_dispatcher/utils/severity_utils.py @@ -0,0 +1,21 @@ +def severity_from_score(score: float, max_score: float): + """ + Returns severity string from score range. + + >> [score] - input score value + + >> [max_score] - could be 10 or 100 (or whatever, calc is proportional) + + << [severity] - severity as string "info", "low", "medium", "high" or "critical" + """ + if 0 <= score < max_score * 0.1: + return "info" + if max_score * 0.1 <= score < max_score * 0.4: + return "low" + if max_score * 0.4 <= score < max_score * 0.7: + return "medium" + if max_score * 0.7 <= score < max_score * 0.9: + return "high" + if max_score * 0.9 <= score < max_score: + return "critical" + return ""