From 59a14dbd9e8d85fd9016fe75200fa91c55679a76 Mon Sep 17 00:00:00 2001 From: Ziad Date: Tue, 29 Mar 2022 22:04:31 +0200 Subject: [PATCH] Add PyPI OSV importer Reference: https://github.com/nexB/vulnerablecode/issues/607 Signed-off-by: Ziad add the necessary changes Signed-off-by: Ziad remove aliases de-duplicate Signed-off-by: Ziad reslove conflicts Signed-off-by: Ziad add dateparser to setup.cfg Signed-off-by: Ziad Resolving conflicts Signed-off-by: Ziad resolve conflicts Signed-off-by: Ziad add a require changes Signed-off-by: Ziad add a require changes Signed-off-by: Ziad --- requirements.txt | 1 + setup.cfg | 1 + vulnerabilities/helpers.py | 13 + vulnerabilities/importers/__init__.py | 2 + vulnerabilities/importers/pysec.py | 252 +++++++++ .../tests/test_data/pysec/pysec_test.json | 280 ++++++++++ vulnerabilities/tests/test_pysec.py | 497 ++++++++++++++++++ 7 files changed, 1046 insertions(+) create mode 100644 vulnerabilities/importers/pysec.py create mode 100644 vulnerabilities/tests/test_data/pysec/pysec_test.json create mode 100644 vulnerabilities/tests/test_pysec.py diff --git a/requirements.txt b/requirements.txt index 8ed9f1f26..3cdf78a69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -113,3 +113,4 @@ wcwidth==0.2.5 websocket-client==0.59.0 yarl==1.7.2 zipp==3.8.0 +dateparser==1.1.1 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index d8e2eb341..34047fc55 100644 --- a/setup.cfg +++ b/setup.cfg @@ -75,6 +75,7 @@ install_requires = lxml>=4.6.4 defusedxml>=0.7.1 Markdown>=3.3.0 + dateparser>=1.1.1 # networking GitPython>=3.1.17 diff --git a/vulnerabilities/helpers.py b/vulnerabilities/helpers.py index 16587f64f..49d813fb1 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/helpers.py @@ -267,3 +267,16 @@ def _get_gh_response(gh_token, graphql_query): endpoint = "https://api.github.com/graphql" headers = {"Authorization": f"bearer {gh_token}"} return requests.post(endpoint, headers=headers, json=graphql_query).json() + + +def dedupe(original: List) -> List: + """ + Remove all duplicate items and return a new list preserving ordering + >>> dedupe(["z","i","a","a","d","d"]) + ['z', 'i', 'a', 'd'] + """ + new_list = [] + for i in original: + if i not in new_list: + new_list.append(i) + return new_list diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index b9c6d184b..dd4160001 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -24,6 +24,7 @@ from vulnerabilities.importers import nginx from vulnerabilities.importers import nvd from vulnerabilities.importers import openssl +from vulnerabilities.importers import pysec IMPORTERS_REGISTRY = [ nginx.NginxImporter, @@ -31,6 +32,7 @@ github.GitHubAPIImporter, nvd.NVDImporter, openssl.OpensslImporter, + pysec.PyPIImporter, ] IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} diff --git a/vulnerabilities/importers/pysec.py b/vulnerabilities/importers/pysec.py new file mode 100644 index 000000000..0c9c5e5dd --- /dev/null +++ b/vulnerabilities/importers/pysec.py @@ -0,0 +1,252 @@ +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. +import json +import logging +from io import BytesIO +from typing import Iterable +from zipfile import ZipFile + +import requests +from dateparser import parse +from packageurl import PackageURL +from univers.version_range import InvalidVersionRange +from univers.version_range import PypiVersionRange +from univers.versions import InvalidVersion +from univers.versions import PypiVersion +from univers.versions import SemverVersion + +from vulnerabilities.helpers import dedupe +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import Importer +from vulnerabilities.importer import Reference +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.severity_systems import SCORING_SYSTEMS + +logger = logging.getLogger(__name__) + + +class PyPIImporter(Importer): + license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" + spdx_license_expression = "CC-BY-4.0" + + def advisory_data(self) -> Iterable[AdvisoryData]: + """ + 1. Fetch the data from osv api + 2. unzip the file + 3. open the file one by one + 4. yield the json file to parse_advisory_data + """ + url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip" + response = requests.get(url).content + with ZipFile(BytesIO(response)) as zip_file: + for file_name in zip_file.namelist(): + with zip_file.open(file_name) as f: + vul_info = json.loads(f.read()) + yield parse_advisory_data(vul_info) + + +def parse_advisory_data(raw_data: dict) -> AdvisoryData: + summary = raw_data.get("summary") or "" + aliases = get_aliases(raw_data) + date_published = get_published_date(raw_data) + severity = get_severity(raw_data) + references = get_references(raw_data, severity) + + affected_packages = [] + if not "affected" in raw_data: + logger.error(f"affected_packages not found - {raw_data['id'] !r}") + + if "affected" in raw_data: + for affected_pkg in raw_data["affected"]: + purl = get_affected_purl(affected_pkg, raw_data["id"]) + if purl.type != "pypi": + logger.error( + f"Non PyPI package found in PYSEC advisories: {purl} - from: {raw_data['id']!r}" + ) + else: + vulnid = raw_data["id"] + affected_version_range = get_affected_version_range(affected_pkg, vulnid) + for fixed_range in affected_pkg.get("ranges", []): + fixed_version = get_fixed_version(fixed_range, vulnid) + + for version in fixed_version: + affected_packages.append( + AffectedPackage( + package=purl, + affected_version_range=affected_version_range, + fixed_version=version, + ) + ) + + return AdvisoryData( + aliases=aliases, + summary=summary, + affected_packages=affected_packages, + references=references, + date_published=date_published, + ) + + +def fixed_filter(fixed_range) -> []: + """ + Return a list of fixed version strings given a ``fixed_range`` mapping of OSV data. + >>> fixed_filter({"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "1.6.0"}]}) + ['1.6.0'] + >>> fixed_filter({"type": "ECOSYSTEM","events":[{"introduced": "0"},{"fixed": "1.0.0"},{"fixed": "9.0.0"}]}) + ['1.0.0', '9.0.0'] + """ + filter_fixed = list(filter(lambda x: x.keys() == {"fixed"}, fixed_range["events"])) + list_fixed = [i["fixed"] for i in filter_fixed] + return list_fixed + + +def get_aliases(raw_data) -> []: + """ + aliases field is optional , id is required and these are all aliases from our perspective + converting list of two fields to a dict then , convert it to a list to make sure a list is unique + >>> get_aliases({"id": "GHSA-j3f7-7rmc-6wqj"}) + ['GHSA-j3f7-7rmc-6wqj'] + >>> get_aliases({"aliases": ["CVE-2021-40831"]}) + ['CVE-2021-40831'] + >>> get_aliases({"aliases": ["CVE-2022-22817", "GHSA-8vj2-vxx3-667w"], "id": "GHSA-j3f7-7rmc-6wqj"}) + ['CVE-2022-22817', 'GHSA-8vj2-vxx3-667w', 'GHSA-j3f7-7rmc-6wqj'] + """ + vulnerability_id = raw_data.get("id") + vulnerability_aliases = raw_data.get("aliases") or [] + if vulnerability_id: + vulnerability_aliases.append(vulnerability_id) + return vulnerability_aliases + + +def get_published_date(raw_data): + if "published" in raw_data: + return parse(raw_data["published"]) + else: + logger.warning(f"date_published not found {raw_data['id'] !r}") + + +def get_severity(raw_data) -> []: + severities = [] + if "severity" in raw_data: + for sever_list in raw_data["severity"]: + if sever_list.get("type") == "CVSS_V3": + severities.append( + VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv3.1_vector"], + value=sever_list["score"], + ) + ) + else: + logger.error(f"NotImplementedError severity type- {raw_data['id']!r}") + + ecosys = raw_data.get("ecosystem_specific") or {} + sever = ecosys.get("severity") + if sever: + severities.append( + VulnerabilitySeverity( + system=SCORING_SYSTEMS["generic_textual"], + value=sever, + ) + ) + + database_specific = raw_data.get("database_specific") or {} + sever = database_specific.get("severity") + if sever: + severities.append( + VulnerabilitySeverity( + system=SCORING_SYSTEMS["generic_textual"], + value=sever, + ) + ) + + if not severities: + logger.warning(f"severity not found- {raw_data['id']!r}") + return [] + else: + return severities + + +def get_references(raw_data, severities) -> []: + references = raw_data.get("references") or [] + return [Reference(url=ref["url"], severities=severities) for ref in references if ref] + + +def get_affected_purl(affected_pkg, raw_id): + package = affected_pkg.get("package") or {} + purl = package.get("purl") + if purl: + try: + return PackageURL.from_string(purl) + except ValueError: + logger.error(f"PackageURL ValueError - {raw_id !r} - purl: {purl !r}") + + ecosys = package.get("ecosystem") + name = package.get("name") + if ecosys and name: + return PackageURL(type=ecosys, name=name) + else: + logger.error(f"purl affected_pkg not found - {raw_id !r}") + + +def get_affected_version_range(affected_pkg, raw_id): + affected_versions = affected_pkg.get("versions") + if affected_versions: + try: + return PypiVersionRange(affected_versions) + except InvalidVersionRange: + logger.error(f"InvalidVersionRange affected_pkg_version_range Error - {raw_id !r} ") + else: + logger.error(f"affected_pkg_version_range not found - {raw_id !r} ") + + +def get_fixed_version(fixed_range, raw_id) -> []: + """ + Return a list of fixed versions, using fixed_filter we get the list of fixed version strings, + then we pass every element to their univers.versions , then we dedupe the result + >>> get_fixed_version({}, "GHSA-j3f7-7rmc-6wqj") + [] + >>> get_fixed_version({"type": "ECOSYSTEM", "events": [{"fixed": "1.7.0"}]}, "GHSA-j3f7-7rmc-6wqj") + [PypiVersion(string='1.7.0')] + """ + fixed_version = [] + if not "type" in fixed_range: + logger.error(f"Invalid type - {raw_id!r}") + else: + list_fixed = fixed_filter(fixed_range) + fixed_range_type = fixed_range["type"] + for i in list_fixed: + if fixed_range_type == "ECOSYSTEM": + try: + fixed_version.append(PypiVersion(i)) + except InvalidVersion: + logger.error(f"Invalid Version - PypiVersion - {raw_id !r} - {i !r}") + if fixed_range_type == "SEMVER": + try: + fixed_version.append(SemverVersion(i)) + except InvalidVersion: + logger.error(f"Invalid Version - SemverVersion - {raw_id !r} - {i !r}") + if fixed_range_type == "GIT": + # TODO add GitHubVersion univers fix_version + logger.error(f"NotImplementedError GIT Version - {raw_id !r} - {i !r}") + + return dedupe(fixed_version) diff --git a/vulnerabilities/tests/test_data/pysec/pysec_test.json b/vulnerabilities/tests/test_data/pysec/pysec_test.json new file mode 100644 index 000000000..7cebf1fcf --- /dev/null +++ b/vulnerabilities/tests/test_data/pysec/pysec_test.json @@ -0,0 +1,280 @@ +[ + { + "id": "GHSA-j3f7-7rmc-6wqj", + "summary": "Improper certificate management in AWS IoT Device SDK v2", + "details": "The AWS IoT Device SDK v2 for Java, Python, C++ and Node.js appends a user supplied Certificate Authority (CA) to the root CAs instead of overriding it on macOS systems. Additionally, SNI validation is also not enabled when the CA has been \u201coverridden\u201d. TLS handshakes will thus succeed if the peer can be verified either from the user-supplied CA or the system\u2019s default trust-store. Attackers with access to a host\u2019s trust stores or are able to compromise a certificate authority already in the host's trust store (note: the attacker must also be able to spoof DNS in this case) may be able to use this issue to bypass CA pinning. An attacker could then spoof the MQTT broker, and either drop traffic and/or respond with the attacker's data, but they would not be able to forward this data on to the MQTT broker because the attacker would still need the user's private keys to authenticate against the MQTT broker. The 'aws_tls_ctx_options_override_default_trust_store_*' function within the aws-c-io submodule has been updated to address this behavior. This issue affects: Amazon Web Services AWS IoT Device SDK v2 for Java versions prior to 1.5.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for Python versions prior to 1.7.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for C++ versions prior to 1.14.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for Node.js versions prior to 1.6.0 on macOS. Amazon Web Services AWS-C-IO 0.10.7 on macOS.", + "aliases": [ + "CVE-2021-40831" + ], + "modified": "2022-02-15T05:31:22.788787Z", + "published": "2021-11-24T20:35:03Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-40831" + }, + { + "type": "WEB", + "url": "https://github.com/aws/aws-iot-device-sdk-cpp-v2" + } + ], + "affected": [ + { + "package": { + "name": "awsiotsdk", + "ecosystem": "PyPI", + "purl": "pkg:pypi/awsiotsdk" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.7.0" + } + ] + } + ], + "versions": [ + "0.2.4", + "0.2.9", + "0.3.0", + "1.0.2", + "1.0.3", + "1.0.5", + "1.0.6", + "1.1.0", + "1.2.0", + "1.2.1", + "1.3.0", + "1.3.1", + "1.3.2", + "1.4.0", + "1.5.0", + "1.5.1", + "1.5.10", + "1.5.11", + "1.5.12", + "1.5.13", + "1.5.14", + "1.5.15", + "1.5.16", + "1.5.17", + "1.5.18", + "1.5.2", + "1.5.3", + "1.5.4", + "1.5.5", + "1.5.6", + "1.5.7", + "1.5.8", + "1.6.0", + "1.6.1", + "1.6.2" + ], + "database_specific": { + "ghsa": "https://github.com/advisories/GHSA-j3f7-7rmc-6wqj", + "cvss": { + "vectorString": "CVSS:3.1/AV:A/AC:H/PR:H/UI:R/S:U/C:H/I:H/A:H", + "score": 6.3 + }, + "cwes": [ + { + "description": "The software does not validate, or incorrectly validates, a certificate.", + "cweId": "CWE-295", + "name": "Improper Certificate Validation" + } + ], + "source": "https://storage.googleapis.com/ghsa-osv/GHSA-j3f7-7rmc-6wqj.json" + } + }, + { + "package": { + "name": "aws-iot-device-sdk-v2", + "ecosystem": "npm", + "purl": "pkg:npm/aws-iot-device-sdk-v2" + }, + "ranges": [ + { + "type": "SEMVER", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.6.0" + } + ] + } + ], + "database_specific": { + "source": "https://storage.googleapis.com/ghsa-osv/GHSA-j3f7-7rmc-6wqj.json", + "ghsa": "https://github.com/advisories/GHSA-j3f7-7rmc-6wqj", + "cvss": { + "vectorString": "CVSS:3.1/AV:A/AC:H/PR:H/UI:R/S:U/C:H/I:H/A:H", + "score": 6.3 + }, + "cwes": [ + { + "description": "The software does not validate, or incorrectly validates, a certificate.", + "cweId": "CWE-295", + "name": "Improper Certificate Validation" + } + ] + } + } + ], + "schema_version": "1.2.0", + "severity": [ + {"type": "CVSS_V3", "score": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N"} + ] + + }, + { + "id": "PYSEC-2022-10", + "details": "PIL.ImageMath.eval in Pillow before 9.0.0 allows evaluation of arbitrary expressions, such as ones that use the Python exec method.", + "aliases": [ + "CVE-2022-22817", + "GHSA-8vj2-vxx3-667w" + ], + "modified": "2022-01-24T23:48:19.853348Z", + "published": "2022-01-10T14:12:00.853348Z", + "references": [ + { + "type": "WEB", + "url": "https://pillow.readthedocs.io/en/stable/releasenotes/9.0.0.html#restrict-builtins-available-to-imagemath-eval" + }, + { + "type": "WEB", + "url": "https://lists.debian.org/debian-lts-announce/2022/01/msg00018.html" + }, + { + "type": "ADVISORY", + "url": "https://github.com/advisories/GHSA-8vj2-vxx3-667w" + } + ], + "affected": [ + { + "package": { + "name": "pillow", + "ecosystem": "PyPI", + "purl": "pkg:pypi/pillow" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "9.0.0" + } + ] + } + ], + "versions": [ + "1.0", + "1.1", + "1.2", + "1.3", + "1.4", + "1.5", + "1.6", + "1.7.0", + "1.7.1", + "1.7.2", + "1.7.3", + "1.7.4", + "1.7.5", + "1.7.6", + "1.7.7", + "1.7.8", + "2.0.0", + "2.1.0", + "2.2.0", + "2.2.1", + "2.2.2", + "2.3.0", + "2.3.1", + "2.3.2", + "2.4.0", + "2.5.0", + "2.5.1", + "2.5.2", + "2.5.3", + "2.6.0", + "2.6.1", + "2.6.2", + "2.7.0", + "2.8.0", + "2.8.1", + "2.8.2", + "2.9.0", + "3.0.0", + "3.1.0", + "3.1.0.rc1", + "3.1.0rc1", + "3.1.1", + "3.1.2", + "3.2.0", + "3.3.0", + "3.3.1", + "3.3.2", + "3.3.3", + "3.4.0", + "3.4.1", + "3.4.2", + "4.0.0", + "4.1.0", + "4.1.1", + "4.2.0", + "4.2.1", + "4.3.0", + "5.0.0", + "5.1.0", + "5.2.0", + "5.3.0", + "5.4.0", + "5.4.0.dev0", + "5.4.1", + "6.0.0", + "6.1.0", + "6.2.0", + "6.2.1", + "6.2.2", + "7.0.0", + "7.1.0", + "7.1.1", + "7.1.2", + "7.2.0", + "8.0.0", + "8.0.1", + "8.1.0", + "8.1.1", + "8.1.2", + "8.2.0", + "8.3.0", + "8.3.1", + "8.3.2", + "8.4.0" + ], + "database_specific": { + "source": "https://github.com/pypa/advisory-database/blob/main/vulns/pillow/PYSEC-2022-10.yaml" + } + } + ], + "schema_version": "1.2.0", + "ecosystem_specific": { + "affected_functions": [ + "eventlet.websocket.WebSocket", + "eventlet.websocket.WebSocketWSGI" + ], + "severity": "HIGH" + + } +} +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_pysec.py b/vulnerabilities/tests/test_pysec.py new file mode 100644 index 000000000..d236cf1b9 --- /dev/null +++ b/vulnerabilities/tests/test_pysec.py @@ -0,0 +1,497 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. +import datetime +import json +import os +from unittest import TestCase + +from packageurl import PackageURL +from univers.version_range import PypiVersionRange +from univers.versions import PypiVersion + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import Reference +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.importers.pysec import fixed_filter +from vulnerabilities.importers.pysec import get_affected_purl +from vulnerabilities.importers.pysec import get_affected_version_range +from vulnerabilities.importers.pysec import get_aliases +from vulnerabilities.importers.pysec import get_fixed_version +from vulnerabilities.importers.pysec import get_published_date +from vulnerabilities.importers.pysec import get_references +from vulnerabilities.importers.pysec import get_severity +from vulnerabilities.importers.pysec import parse_advisory_data +from vulnerabilities.severity_systems import SCORING_SYSTEMS +from vulnerabilities.severity_systems import ScoringSystem + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/pysec", "pysec_test.json") + + +class TestPyPIImporter(TestCase): + def test_to_advisories(self): + first_aff_range = [ + "0.2.4", + "0.2.9", + "0.3.0", + "1.0.2", + "1.0.3", + "1.0.5", + "1.0.6", + "1.1.0", + "1.2.0", + "1.2.1", + "1.3.0", + "1.3.1", + "1.3.2", + "1.4.0", + "1.5.0", + "1.5.1", + "1.5.10", + "1.5.11", + "1.5.12", + "1.5.13", + "1.5.14", + "1.5.15", + "1.5.16", + "1.5.17", + "1.5.18", + "1.5.2", + "1.5.3", + "1.5.4", + "1.5.5", + "1.5.6", + "1.5.7", + "1.5.8", + "1.6.0", + "1.6.1", + "1.6.2", + ] + second_aff_range = [ + "1.0", + "1.1", + "1.2", + "1.3", + "1.4", + "1.5", + "1.6", + "1.7.0", + "1.7.1", + "1.7.2", + "1.7.3", + "1.7.4", + "1.7.5", + "1.7.6", + "1.7.7", + "1.7.8", + "2.0.0", + "2.1.0", + "2.2.0", + "2.2.1", + "2.2.2", + "2.3.0", + "2.3.1", + "2.3.2", + "2.4.0", + "2.5.0", + "2.5.1", + "2.5.2", + "2.5.3", + "2.6.0", + "2.6.1", + "2.6.2", + "2.7.0", + "2.8.0", + "2.8.1", + "2.8.2", + "2.9.0", + "3.0.0", + "3.1.0", + "3.1.0.rc1", + "3.1.0rc1", + "3.1.1", + "3.1.2", + "3.2.0", + "3.3.0", + "3.3.1", + "3.3.2", + "3.3.3", + "3.4.0", + "3.4.1", + "3.4.2", + "4.0.0", + "4.1.0", + "4.1.1", + "4.2.0", + "4.2.1", + "4.3.0", + "5.0.0", + "5.1.0", + "5.2.0", + "5.3.0", + "5.4.0", + "5.4.0.dev0", + "5.4.1", + "6.0.0", + "6.1.0", + "6.2.0", + "6.2.1", + "6.2.2", + "7.0.0", + "7.1.0", + "7.1.1", + "7.1.2", + "7.2.0", + "8.0.0", + "8.0.1", + "8.1.0", + "8.1.1", + "8.1.2", + "8.2.0", + "8.3.0", + "8.3.1", + "8.3.2", + "8.4.0", + ] + with open(TEST_DATA) as f: + mock_response = json.load(f) + + expected_advisories = [ + AdvisoryData( + aliases=["CVE-2021-40831", "GHSA-j3f7-7rmc-6wqj"], + summary="Improper certificate management in AWS IoT Device SDK v2", + affected_packages=[ + AffectedPackage( + package=PackageURL.from_string("pkg:pypi/awsiotsdk"), + affected_version_range=PypiVersionRange(first_aff_range), + fixed_version=PypiVersion("1.7.0"), + ), + ], + references=[ + Reference( + url="https://nvd.nist.gov/vuln/detail/CVE-2021-40831", + severities=[ + VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv3.1_vector"], + value="CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N", + ) + ], + ), + Reference( + url="https://github.com/aws/aws-iot-device-sdk-cpp-v2", + severities=[ + VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv3.1_vector"], + value="CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N", + ) + ], + ), + ], + date_published=datetime.datetime(2021, 11, 24, 20, 35, 3, 0).replace( + tzinfo=datetime.timezone.utc + ), + ), + AdvisoryData( + aliases=["CVE-2022-22817", "GHSA-8vj2-vxx3-667w", "PYSEC-2022-10"], + summary="", + affected_packages=[ + AffectedPackage( + package=PackageURL.from_string("pkg:pypi/pillow"), + affected_version_range=PypiVersionRange(second_aff_range), + fixed_version=PypiVersion("9.0.0"), + ) + ], + references=[ + Reference( + url="https://pillow.readthedocs.io/en/stable/releasenotes/9.0.0.html#restrict-builtins" + "-available-to-imagemath-eval", + severities=[ + VulnerabilitySeverity( + system=ScoringSystem( + identifier="generic_textual", + name="Generic textual severity rating", + url="", + notes="Severity for generic scoring systems. Contains generic textual values like High, Low etc", + ), + value="HIGH", + ) + ], + ), + Reference( + url="https://lists.debian.org/debian-lts-announce/2022/01/msg00018.html", + severities=[ + VulnerabilitySeverity( + system=ScoringSystem( + identifier="generic_textual", + name="Generic textual severity rating", + url="", + notes="Severity for generic scoring systems. Contains generic textual values like High, Low etc", + ), + value="HIGH", + ) + ], + ), + Reference( + url="https://github.com/advisories/GHSA-8vj2-vxx3-667w", + severities=[ + VulnerabilitySeverity( + system=ScoringSystem( + identifier="generic_textual", + name="Generic textual severity rating", + url="", + notes="Severity for generic scoring systems. Contains generic textual values like High, Low etc", + ), + value="HIGH", + ) + ], + ), + ], + date_published=datetime.datetime(2022, 1, 10, 14, 12, 0, 853348).replace( + tzinfo=datetime.timezone.utc + ), + ), + ] + found_data = [] + for response in mock_response: + found_data.append(parse_advisory_data(response)) + + assert expected_advisories == found_data + + def test_fixed_filter(self): + assert fixed_filter( + {"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "1.6.0"}]} + ) == ["1.6.0"] + assert fixed_filter( + { + "type": "ECOSYSTEM", + "events": [ + {"introduced": "0"}, + {"fixed": "1.0.0"}, + {"introduced": "0"}, + {"fixed": "9.0.0"}, + ], + } + ) == ["1.0.0", "9.0.0"] + assert fixed_filter( + { + "type": "ECOSYSTEM", + "events": [ + {"introduced": "1.5.0"}, + {"fixed": "1.5.0"}, + {"introduced": "4.01"}, + {"fixed": "9.0g0"}, + {"introduced": "8.0.4"}, + {"fixed": "10.8"}, + ], + } + ) == ["1.5.0", "9.0g0", "10.8"] + + def test_get_aliases(self): + assert get_aliases({"id": "GHSA-j3f7-7rmc-6wqj"}) == ["GHSA-j3f7-7rmc-6wqj"] + assert get_aliases({"aliases": ["CVE-2021-40831"]}) == ["CVE-2021-40831"] + self.assertCountEqual( + get_aliases({"aliases": ["CVE-2021-40831"], "id": "GHSA-j3f7-7rmc-6wqj"}), + [ + "CVE-2021-40831", + "GHSA-j3f7-7rmc-6wqj", + ], + ) + self.assertCountEqual( + get_aliases( + {"aliases": ["CVE-2022-22817", "GHSA-8vj2-vxx3-667w"], "id": "GHSA-j3f7-7rmc-6wqj"} + ), + ["CVE-2022-22817", "GHSA-8vj2-vxx3-667w", "GHSA-j3f7-7rmc-6wqj"], + ) + + def test_get_published_date(self): + assert get_published_date( + {"id": "GHSA-j3f7-7rmc-6wqj", "published": "2022-01-10T14:12:00Z"} + ) == datetime.datetime(2022, 1, 10, 14, 12, 0, 0).replace(tzinfo=datetime.timezone.utc) + assert get_published_date( + {"id": "GHSA-j3f7-7rmc-6wqj", "published": "2022-01-10T14:12:00.44444Z"} + ) == datetime.datetime(2022, 1, 10, 14, 12, 0, 444440).replace(tzinfo=datetime.timezone.utc) + assert get_published_date({"id": "GHSA-j3f7-7rmc-6wqj"}) is None + + def test_get_severity(self): + assert get_severity( + {"id": "GHSA-j3f7-7rmc-6wqj", "ecosystem_specific": {"severity": "HIGH"}} + ) == [VulnerabilitySeverity(system=SCORING_SYSTEMS["generic_textual"], value="HIGH")] + assert get_severity({"id": "PYSEC-2022-10", "ecosystem_specific": {}}) == [] + assert get_severity({"id": "PYSEC-2022-10", "database_specific": {"severity": "HIGH"}}) == [ + VulnerabilitySeverity(system=SCORING_SYSTEMS["generic_textual"], value="HIGH") + ] + + assert get_severity( + {"id": "PYSEC-2022-10", "database_specific": {"severity": "MODERATE"}} + ) == [VulnerabilitySeverity(system=SCORING_SYSTEMS["generic_textual"], value="MODERATE")] + assert get_severity({"id": "PYSEC-2022-10", "database_specific": {}}) == [] + assert get_severity({"id": "PYSEC-2022-10"}) == [] + assert get_severity( + { + "id": "PYSEC-2022-10", + "severity": [ + {"type": "CVSS_V3", "score": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N"} + ], + } + ) == [ + VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv3.1_vector"], + value="CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N", + ) + ] + + assert get_severity( + { + "id": "PYSEC-2022-10", + "severity": [ + {"type": "CVSS_V3", "score": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N"}, + {"type": "CVSS_V3", "score": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N"}, + ], + } + ) == [ + VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv3.1_vector"], + value="CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N", + ), + VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv3.1_vector"], + value="CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N", + ), + ] + + def test_get_references(self): + assert get_references( + { + "id": "PYSEC-2022-10", + "references": [ + { + "type": "FIX", + "url": "https://github.com/pikepdf/pikepdf/commit/3f38f73218e5e782fe411ccbb3b44a793c0b343a", + } + ], + }, + [], + ) == [ + Reference( + url="https://github.com/pikepdf/pikepdf/commit/3f38f73218e5e782fe411ccbb3b44a793c0b343a", + severities=[], + ) + ] + + assert get_references( + { + "id": "GHSA-j3f7-7rmc-6wqj", + "references": [ + { + "type": "FIX", + "url": "https://github.com/pikepdf/pikepdf/commit/3f38f73218e5e782fe411ccbb3b44a793c0b343a", + }, + { + "type": "REPORT", + "url": "https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=15499", + }, + ], + }, + [], + ) == [ + Reference( + url="https://github.com/pikepdf/pikepdf/commit/3f38f73218e5e782fe411ccbb3b44a793c0b343a", + severities=[], + ), + Reference( + url="https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=15499", severities=[] + ), + ] + + assert get_references({"id": "PYSEC-2022-10"}, []) == [] + + def test_get_affected_purl(self): + assert get_affected_purl( + {"package": {"purl": "pkg:npm/aws-iot-device-sdk-v2"}}, "PYSEC-2022-10" + ) == PackageURL.from_string("pkg:npm/aws-iot-device-sdk-v2") + + assert get_affected_purl( + {"package": {"name": "aws-iot-device-sdk-v2", "ecosystem": "npm"}}, + "GHSA-j3f7-7rmc-6wqj", + ) == PackageURL(type="npm", name="aws-iot-device-sdk-v2") + + def test_get_affected_version_range(self): + aff_version_range = [ + "0.2.4", + "0.2.9", + "0.3.0", + "1.0.2", + "1.0.3", + "1.0.5", + "1.0.6", + "1.1.0", + "1.2.0", + "1.2.1", + "1.3.0", + "1.3.1", + "1.3.2", + "1.4.0", + "1.5.0", + "1.5.1", + "1.5.10", + "1.5.11", + "1.5.12", + "1.5.13", + "1.5.14", + "1.5.15", + "1.5.16", + "1.5.17", + "1.5.18", + "1.5.2", + "1.5.3", + "1.5.4", + "1.5.5", + "1.5.6", + "1.5.7", + "1.5.8", + "1.6.0", + "1.6.1", + "1.6.2", + ] + assert get_affected_version_range( + {"versions": aff_version_range}, "GHSA-j3f7-7rmc-6wqj" + ) == (PypiVersionRange(aff_version_range)) + + def test_get_fixed_version(self): + assert get_fixed_version({}, "GHSA-j3f7-7rmc-6wqj") == [] + assert get_fixed_version( + {"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": "1.7.0"}]}, + "GHSA-j3f7-7rmc-6wqj", + ) == [PypiVersion("1.7.0")] + assert get_fixed_version( + { + "type": "ECOSYSTEM", + "events": [ + {"introduced": "0"}, + {"fixed": "9.0.0"}, + {"introduced": "0"}, + {"fixed": "9.0.1"}, + ], + }, + "GHSA-j3f7-7rmc-6wqj", + ) == [PypiVersion("9.0.0"), PypiVersion("9.0.1")]