From 0d432576879f9992338d6b35d085440379fbf75d Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 23 Oct 2023 11:44:15 -0400 Subject: [PATCH 1/2] add basic namespace check Signed-off-by: Alex Goodman --- manager/src/grype_db_manager/cli/db.py | 7 +- manager/src/grype_db_manager/grypedb.py | 232 ++++++++++++++++++++++++ manager/tests/unit/test_grypedb.py | 45 +++++ 3 files changed, 283 insertions(+), 1 deletion(-) diff --git a/manager/src/grype_db_manager/cli/db.py b/manager/src/grype_db_manager/cli/db.py index ecdbc9c1..545780c5 100644 --- a/manager/src/grype_db_manager/cli/db.py +++ b/manager/src/grype_db_manager/cli/db.py @@ -92,9 +92,10 @@ def show_db(cfg: config.Application, db_uuid: str) -> None: ) @click.option("--verbose", "-v", "verbosity", count=True, help="show details of all comparisons") @click.option("--recapture", "-r", is_flag=True, help="recapture grype results (even if not stale)") +@click.option("--skip-namespace-check", "skip_namespace_check", is_flag=True, help="do not ensure the minimum expected namespaces are present") @click.argument("db-uuid") @click.pass_obj -def validate_db(cfg: config.Application, db_uuid: str, images: list[str], verbosity: int, recapture: bool) -> None: +def validate_db(cfg: config.Application, db_uuid: str, images: list[str], verbosity: int, recapture: bool, skip_namespace_check: bool) -> None: logging.info(f"validating DB {db_uuid}") if not images: @@ -107,6 +108,10 @@ def validate_db(cfg: config.Application, db_uuid: str, images: list[str], verbos click.echo(f"no database found with session id {db_uuid}") return + if not skip_namespace_check: + # ensure the minimum number of namespaces are present + db_manager.validate_namespaces(db_uuid=db_uuid) + # resolve tool versions and install them yardstick.store.config.set_values(store_root=cfg.data.yardstick_root) diff --git a/manager/src/grype_db_manager/grypedb.py b/manager/src/grype_db_manager/grypedb.py index 51edbb8e..c0acbe28 100644 --- a/manager/src/grype_db_manager/grypedb.py +++ b/manager/src/grype_db_manager/grypedb.py @@ -9,6 +9,7 @@ import re import shlex import shutil +import sqlite3 import subprocess import sys import uuid @@ -25,6 +26,206 @@ # TODO: # - add tests for GrypeDB.install* +# these are the minimum expected namespaces that should be present in the DB based on the v4+ schema. +# TODO: ideally this would be coupled to the definitions defined in the vunnel quality gate config file +# https://github.com/anchore/vunnel/blob/v0.17.2/tests/quality/config.yaml#L53 +# however, its important to use the file for the same version of vunnel used by grype-db to build the DB, which +# isn't always possible to know. Ideally this version info would be captured in the vunnel data directory directly. +# For the meantime this is a snapshot of the expected namespaces for vunnel 0.17.2 in Oct 2023 (boo! 👻). +expected_namespaces = [ + "alpine:distro:alpine:3.10", + "alpine:distro:alpine:3.11", + "alpine:distro:alpine:3.12", + "alpine:distro:alpine:3.13", + "alpine:distro:alpine:3.14", + "alpine:distro:alpine:3.15", + "alpine:distro:alpine:3.16", + "alpine:distro:alpine:3.17", + "alpine:distro:alpine:3.18", + "alpine:distro:alpine:3.2", + "alpine:distro:alpine:3.3", + "alpine:distro:alpine:3.4", + "alpine:distro:alpine:3.5", + "alpine:distro:alpine:3.6", + "alpine:distro:alpine:3.7", + "alpine:distro:alpine:3.8", + "alpine:distro:alpine:3.9", + "alpine:distro:alpine:edge", + "amazon:distro:amazonlinux:2", + "amazon:distro:amazonlinux:2022", + "amazon:distro:amazonlinux:2023", + "chainguard:distro:chainguard:rolling", + "debian:distro:debian:10", + "debian:distro:debian:11", + "debian:distro:debian:12", + "debian:distro:debian:13", + "debian:distro:debian:7", + "debian:distro:debian:8", + "debian:distro:debian:9", + "debian:distro:debian:unstable", + "github:language:dart", + "github:language:dotnet", + "github:language:go", + "github:language:java", + "github:language:javascript", + "github:language:php", + "github:language:python", + "github:language:ruby", + "github:language:rust", + "github:language:swift", + "mariner:distro:mariner:1.0", + "mariner:distro:mariner:2.0", + "nvd:cpe", + "oracle:distro:oraclelinux:5", + "oracle:distro:oraclelinux:6", + "oracle:distro:oraclelinux:7", + "oracle:distro:oraclelinux:8", + "oracle:distro:oraclelinux:9", + "redhat:distro:redhat:5", + "redhat:distro:redhat:6", + "redhat:distro:redhat:7", + "redhat:distro:redhat:8", + "redhat:distro:redhat:9", + "sles:distro:sles:11", + "sles:distro:sles:11.1", + "sles:distro:sles:11.2", + "sles:distro:sles:11.3", + "sles:distro:sles:11.4", + "sles:distro:sles:12", + "sles:distro:sles:12.1", + "sles:distro:sles:12.2", + "sles:distro:sles:12.3", + "sles:distro:sles:12.4", + "sles:distro:sles:12.5", + "sles:distro:sles:15", + "sles:distro:sles:15.1", + "sles:distro:sles:15.2", + "sles:distro:sles:15.3", + "sles:distro:sles:15.4", + "sles:distro:sles:15.5", + "ubuntu:distro:ubuntu:12.04", + "ubuntu:distro:ubuntu:12.10", + "ubuntu:distro:ubuntu:13.04", + "ubuntu:distro:ubuntu:14.04", + "ubuntu:distro:ubuntu:14.10", + "ubuntu:distro:ubuntu:15.04", + "ubuntu:distro:ubuntu:15.10", + "ubuntu:distro:ubuntu:16.04", + "ubuntu:distro:ubuntu:16.10", + "ubuntu:distro:ubuntu:17.04", + "ubuntu:distro:ubuntu:17.10", + "ubuntu:distro:ubuntu:18.04", + "ubuntu:distro:ubuntu:18.10", + "ubuntu:distro:ubuntu:19.04", + "ubuntu:distro:ubuntu:19.10", + "ubuntu:distro:ubuntu:20.04", + "ubuntu:distro:ubuntu:20.10", + "ubuntu:distro:ubuntu:21.04", + "ubuntu:distro:ubuntu:21.10", + "ubuntu:distro:ubuntu:22.04", + "ubuntu:distro:ubuntu:22.10", + "ubuntu:distro:ubuntu:23.04", + "ubuntu:distro:ubuntu:23.10", + "wolfi:distro:wolfi:rolling", +] + +v3_expected_namespaces = [ + "alpine:3.10", + "alpine:3.11", + "alpine:3.12", + "alpine:3.13", + "alpine:3.14", + "alpine:3.15", + "alpine:3.16", + "alpine:3.17", + "alpine:3.18", + "alpine:3.2", + "alpine:3.3", + "alpine:3.4", + "alpine:3.5", + "alpine:3.6", + "alpine:3.7", + "alpine:3.8", + "alpine:3.9", + "alpine:edge", + "amzn:2", + "amzn:2022", + "amzn:2023", + "chainguard:rolling", + "debian:10", + "debian:11", + "debian:12", + "debian:13", + "debian:7", + "debian:8", + "debian:9", + "debian:unstable", + "github:composer", + "github:dart", + "github:gem", + "github:go", + "github:java", + "github:npm", + "github:nuget", + "github:python", + "github:rust", + "github:swift", + "mariner:1.0", + "mariner:2.0", + "nvd", + "ol:5", + "ol:6", + "ol:7", + "ol:8", + "ol:9", + "rhel:5", + "rhel:6", + "rhel:7", + "rhel:8", + "rhel:9", + "sles:11", + "sles:11.1", + "sles:11.2", + "sles:11.3", + "sles:11.4", + "sles:12", + "sles:12.1", + "sles:12.2", + "sles:12.3", + "sles:12.4", + "sles:12.5", + "sles:15", + "sles:15.1", + "sles:15.2", + "sles:15.3", + "sles:15.4", + "sles:15.5", + "ubuntu:12.04", + "ubuntu:12.10", + "ubuntu:13.04", + "ubuntu:14.04", + "ubuntu:14.10", + "ubuntu:15.04", + "ubuntu:15.10", + "ubuntu:16.04", + "ubuntu:16.10", + "ubuntu:17.04", + "ubuntu:17.10", + "ubuntu:18.04", + "ubuntu:18.10", + "ubuntu:19.04", + "ubuntu:19.10", + "ubuntu:20.04", + "ubuntu:20.10", + "ubuntu:21.04", + "ubuntu:21.10", + "ubuntu:22.04", + "ubuntu:22.10", + "ubuntu:23.04", + "ubuntu:23.10", + "wolfi:rolling", +] + @dataclasses.dataclass class DBInfo: @@ -40,6 +241,10 @@ class DBInvalidException(Exception): pass +class DBNamespaceException(Exception): + pass + + class DBManager: def __init__(self, root_dir: str): self.db_dir = os.path.join(root_dir, DB_DIR) @@ -65,6 +270,33 @@ def new_session(self) -> str: return db_uuid + def list_namespaces(self, db_uuid: str) -> list[str]: + _, build_dir = self.db_paths(db_uuid=db_uuid) + # a sqlite3 db + db_path = os.path.join(build_dir, "vulnerability.db") + + # select distinct values in the "namespace" column of the "vulnerability" table + con = sqlite3.connect(db_path) + crsr = con.cursor() + crsr.execute("SELECT DISTINCT namespace FROM vulnerability") + result = crsr.fetchall() + con.close() + + return sorted([r[0] for r in result]) + + def validate_namespaces(self, db_uuid: str) -> None: + db_info = self.get_db_info(db_uuid) + + expected = v3_expected_namespaces if db_info.schema_version <= 3 else expected_namespaces + + missing_namespaces = set(expected) - set(self.list_namespaces(db_uuid=db_uuid)) + + if missing_namespaces: + msg = f"missing namespaces in DB {db_uuid!r}: {sorted(missing_namespaces)!r}" + raise DBNamespaceException(msg) + + logging.info(f"minimum expected namespaces present in {db_uuid!r}") + def get_db_info(self, db_uuid: str) -> DBInfo | None: session_dir = os.path.join(self.db_dir, db_uuid) if not os.path.exists(session_dir): diff --git a/manager/tests/unit/test_grypedb.py b/manager/tests/unit/test_grypedb.py index 99193e9f..7c52c610 100644 --- a/manager/tests/unit/test_grypedb.py +++ b/manager/tests/unit/test_grypedb.py @@ -2,6 +2,8 @@ import datetime import pathlib +import pytest + from grype_db_manager import grypedb @@ -63,6 +65,49 @@ def test_new_session(self, tmp_path: pathlib.Path): assert v assert v.year == datetime.datetime.now().year + @pytest.mark.parametrize( + "listed_namespaces, schema_version, expect_error", + [ + pytest.param([], 5, True, id="empty"), + pytest.param(["namespace1"], 5, True, id="too few namespaces"), + pytest.param(grypedb.expected_namespaces, 5, False, id="v5 matches"), + pytest.param(grypedb.expected_namespaces + ["extra_items"], 5, False, id="v5 with extra items"), + pytest.param(list(grypedb.expected_namespaces)[:-5], 5, True, id="v5 missing items"), + pytest.param(grypedb.v3_expected_namespaces, 3, False, id="v3 matches"), + pytest.param(grypedb.v3_expected_namespaces + ["extra_items"], 3, False, id="v3 with extra items"), + pytest.param(list(grypedb.v3_expected_namespaces)[:-5], 3, True, id="v3 missing items"), + ], + ) + def test_validate_namespaces(self, tmp_path: pathlib.Path, mocker, schema_version, listed_namespaces, expect_error): + assert len(grypedb.expected_namespaces) > 0 + + dbm = grypedb.DBManager(root_dir=tmp_path.as_posix()) + session_id = dbm.new_session() + + # patch list_namespaces to return a mock + dbm.list_namespaces = mocker.MagicMock() + dbm.list_namespaces.return_value = listed_namespaces + + # patch db_info to return a mock + dbm.get_db_info = mocker.MagicMock() + dbm.get_db_info.return_value = grypedb.DBInfo( + uuid="", + schema_version=schema_version, + db_checksum="", + db_created="", + data_created="", + archive_path="", + ) + + if expect_error: + with pytest.raises(grypedb.DBNamespaceException): + dbm.validate_namespaces(session_id) + else: + dbm.validate_namespaces(session_id) + + dbm.list_namespaces.assert_called_once() + dbm.get_db_info.assert_called_once() + class TestGrypeDB: def test_list_installed(self, top_level_fixture): From c497aef81ff244e6ebbdc33ef8c76174fb08fc6f Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 23 Oct 2023 16:14:34 -0400 Subject: [PATCH 2/2] fix tests Signed-off-by: Alex Goodman --- manager/src/grype_db_manager/cli/db.py | 25 +++++++++++++++++--- manager/tests/cli/workflow-2-validate-db.sh | 19 ++++++++++++--- manager/tests/cli/workflow-4-full-publish.sh | 4 ++-- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/manager/src/grype_db_manager/cli/db.py b/manager/src/grype_db_manager/cli/db.py index 545780c5..c8644d53 100644 --- a/manager/src/grype_db_manager/cli/db.py +++ b/manager/src/grype_db_manager/cli/db.py @@ -92,10 +92,22 @@ def show_db(cfg: config.Application, db_uuid: str) -> None: ) @click.option("--verbose", "-v", "verbosity", count=True, help="show details of all comparisons") @click.option("--recapture", "-r", is_flag=True, help="recapture grype results (even if not stale)") -@click.option("--skip-namespace-check", "skip_namespace_check", is_flag=True, help="do not ensure the minimum expected namespaces are present") +@click.option( + "--skip-namespace-check", + "skip_namespace_check", + is_flag=True, + help="do not ensure the minimum expected namespaces are present", +) @click.argument("db-uuid") @click.pass_obj -def validate_db(cfg: config.Application, db_uuid: str, images: list[str], verbosity: int, recapture: bool, skip_namespace_check: bool) -> None: +def validate_db( + cfg: config.Application, + db_uuid: str, + images: list[str], + verbosity: int, + recapture: bool, + skip_namespace_check: bool, +) -> None: logging.info(f"validating DB {db_uuid}") if not images: @@ -203,6 +215,12 @@ def upload_db(cfg: config.Application, db_uuid: str, ttl_seconds: int) -> None: @click.option("--schema-version", "-s", required=True, help="the DB schema version to build, validate, and upload") @click.option("--dry-run", "-d", is_flag=True, help="do not upload the DB to S3") @click.option("--skip-validate", is_flag=True, help="skip validation of the DB") +@click.option( + "--skip-namespace-check", + "skip_namespace_check", + is_flag=True, + help="do not ensure the minimum expected namespaces are present", +) @click.option("--verbose", "-v", "verbosity", count=True, help="show details of all comparisons") @click.pass_obj @click.pass_context @@ -212,6 +230,7 @@ def build_and_upload_db( cfg: config.Application, schema_version: str, skip_validate: bool, + skip_namespace_check: bool, dry_run: bool, verbosity: bool, ) -> None: @@ -227,7 +246,7 @@ def build_and_upload_db( click.echo(f"{Format.ITALIC}Skipping validation of DB {db_uuid!r}{Format.RESET}") else: click.echo(f"{Format.BOLD}Validating DB {db_uuid!r}{Format.RESET}") - ctx.invoke(validate_db, db_uuid=db_uuid, verbosity=verbosity) + ctx.invoke(validate_db, db_uuid=db_uuid, verbosity=verbosity, skip_namespace_check=skip_namespace_check) if not dry_run: click.echo(f"{Format.BOLD}Uploading DB {db_uuid!r}{Format.RESET}") diff --git a/manager/tests/cli/workflow-2-validate-db.sh b/manager/tests/cli/workflow-2-validate-db.sh index 41b81b13..377a7d90 100755 --- a/manager/tests/cli/workflow-2-validate-db.sh +++ b/manager/tests/cli/workflow-2-validate-db.sh @@ -20,12 +20,25 @@ header "Case 1: fail DB validation (too many unknowns)" make clean-yardstick-labels -run_expect_fail grype-db-manager db validate $DB_ID -vvv +run_expect_fail grype-db-manager db validate $DB_ID -vvv --skip-namespace-check assert_contains $(last_stderr_file) "current indeterminate matches % is greater than 10.0%" ############################################# -header "Case 2: pass DB validation" +header "Case 2: fail DB validation (missing namespaces)" + +make clean-yardstick-labels +echo "installing labels" +# use the real labels +cp -a ../../../data/vulnerability-match-labels/labels/docker.io+oraclelinux* ./cli-test-data/yardstick/labels/ +tree ./cli-test-data/yardstick/labels/ + +run_expect_fail grype-db-manager db validate $DB_ID -vvv +assert_contains $(last_stderr_file) "missing namespaces in DB" + + +############################################# +header "Case 3: pass DB validation" make clean-yardstick-labels echo "installing labels" @@ -33,7 +46,7 @@ echo "installing labels" cp -a ../../../data/vulnerability-match-labels/labels/docker.io+oraclelinux* ./cli-test-data/yardstick/labels/ tree ./cli-test-data/yardstick/labels/ -run grype-db-manager db validate $DB_ID -vvv +run grype-db-manager db validate $DB_ID -vvv --skip-namespace-check assert_contains $(last_stdout_file) "Validation passed" diff --git a/manager/tests/cli/workflow-4-full-publish.sh b/manager/tests/cli/workflow-4-full-publish.sh index 9ae7fa1f..b5e8ff38 100755 --- a/manager/tests/cli/workflow-4-full-publish.sh +++ b/manager/tests/cli/workflow-4-full-publish.sh @@ -42,10 +42,10 @@ header "Case 1: create and publish a DB" # note: this test is exercising the following commands: # grype-db-manager db build -# grype-db-manager db validate +# grype-db-manager db validate --skip-namespace-check # grype-db-manager db upload -run grype-db-manager db build-and-upload --schema-version $SCHEMA_VERSION +run grype-db-manager db build-and-upload --schema-version $SCHEMA_VERSION --skip-namespace-check assert_contains $(last_stdout_file) "Validation passed" assert_contains $(last_stdout_file) "' uploaded to s3://testbucket/grype/databases"