From 7f654f13edc764d7b133b1bb146158a315988dac Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 29 Sep 2023 10:57:11 -0400 Subject: [PATCH] add namespace checking logic Signed-off-by: Alex Goodman --- tests/quality/config.yaml | 98 ++++++++++++++++++++++++++++++++++++++ tests/quality/configure.py | 6 ++- tests/quality/gate.py | 20 +++++++- 3 files changed, 122 insertions(+), 2 deletions(-) diff --git a/tests/quality/config.yaml b/tests/quality/config.yaml index d3ba64cd..e3cf928f 100644 --- a/tests/quality/config.yaml +++ b/tests/quality/config.yaml @@ -50,11 +50,34 @@ tests: images: - docker.io/alpine:3.2@sha256:ddac200f3ebc9902fb8cfcd599f41feb2151f1118929da21bcef57dc276975f9 - docker.io/anchore/test_images:alpine-package-cpe-vuln-match-bd0aaef@sha256:0825acea611c7c5cc792bc7cc20de44d7413fd287dc5afc4aab9c1891d037b4f + expected-namespaces: + - alpine:distro:alpine:3.2 + - alpine:distro:alpine:3.3 + - alpine:distro:alpine:3.4 + - alpine:distro:alpine:3.5 + - alpine:distro:alpine:3.6 + - alpine:distro:alpine:3.7 + - alpine:distro:alpine:3.8 + - alpine:distro:alpine:3.9 + - alpine:distro:alpine:3.10 + - alpine:distro:alpine:3.11 + - alpine:distro:alpine:3.12 + - alpine:distro:alpine:3.13 + - alpine:distro:alpine:3.14 + - alpine:distro:alpine:3.15 + - alpine:distro:alpine:3.16 + - alpine:distro:alpine:3.17 + - alpine:distro:alpine:3.18 + - alpine:distro:alpine:edge - provider: amazon images: - docker.io/amazonlinux:2@sha256:1301cc9f889f21dc45733df9e58034ac1c318202b4b0f0a08d88b3fdc03004de - docker.io/anchore/test_images:vulnerabilities-amazonlinux-2-5c26ce9@sha256:cf742eca189b02902a0a7926ac3fbb423e799937bf4358b0d2acc6cc36ab82aa + expected-namespacges: + - amazon:distro:amazonlinux:2 + - amazon:distro:amazonlinux:2022 + - amazon:distro:amazonlinux:2023 - provider: chainguard additional_providers: @@ -65,6 +88,8 @@ tests: - src/vunnel/providers/wolfi/** images: - ghcr.io/chainguard-images/scanner-test:latest@sha256:59bddc101fba0c45d5c093575c6bc5bfee7f0e46ff127e6bb4e5acaaafb525f9 + expected-namespaces: + - chainguard:distro:chainguard:rolling - provider: debian # ideally we would not use cache, however, the in order to test if we are properly keeping the processing @@ -74,6 +99,15 @@ tests: images: - docker.io/debian:7@sha256:81e88820a7759038ffa61cff59dfcc12d3772c3a2e75b7cfe963c952da2ad264 - docker.io/bitnami/spark:3.2.4-debian-11-r8@sha256:267d5a6345636710b4b57b7fe981c9760203e7e092c705416310ea30a9806d74 + expected-namespaces: + - debian:distro:debian:7 + - debian:distro:debian:8 + - debian:distro:debian:9 + - debian:distro:debian:10 + - debian:distro:debian:11 + - debian:distro:debian:12 + - debian:distro:debian:13 + - debian:distro:debian:unstable - provider: github additional_providers: @@ -98,19 +132,39 @@ tests: - docker.io/anchore/test_images:grype-quality-java-d89207b@sha256:b3534fc2e37943136d5b54e3a58b55d4ccd4363d926cf7aa5bf55a524cf8275b - docker.io/anchore/test_images:grype-quality-golang-d89207b@sha256:7536ee345532f674ec9e448e3768db4e546c48220ba2b6ec9bc9cfbfb3b7b74a - docker.io/anchore/test_images:grype-quality-ruby-d89207b@sha256:1a5a5f870924e88a6f0f2b8089cf276ef0a79b5244a052cdfe4a47bb9e5a2c10 + expected-namespaces: + - github:language:dotnet + - github:language:go + - github:language:java + - github:language:javascript + - github:language:php + - github:language:python + - github:language:ruby + - github:language:rust - provider: mariner images: - mcr.microsoft.com/cbl-mariner/base/core:2.0.20220731-amd64@sha256:3c0f7e103ff3c39e81e7c9c042d2b321d833fb6d26d8636567f7d88a6bdde74a + expected-namespaces: + - mariner:distro:mariner:1.0 + - mariner:distro:mariner:2.0 - provider: nvd images: - docker.io/busybox:1.28.1@sha256:2107a35b58593c58ec5f4e8f2c4a70d195321078aebfadfbfb223a2ff4a4ed21 + expected-namespaces: + - nvd:cpe - provider: oracle images: - docker.io/oraclelinux:6@sha256:a06327c0f1d18d753f2a60bb17864c84a850bb6dcbcf5946dd1a8123f6e75495 - docker.io/anchore/test_images:appstreams-oraclelinux-8-1a287dd@sha256:c8d664b0e728d52f57eeb98ed1899c16d3b265f02ddfb41303d7a16c31e0b0f1 + expected-namespaces: + - oracle:distro:oraclelinux:5 + - oracle:distro:oraclelinux:6 + - oracle:distro:oraclelinux:7 + - oracle:distro:oraclelinux:8 + - oracle:distro:oraclelinux:9 - provider: rhel # ideally we would not use cache, however, the ubuntu provider is currently very expensive to run. @@ -125,11 +179,36 @@ tests: - docker.io/anchore/test_images:appstreams-rhel-8-1a287dd@sha256:524ff8a75f21fd886ec7ed82387766df386671e8b77e898d05786118d5b7880b - docker.io/anchore/test_images:vulnerabilities-centos@sha256:746d31247006cc06434ce91ccf3523b2c230ff6c378ffed7ca1c60bbb48ea86f + expected-namespaces: + - redhat:distro:redhat:5 + - redhat:distro:redhat:6 + - redhat:distro:redhat:7 + - redhat:distro:redhat:8 + - redhat:distro:redhat:9 + # TODO: add sles test images and labels once there is a capability in grype to do so # - providers: # - name: sles # images: # - +# expected-namespaces: +# - sles:distro:sles:11 +# - sles:distro:sles:11.1 +# - sles:distro:sles:11.2 +# - sles:distro:sles:11.3 +# - sles:distro:sles:11.4 +# - sles:distro:sles:12 +# - sles:distro:sles:12.1 +# - sles:distro:sles:12.2 +# - sles:distro:sles:12.3 +# - sles:distro:sles:12.4 +# - sles:distro:sles:12.5 +# - sles:distro:sles:15 +# - sles:distro:sles:15.1 +# - sles:distro:sles:15.2 +# - sles:distro:sles:15.3 +# - sles:distro:sles:15.4 +# - sles:distro:sles:15.5 - provider: ubuntu # ideally we would not use cache, however, the ubuntu provider is currently very expensive to run. @@ -137,6 +216,23 @@ tests: use_cache: true images: - docker.io/ubuntu:16.10@sha256:8dc9652808dc091400d7d5983949043a9f9c7132b15c14814275d25f94bca18a + expected-namespaces: + - ubuntu:distro:ubuntu:12.04 + - ubuntu:distro:ubuntu:14.04 + - ubuntu:distro:ubuntu:16.04 + - ubuntu:distro:ubuntu:16.10 + - ubuntu:distro:ubuntu:17.04 + - ubuntu:distro:ubuntu:17.10 + - ubuntu:distro:ubuntu:18.04 + - ubuntu:distro:ubuntu:18.10 + - ubuntu:distro:ubuntu:19.04 + - ubuntu:distro:ubuntu:19.10 + - ubuntu:distro:ubuntu:20.04 + - ubuntu:distro:ubuntu:20.10 + - ubuntu:distro:ubuntu:21.04 + - ubuntu:distro:ubuntu:21.10 + - ubuntu:distro:ubuntu:22.04 + - ubuntu:distro:ubuntu:22.10 - provider: wolfi additional_providers: @@ -144,3 +240,5 @@ tests: use_cache: true images: - cgr.dev/chainguard/wolfi-base:latest-20221001@sha256:be3834598c3c4b76ace6a866edcbbe1fa18086f9ee238b57769e4d230cd7d507 + expected-namespaces: + - wolfi:distro:wolfi:rolling diff --git a/tests/quality/configure.py b/tests/quality/configure.py index 39a80230..f1d1f30d 100644 --- a/tests/quality/configure.py +++ b/tests/quality/configure.py @@ -56,6 +56,7 @@ class Test: images: list[str] = field(default_factory=list) additional_providers: list[AdditionalProvider] = field(default_factory=list) additional_trigger_globs: list[str] = field(default_factory=list) + expected_namespaces: list[str] = field(default_factory=list) @dataclass @@ -70,7 +71,10 @@ class Config: tests: list[Test] = field(default_factory=list) @classmethod - def load(cls, path: str = "config.yaml") -> "Config": + def load(cls, path: str = "") -> "Config": + if not path: + path = "config.yaml" + try: with open(path, encoding="utf-8") as f: app_object = yaml.safe_load(f.read()) or {} diff --git a/tests/quality/gate.py b/tests/quality/gate.py index b9a01e2e..406aaaf5 100755 --- a/tests/quality/gate.py +++ b/tests/quality/gate.py @@ -10,6 +10,7 @@ from tabulate import tabulate from yardstick import artifact, comparison, store, utils from yardstick.cli import config, display +from .configure import Config, read_config_state # see the .yardstick.yaml configuration for details default_result_set = "pr_vs_latest_via_sbom" @@ -129,7 +130,24 @@ def get_namespaces_from_db() -> list[str]: conn = sqlite3.connect("build/vulnerability.db") c = conn.cursor() c.execute("SELECT DISTINCT namespace FROM vulnerability") - return [row[0] for row in c.fetchall()] + actual_namespaces = [row[0] for row in c.fetchall()] + + # validate that the namespaces we got are 100% what we expect. If there are any missing or extra namespaces we should fail + config = Config.load() + state = read_config_state() + providers = state.cached_providers + state.uncached_providers + + expected_namespaces = [] + for test in config.tests: + if test.provider in providers: + expected_namespaces.extend(test.expected_namespaces) + + extra = set(actual_namespaces) - set(expected_namespaces) + missing = set(expected_namespaces) - set(actual_namespaces) + if extra or missing: + raise RuntimeError(f"mismatched namespaces:\nextra: {extra}\nmissing: {missing}") + + return actual_namespaces def validate(