diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml index c63deb7d..c131f1cc 100644 --- a/.github/actions/bootstrap/action.yaml +++ b/.github/actions/bootstrap/action.yaml @@ -4,15 +4,17 @@ inputs: go-version: description: "Go version to install" required: true - default: "1.19.x" + default: "1.21.x" python-version: description: "Python version to install" required: true - default: "3.9" + # note: the caller really drives this in a matrix strategy run for unit tests. + # this default value is used for the rest of the workflow. + default: "3.11" poetry-version: description: "Poetry version to install" required: true - default: "1.3.2" + default: "1.7.0" use-poetry-cache: description: "Restore poetry cache" required: true @@ -28,7 +30,7 @@ inputs: cache-key-prefix: description: "Prefix all cache keys with this value" required: true - default: "12a830581c" + default: "8381c12a05" bootstrap-apt-packages: description: "Space delimited list of tools to install via apt" default: "" @@ -49,7 +51,7 @@ runs: - name: Install poetry shell: bash run: | - pipx install poetry==${{ inputs.poetry-version }} + python -m pip install poetry==${{ inputs.poetry-version }} poetry self add "poetry-dynamic-versioning[plugin]" - name: Cache Poetry virtualenv @@ -90,3 +92,10 @@ runs: - name: Install dependencies and package shell: bash run: poetry install + + - name: Restore tox cache + id: cache-tox + uses: actions/cache@v3 + with: + path: .tox + key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('tox.ini') }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 55b71af5..bc391f99 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -28,13 +28,22 @@ jobs: git fetch origin main git merge-base --is-ancestor ${GITHUB_REF##*/} origin/main && echo "${GITHUB_REF##*/} is a commit on main!" - - name: Check validation results + - name: Check static analysis results uses: fountainhead/action-wait-for-check@297be350cf8393728ea4d4b39435c7d7ae167c93 #v1.1.1 - id: validations + id: static-analysis with: token: ${{ secrets.GITHUB_TOKEN }} # This check name is defined as the github action job name (in .github/workflows/validations.yaml) - checkName: "Validations" + checkName: "Static Analysis" + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Check test results + uses: fountainhead/action-wait-for-check@297be350cf8393728ea4d4b39435c7d7ae167c93 #v1.1.1 + id: test + with: + token: ${{ secrets.GITHUB_TOKEN }} + # This check name is defined as the github action job name (in .github/workflows/validations.yaml) + checkName: "Test Gate" ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Check nightly quality gate results @@ -50,9 +59,10 @@ jobs: intervalSeconds: 3 - name: Release quality gate - if: steps.validations.conclusion != 'success' || steps.nightly-quality-gate.conclusion != 'success' + if: steps.static-analysis.conclusion != 'success' || steps.test.conclusion != 'success' || steps.nightly-quality-gate.conclusion != 'success' run: | - echo "Validations Status: ${{ steps.validations.conclusion }}" + echo "Static Analysis Status: ${{ steps.static-analysis.conclusion }}" + echo "Test Status: ${{ steps.test.conclusion }}" echo "Nightly Quality Gate Status: ${{ steps.nightly-quality-gate.conclusion }}" false diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index 5d7047f9..b5d7e196 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -11,7 +11,8 @@ on: jobs: # note: the name for this check is referenced in release.yaml, do not change here without changing there - Validations: + Static-Analysis: + name: "Static Analysis" runs-on: ubuntu-22.04 permissions: contents: read @@ -27,19 +28,63 @@ jobs: - name: Run static analysis run: poetry run make static-analysis - - name: Run unit tests - run: poetry run make unit - - name: Ensure quality gate tools are properly configured run: | cd tests/quality && make validate-test-tool-versions + Test: + runs-on: ubuntu-22.04 + permissions: + contents: read + strategy: + matrix: + # note: this is not a single source of truth (this is also in the tox.ini) + python: + - version: '3.9' + toxEnv: py39 + + - version: '3.10' + toxEnv: py310 + + - version: '3.11' + toxEnv: py311 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1 + with: + # in order to properly resolve the version from git + fetch-depth: 0 + + - name: Bootstrap environment + uses: ./.github/actions/bootstrap + with: + python-version: ${{ matrix.python.version }} + + - name: Run unit tests + run: poetry run tox -e ${{ matrix.python.toxEnv }} + - name: Build assets run: poetry run make build + # this is to help facilitate ensuring all checks have run with the checks API for release + # see https://github.com/orgs/community/discussions/26822#discussioncomment-3305794 + # as well as the release.yaml workflow + Test-Gate: + if: ${{ always() }} + runs-on: ubuntu-22.04 + name: Test Gate + needs: [test] + steps: + - run: | + result="${{ needs.Test.result }}" + if [[ $result == "success" || $result == "skipped" ]]; then + exit 0 + else + exit 1 + fi + Publish-PreProd: runs-on: ubuntu-22.04 - needs: [Validations] + needs: [Static-Analysis, Test] if: github.ref == 'refs/heads/main' permissions: contents: read diff --git a/Makefile b/Makefile index afe39bf1..b3a1d919 100644 --- a/Makefile +++ b/Makefile @@ -123,6 +123,10 @@ check-types: virtual-env-check ## Run type checks (mypy) unit: virtual-env-check ## Run unit tests pytest --cov-report html --cov vunnel -v tests/unit/ +.PHONY: unit-matrix +unit-matrix: virtual-env-check ## Run unit tests for all supported python versions + tox + ## Build-related targets ################################# diff --git a/poetry.lock b/poetry.lock index 4d0f56aa..8831135e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "attrs" @@ -60,6 +60,17 @@ d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + [[package]] name = "certifi" version = "2023.7.22" @@ -82,6 +93,17 @@ files = [ {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, ] +[[package]] +name = "chardet" +version = "5.2.0" +description = "Universal encoding detector for Python 3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, + {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, +] + [[package]] name = "charset-normalizer" version = "3.2.0" @@ -927,6 +949,26 @@ docs = ["alabaster (==0.7.13)", "autodocsumm (==0.2.11)", "sphinx (==7.0.1)", "s lint = ["flake8 (==6.0.0)", "flake8-bugbear (==23.7.10)", "mypy (==1.4.1)", "pre-commit (>=2.4,<4.0)"] tests = ["pytest", "pytz", "simplejson"] +[[package]] +name = "mashumaro" +version = "3.10" +description = "Fast serialization library on top of dataclasses" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mashumaro-3.10-py3-none-any.whl", hash = "sha256:d2c0fd5e7878987629d41f4986d9b0903d362a92eeb299b7d88b87eb113f4f48"}, + {file = "mashumaro-3.10.tar.gz", hash = "sha256:0248a5c8574aa6cd20696621502d38a7ea66af3d6d93c5d03f93b33298edc878"}, +] + +[package.dependencies] +typing-extensions = ">=4.1.0" + +[package.extras] +msgpack = ["msgpack (>=0.5.6)"] +orjson = ["orjson"] +toml = ["tomli (>=1.1.0)", "tomli-w (>=1.0)"] +yaml = ["pyyaml (>=3.13)"] + [[package]] name = "mdurl" version = "0.1.2" @@ -1198,6 +1240,25 @@ files = [ [package.extras] plugins = ["importlib-metadata"] +[[package]] +name = "pyproject-api" +version = "1.6.1" +description = "API to interact with the python pyproject.toml based projects" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyproject_api-1.6.1-py3-none-any.whl", hash = "sha256:4c0116d60476b0786c88692cf4e325a9814965e2469c5998b830bba16b183675"}, + {file = "pyproject_api-1.6.1.tar.gz", hash = "sha256:1817dc018adc0d1ff9ca1ed8c60e1623d5aaca40814b953af14a9cf9a5cae538"}, +] + +[package.dependencies] +packaging = ">=23.1" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +docs = ["furo (>=2023.8.19)", "sphinx (<7.2)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "setuptools (>=68.1.2)", "wheel (>=0.41.2)"] + [[package]] name = "pytest" version = "7.4.3" @@ -1712,7 +1773,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\")"} [package.extras] aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] @@ -1785,6 +1846,33 @@ files = [ {file = "toposort-1.10.tar.gz", hash = "sha256:bfbb479c53d0a696ea7402601f4e693c97b0367837c8898bc6471adfca37a6bd"}, ] +[[package]] +name = "tox" +version = "4.11.3" +description = "tox is a generic virtualenv management and test command line tool" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tox-4.11.3-py3-none-any.whl", hash = "sha256:599af5e5bb0cad0148ac1558a0b66f8fff219ef88363483b8d92a81e4246f28f"}, + {file = "tox-4.11.3.tar.gz", hash = "sha256:5039f68276461fae6a9452a3b2c7295798f00a0e92edcd9a3b78ba1a73577951"}, +] + +[package.dependencies] +cachetools = ">=5.3.1" +chardet = ">=5.2" +colorama = ">=0.4.6" +filelock = ">=3.12.3" +packaging = ">=23.1" +platformdirs = ">=3.10" +pluggy = ">=1.3" +pyproject-api = ">=1.6.1" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} +virtualenv = ">=20.24.3" + +[package.extras] +docs = ["furo (>=2023.8.19)", "sphinx (>=7.2.4)", "sphinx-argparse-cli (>=1.11.1)", "sphinx-autodoc-typehints (>=1.24)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +testing = ["build[virtualenv] (>=0.10)", "covdefaults (>=2.3)", "detect-test-pollution (>=1.1.1)", "devpi-process (>=1)", "diff-cover (>=7.7)", "distlib (>=0.3.7)", "flaky (>=3.7)", "hatch-vcs (>=0.3)", "hatchling (>=1.18)", "psutil (>=5.9.5)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-xdist (>=3.3.1)", "re-assert (>=1.1)", "time-machine (>=2.12)", "wheel (>=0.41.2)"] + [[package]] name = "types-pyyaml" version = "6.0.12.12" @@ -2088,4 +2176,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "a116944dd8bc4d0165191d943c8d4a4edaded4036ab64d15b717f09dabfec4eb" +content-hash = "16310a7ed142ac1cfc2a264b5d40ba92ea03d45a66c95edc4554f7f45e99a3ff" diff --git a/pyproject.toml b/pyproject.toml index 65552b3b..b4768449 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,6 @@ xxhash = "^3.1.0" cvss = "^2.6" python-dateutil = "^2.8.2" defusedxml = "^0.7.1" -dataclass-wizard = "^0.22.2" orjson = "^3.8.6" SQLAlchemy = ">= 1.4.46, < 2.0" # note: 1.4.x currently required for enterprise mergedeep = "^1.3.4" @@ -59,6 +58,7 @@ future = "^0.18.3" importlib-metadata = "^6.1.0" xsdata = {extras = ["cli", "lxml", "soap"], version = ">=22.12,<24.0"} pytest-snapshot = "^0.9.0" +mashumaro = "^3.10" [tool.poetry.group.dev.dependencies] pytest = "^7.2.2" @@ -80,6 +80,7 @@ dunamai = "^1.15.0" ruff = ">=0.0.254,<0.1.5" yardstick = {git = "https://github.com/anchore/yardstick", rev = "v0.9.1"} tabulate = "0.9.0" +tox = "^4.11.3" [build-system] requires = ["poetry-core>=1.3.0", "poetry-dynamic-versioning"] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..37f72ea0 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = -vv --color=yes diff --git a/src/vunnel/cli/config.py b/src/vunnel/cli/config.py index a66d8c30..db208f7f 100644 --- a/src/vunnel/cli/config.py +++ b/src/vunnel/cli/config.py @@ -6,7 +6,7 @@ import mergedeep import yaml -from dataclass_wizard import asdict, fromdict +from mashumaro.mixins.dict import DataClassDictMixin from vunnel import providers @@ -49,7 +49,7 @@ def __post_init__(self) -> None: @dataclass -class Application: +class Application(DataClassDictMixin): root: str = "./data" log: Log = field(default_factory=Log) providers: Providers = field(default_factory=Providers) @@ -60,18 +60,15 @@ def load(path: str = ".vunnel.yaml") -> Application: with open(path, encoding="utf-8") as f: app_object = yaml.safe_load(f.read()) or {} # we need a full default application config first then merge the loaded config on top. - # Why? dataclass_wizard.fromdict() will create instances from the dataclass default + # Why? cls.from_dict() will create instances from the dataclass default # and NOT the field definition from the container. So it is possible to specify a # single field in the config and all other fields would be set to the default value # based on the dataclass definition and not any field(default_factory=...) hints # from the containing class. - instance = asdict(Application()) + instance = Application().to_dict() mergedeep.merge(instance, app_object) - cfg = fromdict( - Application, - instance, - ) + cfg = Application.from_dict(instance) if cfg is None: raise FileNotFoundError("parsed empty config") except FileNotFoundError: diff --git a/src/vunnel/providers/amazon/__init__.py b/src/vunnel/providers/amazon/__init__.py index 26322802..dc0c0933 100644 --- a/src/vunnel/providers/amazon/__init__.py +++ b/src/vunnel/providers/amazon/__init__.py @@ -14,7 +14,7 @@ @dataclass class Config: - security_advisories: dict[Any, str] = field(default_factory=lambda: amazon_security_advisories) + security_advisories: dict[Any, str] = field(default_factory=lambda: amazon_security_advisories.copy()) runtime: provider.RuntimeConfig = field( default_factory=lambda: provider.RuntimeConfig( result_store=result.StoreStrategy.SQLITE, diff --git a/src/vunnel/workspace.py b/src/vunnel/workspace.py index 649010d2..0ee80d46 100644 --- a/src/vunnel/workspace.py +++ b/src/vunnel/workspace.py @@ -7,11 +7,11 @@ import shutil import sqlite3 from dataclasses import asdict, dataclass, field -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Optional import orjson import xxhash -from dataclass_wizard import fromdict +from mashumaro.mixins.dict import DataClassDictMixin from vunnel import schema as schemaDef from vunnel import utils @@ -31,23 +31,20 @@ class File: @dataclass -class State: +class State(DataClassDictMixin): provider: str urls: list[str] store: str timestamp: datetime.datetime version: int = 1 - listing: File | None = None + listing: Optional[File] = None # noqa:UP007 # why use Optional? mashumaro does not support this on python 3.9 schema: schemaDef.Schema = field(default_factory=schemaDef.ProviderStateSchema) @staticmethod def read(root: str) -> State: metadata_path = os.path.join(root, METADATA_FILENAME) with open(metadata_path, encoding="utf-8") as f: - return fromdict( - State, - orjson.loads(f.read()), - ) + return State.from_dict(orjson.loads(f.read())) def write(self, root: str, results: str, update_listing: bool = True) -> str: metadata_path = os.path.join(root, METADATA_FILENAME) diff --git a/tests/quality/configure.py b/tests/quality/configure.py index 05334ca3..83fab77b 100644 --- a/tests/quality/configure.py +++ b/tests/quality/configure.py @@ -18,21 +18,25 @@ import mergedeep import requests import yaml -from dataclass_wizard import DumpMeta, asdict, fromdict +from mashumaro.mixins.dict import DataClassDictMixin from yardstick.cli.config import ( - Application, ResultSet, ScanMatrix, Tool, ) +from yardstick.cli.config import Application as YardstickApplication BIN_DIR = "./bin" CLONE_DIR = f"{BIN_DIR}/grype-db-src" GRYPE_DB = f"{BIN_DIR}/grype-db" +class Application(YardstickApplication, DataClassDictMixin): + pass + + @dataclass -class ConfigurationState: +class ConfigurationState(DataClassDictMixin): uncached_providers: list[str] = field(default_factory=list) cached_providers: list[str] = field(default_factory=list) @@ -65,7 +69,7 @@ class GrypeDB: @dataclass -class Config: +class Config(DataClassDictMixin): yardstick: Yardstick = field(default_factory=Yardstick) grype_db: GrypeDB = field(default_factory=GrypeDB) tests: list[Test] = field(default_factory=list) @@ -79,18 +83,15 @@ def load(cls, path: str = "") -> "Config": with open(path, encoding="utf-8") as f: app_object = yaml.safe_load(f.read()) or {} # we need a full default application config first then merge the loaded config on top. - # Why? dataclass_wizard.fromdict() will create instances from the dataclass default + # Why? cls.from_dict() will create instances from the dataclass default # and NOT the field definition from the container. So it is possible to specify a # single field in the config and all other fields would be set to the default value # based on the dataclass definition and not any field(default_factory=...) hints # from the containing class. - instance = asdict(cls()) + instance = cls().to_dict() mergedeep.merge(instance, app_object) - cfg = fromdict( - cls, - instance, - ) + cfg = cls.from_dict(instance) if cfg is None: raise FileNotFoundError("parsed empty config") except FileNotFoundError: @@ -262,7 +263,7 @@ def write_config_state(cached_providers: list[str], uncached_providers: list[str logging.info(f"writing configuration state to {path!r}") with open(path, "w") as f: - f.write(yaml.dump(asdict(ConfigurationState(cached_providers=cached_providers, uncached_providers=uncached_providers)))) + f.write(yaml.dump(ConfigurationState(cached_providers=cached_providers, uncached_providers=uncached_providers).to_dict())) def read_config_state(path: str = ".state.yaml"): @@ -270,7 +271,7 @@ def read_config_state(path: str = ".state.yaml"): try: with open(path) as f: - return fromdict(ConfigurationState, yaml.safe_load(f.read())) + return ConfigurationState.from_dict(yaml.safe_load(f.read())) except FileNotFoundError: return ConfigurationState() @@ -278,10 +279,8 @@ def read_config_state(path: str = ".state.yaml"): def write_yardstick_config(cfg: Application, path: str = ".yardstick.yaml"): logging.info(f"writing yardstick config to {path!r}") - DumpMeta(key_transform="SNAKE", skip_defaults=True).bind_to(Application) - with open(path, "w") as f: - f.write(yaml.dump(asdict(cfg))) + f.write(yaml.dump(cfg.to_dict())) def write_grype_db_config(providers: set[str], path: str = ".grype-db.yaml"): diff --git a/tests/unit/cli/test_config.py b/tests/unit/cli/test_config.py index 2912afd0..e240fb37 100644 --- a/tests/unit/cli/test_config.py +++ b/tests/unit/cli/test_config.py @@ -41,7 +41,14 @@ def test_full_config(helpers): request_timeout=20, ), amazon=providers.amazon.Config( - security_advisories={42: "https://alas.aws.amazon.com/AL2/alas-42.rss"}, # cap sensitive! + security_advisories={ + # this is what we added in the config + 42: "https://alas.aws.amazon.com/AL2/alas-42.rss", + # this is the defaults... + 2: "https://alas.aws.amazon.com/AL2/alas.rss", + 2022: "https://alas.aws.amazon.com/AL2022/alas.rss", + 2023: "https://alas.aws.amazon.com/AL2023/alas.rss", + }, runtime=runtime_cfg, request_timeout=20, ), @@ -51,6 +58,7 @@ def test_full_config(helpers): ), debian=providers.debian.Config( releases={ + # this is the defaults... "trixie": "13", "bookworm": "12", "bullseye": "11", @@ -59,6 +67,7 @@ def test_full_config(helpers): "jessie": "8", "wheezy": "7", "sid": "unstable", + # this is what we added in the config "jinx": 87, }, runtime=runtime_cfg, diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..7944ae11 --- /dev/null +++ b/tox.ini @@ -0,0 +1,12 @@ +[tox] +# note: this is not a single source of truth (this is also in the .github/workflows/valiations.yml file) +envlist = py39, py310, py311 +isolated_build = True + +[testenv] +allowlist_externals = poetry +skip_install = true + +commands = + poetry install -vvv + poetry run pytest --cov-report html --cov vunnel -v tests/unit/