From 01ebd6aa187f9e091462a29996629f66f104767d Mon Sep 17 00:00:00 2001 From: Jannis Mittenzwei Date: Wed, 18 Sep 2024 11:34:55 +0200 Subject: [PATCH] formating --- .../__init__.py | 6 +- .../cli/commands/run_ci.py | 76 +++++---- .../cli/commands/run_release.py | 93 ++++++----- .../lib/asset_uploader.py | 36 +++-- .../lib/branch_config.py | 44 ++++-- .../lib/ci.py | 119 ++++++++------ .../lib/ci_build.py | 88 ++++++----- .../lib/ci_export.py | 27 ++-- .../lib/ci_prepare.py | 4 +- .../lib/ci_push.py | 47 +++--- .../lib/ci_security_scan.py | 24 +-- .../lib/ci_step_output_printer.py | 16 +- .../lib/ci_test.py | 146 ++++++++++-------- .../lib/common.py | 4 +- .../lib/config/config_data_model.py | 2 +- .../lib/config/data_model_generator.py | 11 +- .../lib/git_access.py | 1 - .../lib/github_release_asset_uploader.py | 22 ++- .../lib/release.py | 77 +++++---- .../lib/release_uploader.py | 19 ++- .../lib/render_template.py | 7 +- exasol_script_languages_container_ci/main.py | 2 +- scripts/build/check_release.py | 10 +- test/asserts.py | 2 +- test/conftest.py | 34 ++-- test/contract_tests/test_ci_test.py | 62 +++++--- test/integration_tests/test_ci_build.py | 30 ++-- test/integration_tests/test_ci_export.py | 4 +- test/integration_tests/test_ci_prepare.py | 7 +- test/integration_tests/test_ci_push.py | 26 ++-- .../test_ci_security_scan.py | 20 +-- test/integration_tests/test_ci_test.py | 23 ++- test/mock_cast.py | 2 +- test/unit_tests/ci_calls.py | 90 ++++++----- test/unit_tests/test_asset_uploader.py | 23 +-- test/unit_tests/test_ci.py | 141 +++++++++-------- test/unit_tests/test_ci_prepare.py | 20 ++- test/unit_tests/test_ci_test.py | 94 +++++++---- test/unit_tests/test_config_data_model.py | 23 +-- .../test_config_data_model_generator.py | 23 ++- test/unit_tests/test_env.py | 1 - test/unit_tests/test_ignore_folders.py | 137 ++++++++++++---- test/unit_tests/test_release.py | 80 +++++----- test/unit_tests/test_release_uploader.py | 57 +++---- 44 files changed, 1074 insertions(+), 706 deletions(-) diff --git a/exasol_script_languages_container_ci/__init__.py b/exasol_script_languages_container_ci/__init__.py index 55dee98..0b25b98 100644 --- a/exasol_script_languages_container_ci/__init__.py +++ b/exasol_script_languages_container_ci/__init__.py @@ -1,5 +1 @@ - -from exasol_script_languages_container_ci.cli.commands import ( - run_ci, - run_release -) +from exasol_script_languages_container_ci.cli.commands import run_ci, run_release diff --git a/exasol_script_languages_container_ci/cli/commands/run_ci.py b/exasol_script_languages_container_ci/cli/commands/run_ci.py index 8a2c392..2cc7443 100644 --- a/exasol_script_languages_container_ci/cli/commands/run_ci.py +++ b/exasol_script_languages_container_ci/cli/commands/run_ci.py @@ -9,40 +9,48 @@ @cli.command() -@click.option('--flavor', required=True, type=str, - help="Flavor name.") -@click.option('--branch-name', required=True, type=str, - help="Branch name.") -@click.option('--docker-user', required=True, type=str, - help="Docker user name") -@click.option('--docker-password', required=True, type=str, - help="Docker password") -@click.option('--docker-build-repository', required=True, type=str, - help="Docker build repository") -@click.option('--docker-release-repository', required=True, type=str, - help="Docker release repository") -@click.option('--commit-sha', required=True, type=str, - help="Commit SHA") -@click.option('--config-file', required=True, type=click.Path(exists=True, file_okay=True, dir_okay=False), - help="The build config file (project specific)") +@click.option("--flavor", required=True, type=str, help="Flavor name.") +@click.option("--branch-name", required=True, type=str, help="Branch name.") +@click.option("--docker-user", required=True, type=str, help="Docker user name") +@click.option("--docker-password", required=True, type=str, help="Docker password") +@click.option( + "--docker-build-repository", required=True, type=str, help="Docker build repository" +) +@click.option( + "--docker-release-repository", + required=True, + type=str, + help="Docker release repository", +) +@click.option("--commit-sha", required=True, type=str, help="Commit SHA") +@click.option( + "--config-file", + required=True, + type=click.Path(exists=True, file_okay=True, dir_okay=False), + help="The build config file (project specific)", +) @click.pass_context -def run_ci(ctx: click.Context, - flavor: str, - branch_name: str, - docker_user: str, - docker_password: str, - docker_build_repository: str, - docker_release_repository: str, - commit_sha: str, - config_file: str): +def run_ci( + ctx: click.Context, + flavor: str, + branch_name: str, + docker_user: str, + docker_password: str, + docker_build_repository: str, + docker_release_repository: str, + commit_sha: str, + config_file: str, +): logging.basicConfig(level=logging.INFO) build_config = Config.parse_file(config_file) - ci(flavor=flavor, - branch_name=branch_name, - docker_user=docker_user, - docker_password=docker_password, - docker_build_repository=docker_build_repository, - docker_release_repository=docker_release_repository, - commit_sha=commit_sha, - build_config=build_config, - git_access=GitAccess()) + ci( + flavor=flavor, + branch_name=branch_name, + docker_user=docker_user, + docker_password=docker_password, + docker_build_repository=docker_build_repository, + docker_release_repository=docker_release_repository, + commit_sha=commit_sha, + build_config=build_config, + git_access=GitAccess(), + ) diff --git a/exasol_script_languages_container_ci/cli/commands/run_release.py b/exasol_script_languages_container_ci/cli/commands/run_release.py index 26af13e..9d9987a 100644 --- a/exasol_script_languages_container_ci/cli/commands/run_release.py +++ b/exasol_script_languages_container_ci/cli/commands/run_release.py @@ -6,50 +6,69 @@ from exasol_script_languages_container_ci.cli.cli import cli from exasol_script_languages_container_ci.lib.asset_uploader import AssetUploader from exasol_script_languages_container_ci.lib.config.config_data_model import Config -from exasol_script_languages_container_ci.lib.github_release_asset_uploader import GithubReleaseAssetUploader +from exasol_script_languages_container_ci.lib.github_release_asset_uploader import ( + GithubReleaseAssetUploader, +) from exasol_script_languages_container_ci.lib.release import release from exasol_script_languages_container_ci.lib.release_uploader import ReleaseUploader @cli.command() -@click.option('--flavor', required=True, type=str, - help="Flavor name.") -@click.option('--docker-user', required=True, type=str, - help="Docker user name") -@click.option('--docker-password', required=True, type=str, - help="Docker password") -@click.option('--docker-release-repository', required=True, type=str, - help="Docker release repository") -@click.option('--config-file', required=True, type=click.Path(exists=True, file_okay=True, dir_okay=False), - help="The build config file (project specific)") -@click.option('--source-repo-url', required=True, type=str, - help="The url of the repository. Usually set by AWS under env variable CODEBUILD_SOURCE_REPO_URL.") -@click.option('--release-id', required=True, type=int, - help="The id of the release.") -@click.option('--dry-run/--no-dry-run', default=False, - help="If true, runs release without pushing the container to the docker release repository." - "If false, also pushes the container to the docker release repository.") +@click.option("--flavor", required=True, type=str, help="Flavor name.") +@click.option("--docker-user", required=True, type=str, help="Docker user name") +@click.option("--docker-password", required=True, type=str, help="Docker password") +@click.option( + "--docker-release-repository", + required=True, + type=str, + help="Docker release repository", +) +@click.option( + "--config-file", + required=True, + type=click.Path(exists=True, file_okay=True, dir_okay=False), + help="The build config file (project specific)", +) +@click.option( + "--source-repo-url", + required=True, + type=str, + help="The url of the repository. Usually set by AWS under env variable CODEBUILD_SOURCE_REPO_URL.", +) +@click.option("--release-id", required=True, type=int, help="The id of the release.") +@click.option( + "--dry-run/--no-dry-run", + default=False, + help="If true, runs release without pushing the container to the docker release repository." + "If false, also pushes the container to the docker release repository.", +) @click.pass_context -def run_release(ctx: click.Context, - flavor: str, - docker_user: str, - docker_password: str, - docker_release_repository: str, - config_file: str, - source_repo_url: str, - release_id: int, - dry_run: bool): +def run_release( + ctx: click.Context, + flavor: str, + docker_user: str, + docker_password: str, + docker_release_repository: str, + config_file: str, + source_repo_url: str, + release_id: int, + dry_run: bool, +): logging.basicConfig(level=logging.INFO) - github_release_asset_uploader = GithubReleaseAssetUploader(os.getenv("GITHUB_TOKEN")) + github_release_asset_uploader = GithubReleaseAssetUploader( + os.getenv("GITHUB_TOKEN") + ) asset_uploader = AssetUploader(release_asset_uploader=github_release_asset_uploader) release_uploader = ReleaseUploader(asset_uploader=asset_uploader) build_config = Config.parse_file(config_file) - release(flavor=flavor, - docker_user=docker_user, - docker_password=docker_password, - docker_release_repository=docker_release_repository, - build_config=build_config, - source_repo_url=source_repo_url, - release_id=release_id, - release_uploader=release_uploader, - is_dry_run=dry_run) + release( + flavor=flavor, + docker_user=docker_user, + docker_password=docker_password, + docker_release_repository=docker_release_repository, + build_config=build_config, + source_repo_url=source_repo_url, + release_id=release_id, + release_uploader=release_uploader, + is_dry_run=dry_run, + ) diff --git a/exasol_script_languages_container_ci/lib/asset_uploader.py b/exasol_script_languages_container_ci/lib/asset_uploader.py index 2eb1896..1479bc3 100644 --- a/exasol_script_languages_container_ci/lib/asset_uploader.py +++ b/exasol_script_languages_container_ci/lib/asset_uploader.py @@ -2,7 +2,9 @@ import logging from pathlib import Path -from exasol_script_languages_container_ci.lib.github_release_asset_uploader import GithubReleaseAssetUploader +from exasol_script_languages_container_ci.lib.github_release_asset_uploader import ( + GithubReleaseAssetUploader, +) class AssetUploader: @@ -10,17 +12,29 @@ class AssetUploader: def __init__(self, release_asset_uploader: GithubReleaseAssetUploader): self._release_asset_uploader = release_asset_uploader - def upload_assets(self, - repo_id: str, release_id: int, content_type: str, - artifact_path: str, file_suffix: str, label_prefix: str): - release_artifacts = glob.glob(f'{artifact_path}/*{file_suffix}') + def upload_assets( + self, + repo_id: str, + release_id: int, + content_type: str, + artifact_path: str, + file_suffix: str, + label_prefix: str, + ): + release_artifacts = glob.glob(f"{artifact_path}/*{file_suffix}") for release_artifact in release_artifacts: artifact_file_name = Path(release_artifact).name if artifact_file_name.endswith(file_suffix): - artifact_file_name = artifact_file_name[:-len(file_suffix)] + artifact_file_name = artifact_file_name[: -len(file_suffix)] else: - logging.error(f"Artifact file: {artifact_file_name} does not end with {file_suffix}. " - f"Using {artifact_file_name} as label.") - self._release_asset_uploader.upload(archive_path=release_artifact, - label=f"{label_prefix} {artifact_file_name}", - repo_id=repo_id, release_id=release_id, content_type=content_type) + logging.error( + f"Artifact file: {artifact_file_name} does not end with {file_suffix}. " + f"Using {artifact_file_name} as label." + ) + self._release_asset_uploader.upload( + archive_path=release_artifact, + label=f"{label_prefix} {artifact_file_name}", + repo_id=repo_id, + release_id=release_id, + content_type=content_type, + ) diff --git a/exasol_script_languages_container_ci/lib/branch_config.py b/exasol_script_languages_container_ci/lib/branch_config.py index 750f49a..7b276f4 100644 --- a/exasol_script_languages_container_ci/lib/branch_config.py +++ b/exasol_script_languages_container_ci/lib/branch_config.py @@ -1,5 +1,5 @@ -from enum import Enum, auto import re +from enum import Enum, auto class BuildSteps(Enum): @@ -9,14 +9,26 @@ class BuildSteps(Enum): class BranchConfig(Enum): - DEVELOP = {BuildSteps.BUILD_ALL_ALWAYS: True, BuildSteps.REBUILD: True, - BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False} - MAIN = {BuildSteps.BUILD_ALL_ALWAYS: True, BuildSteps.REBUILD: True, - BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: True} - REBUILD = {BuildSteps.BUILD_ALL_ALWAYS: True, BuildSteps.REBUILD: True, - BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False} - OTHER = {BuildSteps.BUILD_ALL_ALWAYS: False, BuildSteps.REBUILD: False, - BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False} + DEVELOP = { + BuildSteps.BUILD_ALL_ALWAYS: True, + BuildSteps.REBUILD: True, + BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False, + } + MAIN = { + BuildSteps.BUILD_ALL_ALWAYS: True, + BuildSteps.REBUILD: True, + BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: True, + } + REBUILD = { + BuildSteps.BUILD_ALL_ALWAYS: True, + BuildSteps.REBUILD: True, + BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False, + } + OTHER = { + BuildSteps.BUILD_ALL_ALWAYS: False, + BuildSteps.REBUILD: False, + BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False, + } @staticmethod def build_always(branch_name: str) -> bool: @@ -28,16 +40,20 @@ def rebuild(branch_name) -> bool: @staticmethod def push_to_docker_release_repo(branch_name: str) -> bool: - return get_branch_config(branch_name).value[BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO] + return get_branch_config(branch_name).value[ + BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO + ] def get_branch_config(branch_name: str) -> BranchConfig: - matches = ((re.compile(r"refs/heads/(master|main)"), BranchConfig.MAIN), - (re.compile(r"refs/heads/develop"), BranchConfig.DEVELOP), - (re.compile(r"refs/heads/rebuild/.*"), BranchConfig.REBUILD)) + matches = ( + (re.compile(r"refs/heads/(master|main)"), BranchConfig.MAIN), + (re.compile(r"refs/heads/develop"), BranchConfig.DEVELOP), + (re.compile(r"refs/heads/rebuild/.*"), BranchConfig.REBUILD), + ) branch_cfg = BranchConfig.OTHER - for (branch_regex, branch_config) in matches: + for branch_regex, branch_config in matches: if branch_regex.match(branch_name): branch_cfg = branch_config break diff --git a/exasol_script_languages_container_ci/lib/ci.py b/exasol_script_languages_container_ci/lib/ci.py index 329d4fb..cbe8601 100644 --- a/exasol_script_languages_container_ci/lib/ci.py +++ b/exasol_script_languages_container_ci/lib/ci.py @@ -1,5 +1,5 @@ import logging -from typing import Set, Callable +from typing import Callable, Set from exasol_script_languages_container_ci.lib.branch_config import BranchConfig from exasol_script_languages_container_ci.lib.ci_build import CIBuild @@ -21,39 +21,52 @@ def get_all_affected_files(git_access: GitAccess, base_branch: str) -> Set[str]: return changed_files -def check_if_need_to_build(branch_name: str, config: Config, flavor: str, git_access: GitAccess): +def check_if_need_to_build( + branch_name: str, config: Config, flavor: str, git_access: GitAccess +): if BranchConfig.build_always(branch_name): return True if "[rebuild]" in git_access.get_last_commit_message(): return True affected_files = list(get_all_affected_files(git_access, config.build.base_branch)) - logging.debug(f"check_if_need_to_build: Found files of last commits: {affected_files}") + logging.debug( + f"check_if_need_to_build: Found files of last commits: {affected_files}" + ) for ignore_path in config.build.ignore.paths: - affected_files = list(filter(lambda file: not file.startswith(ignore_path), affected_files)) + affected_files = list( + filter(lambda file: not file.startswith(ignore_path), affected_files) + ) if len(affected_files) > 0: # Now filter out also other flavor folders this_flavor_path = f"flavors/{flavor}" - affected_files = list(filter(lambda file: not file.startswith("flavors") or file.startswith(this_flavor_path), - affected_files)) + affected_files = list( + filter( + lambda file: not file.startswith("flavors") + or file.startswith(this_flavor_path), + affected_files, + ) + ) logging.debug(f"check_if_need_to_build: filtered files: {affected_files}") return len(affected_files) > 0 -def ci(flavor: str, - branch_name: str, - docker_user: str, - docker_password: str, - docker_build_repository: str, - docker_release_repository: str, - commit_sha: str, - build_config: Config, - git_access: GitAccess, - ci_build: CIBuild = CIBuild(), - ci_execute_tests: CIExecuteTest = CIExecuteTest(), - ci_push: CIPush = CIPush(), - ci_security_scan: CISecurityScan = CISecurityScan(), - ci_prepare: CIPrepare = CIPrepare()): +def ci( + flavor: str, + branch_name: str, + docker_user: str, + docker_password: str, + docker_build_repository: str, + docker_release_repository: str, + commit_sha: str, + build_config: Config, + git_access: GitAccess, + ci_build: CIBuild = CIBuild(), + ci_execute_tests: CIExecuteTest = CIExecuteTest(), + ci_push: CIPush = CIPush(), + ci_security_scan: CISecurityScan = CISecurityScan(), + ci_prepare: CIPrepare = CIPrepare(), +): """ Run CI build: 1. Build image @@ -66,36 +79,48 @@ def ci(flavor: str, flavor_path = (f"flavors/{flavor}",) test_container_folder = "test_container" rebuild = BranchConfig.rebuild(branch_name) - needs_to_build = check_if_need_to_build(branch_name, build_config, flavor, git_access) + needs_to_build = check_if_need_to_build( + branch_name, build_config, flavor, git_access + ) if needs_to_build: ci_prepare.prepare() - ci_build.build(flavor_path=flavor_path, - rebuild=rebuild, - build_docker_repository=docker_build_repository, - commit_sha=commit_sha, - docker_user=docker_user, - docker_password=docker_password, - test_container_folder=test_container_folder) - ci_execute_tests.execute_tests(flavor_path=flavor_path, - docker_user=docker_user, - docker_password=docker_password, - test_container_folder=test_container_folder) + ci_build.build( + flavor_path=flavor_path, + rebuild=rebuild, + build_docker_repository=docker_build_repository, + commit_sha=commit_sha, + docker_user=docker_user, + docker_password=docker_password, + test_container_folder=test_container_folder, + ) + ci_execute_tests.execute_tests( + flavor_path=flavor_path, + docker_user=docker_user, + docker_password=docker_password, + test_container_folder=test_container_folder, + ) ci_security_scan.run_security_scan(flavor_path=flavor_path) - ci_push.push(flavor_path=flavor_path, - target_docker_repository=docker_build_repository, - target_docker_tag_prefix=commit_sha, - docker_user=docker_user, - docker_password=docker_password) - ci_push.push(flavor_path=flavor_path, - target_docker_repository=docker_build_repository, - target_docker_tag_prefix="", - docker_user=docker_user, - docker_password=docker_password) + ci_push.push( + flavor_path=flavor_path, + target_docker_repository=docker_build_repository, + target_docker_tag_prefix=commit_sha, + docker_user=docker_user, + docker_password=docker_password, + ) + ci_push.push( + flavor_path=flavor_path, + target_docker_repository=docker_build_repository, + target_docker_tag_prefix="", + docker_user=docker_user, + docker_password=docker_password, + ) if BranchConfig.push_to_docker_release_repo(branch_name): - ci_push.push(flavor_path=flavor_path, - target_docker_repository=docker_release_repository, - target_docker_tag_prefix="", - docker_user=docker_user, - docker_password=docker_password) + ci_push.push( + flavor_path=flavor_path, + target_docker_repository=docker_release_repository, + target_docker_tag_prefix="", + docker_user=docker_user, + docker_password=docker_password, + ) else: logging.warning(f"Skipping build...") diff --git a/exasol_script_languages_container_ci/lib/ci_build.py b/exasol_script_languages_container_ci/lib/ci_build.py index f234d7f..fb35f42 100644 --- a/exasol_script_languages_container_ci/lib/ci_build.py +++ b/exasol_script_languages_container_ci/lib/ci_build.py @@ -1,54 +1,70 @@ import logging -from typing import Tuple, Optional +from typing import Optional, Tuple -from exasol_integration_test_docker_environment.lib.api.build_test_container import build_test_container from exasol.slc.api import build -from exasol.slc.internal.tasks.test.test_container_content import build_test_container_content +from exasol.slc.internal.tasks.test.test_container_content import ( + build_test_container_content, +) +from exasol_integration_test_docker_environment.lib.api.build_test_container import ( + build_test_container, +) -from exasol_script_languages_container_ci.lib.ci_step_output_printer import CIStepOutputPrinterProtocol, \ - CIStepOutputPrinter +from exasol_script_languages_container_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinter, + CIStepOutputPrinterProtocol, +) class CIBuild: - def __init__(self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info)): + def __init__( + self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info) + ): self._printer = printer - def build(self, - flavor_path: Tuple[str, ...], - rebuild: bool, - build_docker_repository: Optional[str], - commit_sha: str, - docker_user: str, - docker_password: str, - test_container_folder: str): + def build( + self, + flavor_path: Tuple[str, ...], + rebuild: bool, + build_docker_repository: Optional[str], + commit_sha: str, + docker_user: str, + docker_password: str, + test_container_folder: str, + ): """ Build the script-language container for given flavor. And also build the test container """ logging.info(f"Running command 'build' with parameters: {locals()}") if build_docker_repository is None: - slc_image_infos = build(flavor_path=flavor_path, force_rebuild=rebuild, - source_docker_tag_prefix=commit_sha, - source_docker_username=docker_user, - source_docker_password=docker_password, - shortcut_build=False, - workers=7, - log_level="WARNING", - use_job_specific_log_file=True - ) + slc_image_infos = build( + flavor_path=flavor_path, + force_rebuild=rebuild, + source_docker_tag_prefix=commit_sha, + source_docker_username=docker_user, + source_docker_password=docker_password, + shortcut_build=False, + workers=7, + log_level="WARNING", + use_job_specific_log_file=True, + ) else: - slc_image_infos = build(flavor_path=flavor_path, force_rebuild=rebuild, - source_docker_repository_name=build_docker_repository, - source_docker_tag_prefix=commit_sha, - source_docker_username=docker_user, - source_docker_password=docker_password, - shortcut_build=False, - workers=7, - log_level="WARNING", - use_job_specific_log_file=True - ) - logging.info(f"Running command 'build_test_container' with parameters: {locals()}") + slc_image_infos = build( + flavor_path=flavor_path, + force_rebuild=rebuild, + source_docker_repository_name=build_docker_repository, + source_docker_tag_prefix=commit_sha, + source_docker_username=docker_user, + source_docker_password=docker_password, + shortcut_build=False, + workers=7, + log_level="WARNING", + use_job_specific_log_file=True, + ) + logging.info( + f"Running command 'build_test_container' with parameters: {locals()}" + ) content = build_test_container_content(test_container_folder) test_container_image_infos = build_test_container( force_rebuild=rebuild, @@ -57,6 +73,6 @@ def build(self, source_docker_repository_name=build_docker_repository, source_docker_tag_prefix=commit_sha, log_level="WARNING", - use_job_specific_log_file=True + use_job_specific_log_file=True, ) - self._printer.print_exasol_docker_images() \ No newline at end of file + self._printer.print_exasol_docker_images() diff --git a/exasol_script_languages_container_ci/lib/ci_export.py b/exasol_script_languages_container_ci/lib/ci_export.py index afc2272..c0f292f 100644 --- a/exasol_script_languages_container_ci/lib/ci_export.py +++ b/exasol_script_languages_container_ci/lib/ci_export.py @@ -3,27 +3,30 @@ from exasol.slc.api import export -from exasol_script_languages_container_ci.lib.ci_step_output_printer import CIStepOutputPrinterProtocol, \ - CIStepOutputPrinter +from exasol_script_languages_container_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinter, + CIStepOutputPrinterProtocol, +) class CIExport: - def __init__(self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info)): + def __init__( + self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info) + ): self._printer = printer - def export(self, - flavor_path: Tuple[str, ...], - export_path: str): + def export(self, flavor_path: Tuple[str, ...], export_path: str): """ Export the flavor as tar.gz file """ logging.info(f"Running command 'push' with parameters: {locals()}") - export_result = export(flavor_path=flavor_path, - export_path=export_path, - workers=7, - log_level="WARNING", - use_job_specific_log_file=True - ) + export_result = export( + flavor_path=flavor_path, + export_path=export_path, + workers=7, + log_level="WARNING", + use_job_specific_log_file=True, + ) self._printer.print_exasol_docker_images() diff --git a/exasol_script_languages_container_ci/lib/ci_prepare.py b/exasol_script_languages_container_ci/lib/ci_prepare.py index d921d67..326ef19 100644 --- a/exasol_script_languages_container_ci/lib/ci_prepare.py +++ b/exasol_script_languages_container_ci/lib/ci_prepare.py @@ -1,7 +1,9 @@ import os from pathlib import Path -from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY +from exasol_integration_test_docker_environment.cli.options.system_options import ( + DEFAULT_OUTPUT_DIRECTORY, +) from exasol_integration_test_docker_environment.lib.base import luigi_log_config diff --git a/exasol_script_languages_container_ci/lib/ci_push.py b/exasol_script_languages_container_ci/lib/ci_push.py index d6ff1a8..95a30fe 100644 --- a/exasol_script_languages_container_ci/lib/ci_push.py +++ b/exasol_script_languages_container_ci/lib/ci_push.py @@ -3,35 +3,42 @@ from exasol.slc.api.push import push -from exasol_script_languages_container_ci.lib.ci_step_output_printer import CIStepOutputPrinterProtocol, \ - CIStepOutputPrinter +from exasol_script_languages_container_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinter, + CIStepOutputPrinterProtocol, +) class CIPush: - def __init__(self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info)): + def __init__( + self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info) + ): self._printer = printer - def push(self, - flavor_path: Tuple[str, ...], - target_docker_repository: str, - target_docker_tag_prefix: str, - docker_user: str, - docker_password: str): + def push( + self, + flavor_path: Tuple[str, ...], + target_docker_repository: str, + target_docker_tag_prefix: str, + docker_user: str, + docker_password: str, + ): """ Push the docker image to Dockerhub """ logging.info(f"Running command 'push' with parameters: {locals()}") - push(flavor_path=flavor_path, - push_all=True, - force_push=True, - workers=7, - target_docker_repository_name=target_docker_repository, - target_docker_tag_prefix=target_docker_tag_prefix, - target_docker_username=docker_user, - target_docker_password=docker_password, - log_level="WARNING", - use_job_specific_log_file=True - ) + push( + flavor_path=flavor_path, + push_all=True, + force_push=True, + workers=7, + target_docker_repository_name=target_docker_repository, + target_docker_tag_prefix=target_docker_tag_prefix, + target_docker_username=docker_user, + target_docker_password=docker_password, + log_level="WARNING", + use_job_specific_log_file=True, + ) self._printer.print_exasol_docker_images() diff --git a/exasol_script_languages_container_ci/lib/ci_security_scan.py b/exasol_script_languages_container_ci/lib/ci_security_scan.py index 7b58157..cadf232 100644 --- a/exasol_script_languages_container_ci/lib/ci_security_scan.py +++ b/exasol_script_languages_container_ci/lib/ci_security_scan.py @@ -4,27 +4,31 @@ from exasol.slc.api import security_scan -from exasol_script_languages_container_ci.lib.ci_step_output_printer import CIStepOutputPrinterProtocol, \ - CIStepOutputPrinter +from exasol_script_languages_container_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinter, + CIStepOutputPrinterProtocol, +) class CISecurityScan: - def __init__(self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info)): + def __init__( + self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info) + ): self._printer = printer - def run_security_scan(self, - flavor_path: Tuple[str, ...]): + def run_security_scan(self, flavor_path: Tuple[str, ...]): """ Run security scan and print result """ logging.info(f"Running command 'security_scan' with parameters {locals()}") - security_scan_result = security_scan(flavor_path=flavor_path, - workers=7, - log_level="WARNING", - use_job_specific_log_file=True - ) + security_scan_result = security_scan( + flavor_path=flavor_path, + workers=7, + log_level="WARNING", + use_job_specific_log_file=True, + ) logging.info("============= SECURITY REPORT ===========") self._printer.print_file(Path(security_scan_result.report_path)) self._printer.print_exasol_docker_images() diff --git a/exasol_script_languages_container_ci/lib/ci_step_output_printer.py b/exasol_script_languages_container_ci/lib/ci_step_output_printer.py index 40392ae..b660a0f 100644 --- a/exasol_script_languages_container_ci/lib/ci_step_output_printer.py +++ b/exasol_script_languages_container_ci/lib/ci_step_output_printer.py @@ -17,7 +17,9 @@ def print_file(self, filename: Path): def _get_exasol_docker_images(): docker_client = docker.from_env() try: - exa_images = [str(img) for img in docker_client.images.list() if "exasol" in str(img)] + exa_images = [ + str(img) for img in docker_client.images.list() if "exasol" in str(img) + ] return exa_images finally: docker_client.close() @@ -35,17 +37,19 @@ def print_exasol_docker_images(self): :return: None """ - self._writer(cleandoc(""" + self._writer( + cleandoc( + """ {seperator} Printing docker images {seperator} - {images}""").format( - seperator=20 * "=", images="\n".join(_get_exasol_docker_images()) - )) + {images}""" + ).format(seperator=20 * "=", images="\n".join(_get_exasol_docker_images())) + ) def print_file(self, filename: Path): """ Print the file's content to the writer. """ - with open(filename, "r") as f: + with open(filename) as f: self._writer(f.read()) diff --git a/exasol_script_languages_container_ci/lib/ci_test.py b/exasol_script_languages_container_ci/lib/ci_test.py index 21728b0..ec05960 100644 --- a/exasol_script_languages_container_ci/lib/ci_test.py +++ b/exasol_script_languages_container_ci/lib/ci_test.py @@ -1,34 +1,40 @@ import logging -from typing import Tuple, Protocol +from typing import Protocol, Tuple from exasol.slc.api.run_db_tests import run_db_test from exasol.slc.models.test_result import AllTestsResult -from exasol_script_languages_container_ci.lib.ci_step_output_printer import CIStepOutputPrinterProtocol, \ - CIStepOutputPrinter +from exasol_script_languages_container_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinter, + CIStepOutputPrinterProtocol, +) class DBTestRunnerProtocol(Protocol): - def run(self, - flavor_path: Tuple[str, ...], - release_goal: Tuple[str, ...], - test_folder: Tuple[str, ...], - test_container_folder: str, - workers: int, - docker_username: str, - docker_password: str) -> AllTestsResult: + def run( + self, + flavor_path: Tuple[str, ...], + release_goal: Tuple[str, ...], + test_folder: Tuple[str, ...], + test_container_folder: str, + workers: int, + docker_username: str, + docker_password: str, + ) -> AllTestsResult: raise NotImplementedError() class DBTestRunner(DBTestRunnerProtocol): - def run(self, - flavor_path: Tuple[str, ...], - release_goal: Tuple[str, ...], - test_folder: Tuple[str, ...], - test_container_folder: str, - workers: int, - docker_username: str, - docker_password: str) -> AllTestsResult: + def run( + self, + flavor_path: Tuple[str, ...], + release_goal: Tuple[str, ...], + test_folder: Tuple[str, ...], + test_container_folder: str, + workers: int, + docker_username: str, + docker_password: str, + ) -> AllTestsResult: return run_db_test( flavor_path=flavor_path, release_goal=release_goal, @@ -38,67 +44,85 @@ def run(self, source_docker_username=docker_username, source_docker_password=docker_password, log_level="WARNING", - use_job_specific_log_file=True + use_job_specific_log_file=True, ) class CIExecuteTest: - def __init__(self, - db_test_runner: DBTestRunnerProtocol = DBTestRunner(), - printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info)): + def __init__( + self, + db_test_runner: DBTestRunnerProtocol = DBTestRunner(), + printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info), + ): self._db_test_runner = db_test_runner self._printer = printer - def execute_tests(self, - flavor_path: Tuple[str, ...], - docker_user: str, - docker_password: str, - test_container_folder: str): + def execute_tests( + self, + flavor_path: Tuple[str, ...], + docker_user: str, + docker_password: str, + test_container_folder: str, + ): """ Run db tests """ - db_tests_are_ok = self.run_db_tests(flavor_path=flavor_path, - docker_user=docker_user, - docker_password=docker_password, - test_container_folder=test_container_folder) - linker_namespace_tests_are_ok = self.run_linker_namespace_tests(flavor_path=flavor_path, - docker_user=docker_user, - docker_password=docker_password, - test_container_folder=test_container_folder) + db_tests_are_ok = self.run_db_tests( + flavor_path=flavor_path, + docker_user=docker_user, + docker_password=docker_password, + test_container_folder=test_container_folder, + ) + linker_namespace_tests_are_ok = self.run_linker_namespace_tests( + flavor_path=flavor_path, + docker_user=docker_user, + docker_password=docker_password, + test_container_folder=test_container_folder, + ) self._printer.print_exasol_docker_images() tests_are_ok = db_tests_are_ok and linker_namespace_tests_are_ok if not tests_are_ok: raise AssertionError("Not all tests are ok!") - def run_db_tests(self, flavor_path: Tuple[str, ...], - docker_user: str, - docker_password: str, - test_container_folder: str) -> bool: + def run_db_tests( + self, + flavor_path: Tuple[str, ...], + docker_user: str, + docker_password: str, + test_container_folder: str, + ) -> bool: logging.info(f"Running command 'run_db_test' for flavor-path {flavor_path}") - db_test_result = \ - self._db_test_runner.run(flavor_path=flavor_path, - test_folder=tuple(), - release_goal=('release',), - workers=7, - docker_username=docker_user, - docker_password=docker_password, - test_container_folder=test_container_folder) + db_test_result = self._db_test_runner.run( + flavor_path=flavor_path, + test_folder=tuple(), + release_goal=("release",), + workers=7, + docker_username=docker_user, + docker_password=docker_password, + test_container_folder=test_container_folder, + ) self._printer.print_file(db_test_result.command_line_output_path) return db_test_result.tests_are_ok - def run_linker_namespace_tests(self, - flavor_path: Tuple[str, ...], - docker_user: str, - docker_password: str, - test_container_folder: str) -> bool: - logging.info(f"Running command 'run_db_test' for linker_namespace_sanity for flavor-path {flavor_path}") - linker_namespace_test_result = \ - self._db_test_runner.run(flavor_path=flavor_path, workers=7, - test_folder=("test/linker_namespace_sanity",), - release_goal=("base_test_build_run",), - docker_username=docker_user, - docker_password=docker_password, - test_container_folder=test_container_folder) + def run_linker_namespace_tests( + self, + flavor_path: Tuple[str, ...], + docker_user: str, + docker_password: str, + test_container_folder: str, + ) -> bool: + logging.info( + f"Running command 'run_db_test' for linker_namespace_sanity for flavor-path {flavor_path}" + ) + linker_namespace_test_result = self._db_test_runner.run( + flavor_path=flavor_path, + workers=7, + test_folder=("test/linker_namespace_sanity",), + release_goal=("base_test_build_run",), + docker_username=docker_user, + docker_password=docker_password, + test_container_folder=test_container_folder, + ) self._printer.print_file(linker_namespace_test_result.command_line_output_path) return linker_namespace_test_result.tests_are_ok diff --git a/exasol_script_languages_container_ci/lib/common.py b/exasol_script_languages_container_ci/lib/common.py index 4fc93ae..b909599 100644 --- a/exasol_script_languages_container_ci/lib/common.py +++ b/exasol_script_languages_container_ci/lib/common.py @@ -1,8 +1,8 @@ import json from contextlib import contextmanager +from inspect import cleandoc from pathlib import Path from typing import Callable -from inspect import cleandoc import docker @@ -12,5 +12,5 @@ def get_config(config_file: str): """ Opens config file and returns parsed JSON object. """ - with open(config_file, "r") as f: + with open(config_file) as f: yield json.load(f) diff --git a/exasol_script_languages_container_ci/lib/config/config_data_model.py b/exasol_script_languages_container_ci/lib/config/config_data_model.py index 7954696..550194b 100644 --- a/exasol_script_languages_container_ci/lib/config/config_data_model.py +++ b/exasol_script_languages_container_ci/lib/config/config_data_model.py @@ -12,7 +12,7 @@ class Ignore(BaseModel): class Config: extra = Extra.forbid - paths: List[str] + paths: list[str] class Build(BaseModel): diff --git a/exasol_script_languages_container_ci/lib/config/data_model_generator.py b/exasol_script_languages_container_ci/lib/config/data_model_generator.py index 8916502..6146120 100644 --- a/exasol_script_languages_container_ci/lib/config/data_model_generator.py +++ b/exasol_script_languages_container_ci/lib/config/data_model_generator.py @@ -3,7 +3,7 @@ from pathlib import Path from tempfile import TemporaryDirectory -from datamodel_code_generator import generate, InputFileType +from datamodel_code_generator import InputFileType, generate from exasol_script_languages_container_ci.lib.render_template import render_template @@ -21,8 +21,13 @@ def generate_config_data_model(output_file: Path) -> Path: schema_json = json.dumps(schema_dict) with TemporaryDirectory() as directory: temp_output_file = Path(directory) / CONFIG_DATA_MODEL_FILE_NAME - generate(schema_json, input_file_type=InputFileType.JsonSchema, output=temp_output_file, - class_name="Config", apply_default_values_for_required_fields=True) + generate( + schema_json, + input_file_type=InputFileType.JsonSchema, + output=temp_output_file, + class_name="Config", + apply_default_values_for_required_fields=True, + ) with temp_output_file.open("rt") as temp_output_file_handle: with output_file.open("wt") as output_file_handle: lines = (line for line in temp_output_file_handle) diff --git a/exasol_script_languages_container_ci/lib/git_access.py b/exasol_script_languages_container_ci/lib/git_access.py index b342d08..51ddc8a 100644 --- a/exasol_script_languages_container_ci/lib/git_access.py +++ b/exasol_script_languages_container_ci/lib/git_access.py @@ -41,4 +41,3 @@ def get_files_of_commit(self, commit_sha) -> Iterable[str]: """ repo = Repo() return repo.commit(commit_sha).stats.files.keys() - diff --git a/exasol_script_languages_container_ci/lib/github_release_asset_uploader.py b/exasol_script_languages_container_ci/lib/github_release_asset_uploader.py index 0c7a33f..81b638f 100644 --- a/exasol_script_languages_container_ci/lib/github_release_asset_uploader.py +++ b/exasol_script_languages_container_ci/lib/github_release_asset_uploader.py @@ -4,26 +4,38 @@ from github import Github, GithubException -class GithubReleaseAssetUploader(object): +class GithubReleaseAssetUploader: """ Implements upload to a Github Release. See https://docs.github.com/en/rest/releases/assets#upload-a-release-asset for details. The access token needs to be stored in the environment variable GITHUB_TOKEN """ + def __init__(self, token): self._token = token - def upload(self, archive_path: str, label: str, repo_id: str, release_id: int, content_type: str): + def upload( + self, + archive_path: str, + label: str, + repo_id: str, + release_id: int, + content_type: str, + ): gh = Github(self._token) gh_repo = gh.get_repo(repo_id) release = gh_repo.get_release(release_id) # Check GH limitation # https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases#storage-and-bandwidth-quotas - if Path(archive_path).stat().st_size >= 2 * (2 ** 30): + if Path(archive_path).stat().st_size >= 2 * (2**30): logging.error("File larger than 2GB. Skipping it...") else: try: - release.upload_asset(path=archive_path, label=label, content_type=content_type) + release.upload_asset( + path=archive_path, label=label, content_type=content_type + ) except GithubException as ex: - logging.error(f"Upload of asset {archive_path} to release {release_id} failed: {ex}") + logging.error( + f"Upload of asset {archive_path} to release {release_id} failed: {ex}" + ) raise ex diff --git a/exasol_script_languages_container_ci/lib/release.py b/exasol_script_languages_container_ci/lib/release.py index a45a244..8451f15 100644 --- a/exasol_script_languages_container_ci/lib/release.py +++ b/exasol_script_languages_container_ci/lib/release.py @@ -3,7 +3,9 @@ from pathlib import Path from typing import Callable -from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY +from exasol_integration_test_docker_environment.cli.options.system_options import ( + DEFAULT_OUTPUT_DIRECTORY, +) from exasol_integration_test_docker_environment.lib.base import luigi_log_config from exasol_script_languages_container_ci.lib.ci_build import CIBuild @@ -15,20 +17,22 @@ from exasol_script_languages_container_ci.lib.release_uploader import ReleaseUploader -def release(flavor: str, - docker_user: str, - docker_password: str, - docker_release_repository: str, - build_config: Config, - source_repo_url: str, - release_id: int, - is_dry_run: bool, - release_uploader: ReleaseUploader, - ci_build: CIBuild = CIBuild(), - ci_execute_tests: CIExecuteTest = CIExecuteTest(), - ci_push: CIPush = CIPush(), - ci_security_scan: CISecurityScan = CISecurityScan(), - ci_prepare: CIPrepare = CIPrepare()): +def release( + flavor: str, + docker_user: str, + docker_password: str, + docker_release_repository: str, + build_config: Config, + source_repo_url: str, + release_id: int, + is_dry_run: bool, + release_uploader: ReleaseUploader, + ci_build: CIBuild = CIBuild(), + ci_execute_tests: CIExecuteTest = CIExecuteTest(), + ci_push: CIPush = CIPush(), + ci_security_scan: CISecurityScan = CISecurityScan(), + ci_prepare: CIPrepare = CIPrepare(), +): """ Run Release build: 1. Build image @@ -41,25 +45,32 @@ def release(flavor: str, flavor_path = (f"flavors/{flavor}",) test_container_folder = "test_container" ci_prepare.prepare() - ci_build.build(flavor_path=flavor_path, rebuild=True, - build_docker_repository=None, - commit_sha="", - docker_user=None, - docker_password=None, - test_container_folder=test_container_folder) - ci_execute_tests.execute_tests(flavor_path=flavor_path, - docker_user=docker_user, - docker_password=docker_password, - test_container_folder=test_container_folder) + ci_build.build( + flavor_path=flavor_path, + rebuild=True, + build_docker_repository=None, + commit_sha="", + docker_user=None, + docker_password=None, + test_container_folder=test_container_folder, + ) + ci_execute_tests.execute_tests( + flavor_path=flavor_path, + docker_user=docker_user, + docker_password=docker_password, + test_container_folder=test_container_folder, + ) ci_security_scan.run_security_scan(flavor_path=flavor_path) if not is_dry_run: - ci_push.push(flavor_path=flavor_path, - target_docker_repository=docker_release_repository, - target_docker_tag_prefix="", - docker_user=docker_user, - docker_password=docker_password) + ci_push.push( + flavor_path=flavor_path, + target_docker_repository=docker_release_repository, + target_docker_tag_prefix="", + docker_user=docker_user, + docker_password=docker_password, + ) else: logging.info("Skipping push to docker release repository due to dry-run.") - release_uploader.release_upload(flavor_path=flavor_path, - source_repo_url=source_repo_url, - release_id=release_id) + release_uploader.release_upload( + flavor_path=flavor_path, source_repo_url=source_repo_url, release_id=release_id + ) diff --git a/exasol_script_languages_container_ci/lib/release_uploader.py b/exasol_script_languages_container_ci/lib/release_uploader.py index c9d1d66..bddaa30 100644 --- a/exasol_script_languages_container_ci/lib/release_uploader.py +++ b/exasol_script_languages_container_ci/lib/release_uploader.py @@ -15,8 +15,10 @@ def _parse_repo_url(source_repo_url: str) -> str: """ res = re.search(r"^https://github.com/([a-zA-Z0-9\-_/]+)$", source_repo_url) if res is None: - raise ValueError(f"Parameter source_repo_url={source_repo_url} does not match the following regex: " - f"^https://github.com/([a-zA-Z0-9\-_/]+)$") + raise ValueError( + f"Parameter source_repo_url={source_repo_url} does not match the following regex: " + f"^https://github.com/([a-zA-Z0-9\-_/]+)$" + ) return res.groups()[0] @@ -26,10 +28,9 @@ def __init__(self, asset_uploader: AssetUploader, ci_export: CIExport = CIExport self._ci_export = ci_export self._asset_uploader = asset_uploader - def release_upload(self, - flavor_path: Tuple[str, ...], - source_repo_url: str, - release_id: int) -> None: + def release_upload( + self, flavor_path: Tuple[str, ...], source_repo_url: str, release_id: int + ) -> None: """ Exports the container into tar.gz(s) and uploads to the repository / release. release_key is expected to have the following format: "{key}:{value}" where {key} can be: @@ -47,11 +48,13 @@ def release_upload(self, content_type="application/gzip", artifact_path=temp_dir, file_suffix=".tar.gz", - label_prefix="Flavor") + label_prefix="Flavor", + ) self._asset_uploader.upload_assets( repo_id=repo_id, release_id=release_id, content_type="text/plain", artifact_path=temp_dir, file_suffix=".tar.gz.sha512sum", - label_prefix="Checksum") + label_prefix="Checksum", + ) diff --git a/exasol_script_languages_container_ci/lib/render_template.py b/exasol_script_languages_container_ci/lib/render_template.py index 7b26bc8..41204e1 100644 --- a/exasol_script_languages_container_ci/lib/render_template.py +++ b/exasol_script_languages_container_ci/lib/render_template.py @@ -2,7 +2,10 @@ def render_template(template: str, **kwargs): - env = jinja2.Environment(loader=jinja2.PackageLoader("exasol_script_languages_container_ci"), - autoescape=jinja2.select_autoescape(), keep_trailing_newline=True) + env = jinja2.Environment( + loader=jinja2.PackageLoader("exasol_script_languages_container_ci"), + autoescape=jinja2.select_autoescape(), + keep_trailing_newline=True, + ) t = env.get_template(template) return t.render(**kwargs) diff --git a/exasol_script_languages_container_ci/main.py b/exasol_script_languages_container_ci/main.py index ddbdfb5..bfb017f 100755 --- a/exasol_script_languages_container_ci/main.py +++ b/exasol_script_languages_container_ci/main.py @@ -2,5 +2,5 @@ # from exasol_script_languages_container_ci.cli.cli import cli -if __name__ == '__main__': +if __name__ == "__main__": cli() diff --git a/scripts/build/check_release.py b/scripts/build/check_release.py index e3434b6..0d0017a 100644 --- a/scripts/build/check_release.py +++ b/scripts/build/check_release.py @@ -1,8 +1,8 @@ import re from pathlib import Path -from git import Repo import toml +from git import Repo def get_git_version(): @@ -20,13 +20,15 @@ def get_git_version(): def get_poetry_version(): - parsed_toml = toml.load('pyproject.toml') + parsed_toml = toml.load("pyproject.toml") return parsed_toml["tool"]["poetry"]["version"].strip() def get_change_log_version(): # Path overloads __truediv__ - with open(Path(__file__).parent / ".." / ".." / "doc" / "changes" / "changelog.md") as changelog: + with open( + Path(__file__).parent / ".." / ".." / "doc" / "changes" / "changelog.md" + ) as changelog: changelog_str = changelog.read() # Search for the FIRST pattern like: "* [0.5.0](changes_0.5.0.md)" in the changelog file. # Note that we encapsulate the [(0.5.0)] with parenthesis, which tells re to return the matching string as group @@ -34,7 +36,7 @@ def get_change_log_version(): return version_match.groups()[0] -if __name__ == '__main__': +if __name__ == "__main__": poetry_version = get_poetry_version() latest_tag = get_git_version() changelog_version = get_change_log_version() diff --git a/test/asserts.py b/test/asserts.py index b24adf0..3714fdc 100644 --- a/test/asserts.py +++ b/test/asserts.py @@ -8,4 +8,4 @@ def not_raises(exception): try: yield except exception: - raise pytest.fail("DID RAISE {0}".format(exception)) + raise pytest.fail(f"DID RAISE {exception}") diff --git a/test/conftest.py b/test/conftest.py index a86cd6c..0904597 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -5,13 +5,20 @@ from pathlib import Path from tempfile import TemporaryDirectory from unittest import mock -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch import pytest -from exasol_script_languages_container_ci.lib.config.config_data_model import Config, Build, Ignore, Release -from exasol_script_languages_container_ci.lib.config.data_model_generator import config_data_model_default_output_file, \ - regenerate_config_data_model +from exasol_script_languages_container_ci.lib.config.config_data_model import ( + Build, + Config, + Ignore, + Release, +) +from exasol_script_languages_container_ci.lib.config.data_model_generator import ( + config_data_model_default_output_file, + regenerate_config_data_model, +) script_path = Path(__file__).absolute().parent @@ -22,8 +29,10 @@ def pytest_addoption(parser): parser.addoption( - DISABLE_PYDANTIC_MODEL_GENERATION, action="store_true", default=False, - help="Disables the generation of the pydantic models from the json schemas" + DISABLE_PYDANTIC_MODEL_GENERATION, + action="store_true", + default=False, + help="Disables the generation of the pydantic models from the json schemas", ) @@ -78,10 +87,9 @@ def tmp_test_dir(): @pytest.fixture def build_config() -> Config: return Config( - build=Build( - ignore=Ignore(paths=["doc"]), - base_branch="master"), - release=Release(timeout_in_minutes=1)) + build=Build(ignore=Ignore(paths=["doc"]), base_branch="master"), + release=Release(timeout_in_minutes=1), + ) @pytest.fixture() @@ -99,7 +107,8 @@ def git_access_mock(): @pytest.fixture def expected_json_config() -> str: - json = cleandoc(""" + json = cleandoc( + """ { "build": { "ignore": { @@ -113,5 +122,6 @@ def expected_json_config() -> str: "release": { "timeout_in_minutes": 1 } - }""") + }""" + ) return json diff --git a/test/contract_tests/test_ci_test.py b/test/contract_tests/test_ci_test.py index 2188a31..38c5c5f 100644 --- a/test/contract_tests/test_ci_test.py +++ b/test/contract_tests/test_ci_test.py @@ -1,5 +1,7 @@ import pytest -from exasol_integration_test_docker_environment.lib.api.api_errors import TaskRuntimeError +from exasol_integration_test_docker_environment.lib.api.api_errors import ( + TaskRuntimeError, +) from exasol_script_languages_container_ci.lib.ci_test import DBTestRunnerProtocol @@ -22,10 +24,15 @@ def db_test_runner(self) -> DBTestRunnerProtocol: raise NotImplementedError() def test(self, db_test_runner, test_container, flavor_path): - result = db_test_runner.run(flavor_path=(flavor_path,), test_folder=(), - release_goal=('release',), workers=7, - docker_username=None, docker_password=None, - test_container_folder=test_container) + result = db_test_runner.run( + flavor_path=(flavor_path,), + test_folder=(), + release_goal=("release",), + workers=7, + docker_username=None, + docker_password=None, + test_container_folder=test_container, + ) assert result.tests_are_ok and result.command_line_output_path.exists() @@ -36,11 +43,15 @@ def db_test_runner(self) -> DBTestRunnerProtocol: raise NotImplementedError() def test(self, db_test_runner, test_container, flavor_path): - result = db_test_runner.run(flavor_path=(flavor_path,), workers=7, - test_folder=('test/linker_namespace_sanity',), - release_goal=('base_test_build_run',), - docker_username=None, docker_password=None, - test_container_folder=test_container) + result = db_test_runner.run( + flavor_path=(flavor_path,), + workers=7, + test_folder=("test/linker_namespace_sanity",), + release_goal=("base_test_build_run",), + docker_username=None, + docker_password=None, + test_container_folder=test_container, + ) assert result.tests_are_ok and result.command_line_output_path.exists() @@ -62,23 +73,34 @@ def db_test_runner(self) -> DBTestRunnerProtocol: raise NotImplementedError() def test(self, db_test_runner, test_container, flavor_path): - result = db_test_runner.run(flavor_path=(flavor_path,), test_folder=(), - release_goal=('release',), workers=7, - docker_username=None, docker_password=None, - test_container_folder=test_container) + result = db_test_runner.run( + flavor_path=(flavor_path,), + test_folder=(), + release_goal=("release",), + workers=7, + docker_username=None, + docker_password=None, + test_container_folder=test_container, + ) assert not result.tests_are_ok and result.command_line_output_path.exists() -class FailingRunDBTestFlavorLinkerNamespaceTestsContract(FailingRunDBTestFlavorContract): +class FailingRunDBTestFlavorLinkerNamespaceTestsContract( + FailingRunDBTestFlavorContract +): @pytest.fixture def db_test_runner(self) -> DBTestRunnerProtocol: raise NotImplementedError() def test(self, db_test_runner, test_container, flavor_path): - result = db_test_runner.run(flavor_path=(flavor_path,), workers=7, - test_folder=('linker_namespace_sanity',), - release_goal=('base_test_build_run',), - docker_username=None, docker_password=None, - test_container_folder=test_container) + result = db_test_runner.run( + flavor_path=(flavor_path,), + workers=7, + test_folder=("linker_namespace_sanity",), + release_goal=("base_test_build_run",), + docker_username=None, + docker_password=None, + test_container_folder=test_container, + ) assert not result.tests_are_ok and result.command_line_output_path.exists() diff --git a/test/integration_tests/test_ci_build.py b/test/integration_tests/test_ci_build.py index a39b95f..9444330 100644 --- a/test/integration_tests/test_ci_build.py +++ b/test/integration_tests/test_ci_build.py @@ -1,27 +1,35 @@ import logging import os from pathlib import Path +from test.asserts import not_raises import pytest from exasol_script_languages_container_ci.lib.ci_build import CIBuild -from test.asserts import not_raises testdata = [ - ("test_docker_build_repository", "test_commit_sha", "test_docker_build_repository", "test_commit_sha_"), - (None, "", "exasol/script-language-container", "") + ( + "test_docker_build_repository", + "test_commit_sha", + "test_docker_build_repository", + "test_commit_sha_", + ), + (None, "", "exasol/script-language-container", ""), ] @pytest.mark.parametrize( "input_docker_build_repository,input_commit_sha,expected_docker_build_repository,expected_source_tag_prefix", - testdata) -def test(input_docker_build_repository, - input_commit_sha, - expected_docker_build_repository, - expected_source_tag_prefix, - flavors_path, - test_containers_folder): + testdata, +) +def test( + input_docker_build_repository, + input_commit_sha, + expected_docker_build_repository, + expected_source_tag_prefix, + flavors_path, + test_containers_folder, +): test_type = "successful" flavor_path = str(flavors_path / test_type) test_container_folder = str(test_containers_folder / test_type) @@ -33,5 +41,5 @@ def test(input_docker_build_repository, build_docker_repository=input_docker_build_repository, docker_user=None, docker_password=None, - test_container_folder=test_container_folder + test_container_folder=test_container_folder, ) diff --git a/test/integration_tests/test_ci_export.py b/test/integration_tests/test_ci_export.py index cbf790c..31be4e0 100644 --- a/test/integration_tests/test_ci_export.py +++ b/test/integration_tests/test_ci_export.py @@ -11,5 +11,7 @@ def test(flavors_path): CIExport().export(flavor_path=(flavor_path,), export_path=temp_dir) temp_dir_path = Path(temp_dir) temp_dir_content = set(temp_dir_path.iterdir()) - files_start_with_flavor_name = [str(file.name).startswith(flavor_name) for file in temp_dir_content] + files_start_with_flavor_name = [ + str(file.name).startswith(flavor_name) for file in temp_dir_content + ] assert len(temp_dir_content) == 2 and all(files_start_with_flavor_name) diff --git a/test/integration_tests/test_ci_prepare.py b/test/integration_tests/test_ci_prepare.py index 6f70f99..e584b9a 100644 --- a/test/integration_tests/test_ci_prepare.py +++ b/test/integration_tests/test_ci_prepare.py @@ -8,9 +8,8 @@ from exasol_script_languages_container_ci.lib.ci_build import CIBuild from exasol_script_languages_container_ci.lib.ci_prepare import CIPrepare -def test(flavors_path, - test_containers_folder, - mock_settings_env_vars): + +def test(flavors_path, test_containers_folder, mock_settings_env_vars): test_type = "successful" flavor_path = str(flavors_path / test_type) test_container_folder = str(test_containers_folder / test_type) @@ -22,7 +21,7 @@ def test(flavors_path, build_docker_repository="input_docker_build_repository", docker_user=None, docker_password=None, - test_container_folder=test_container_folder + test_container_folder=test_container_folder, ) log_path = Path(os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME]) assert log_path.is_file() diff --git a/test/integration_tests/test_ci_push.py b/test/integration_tests/test_ci_push.py index e540ed0..685d9a7 100644 --- a/test/integration_tests/test_ci_push.py +++ b/test/integration_tests/test_ci_push.py @@ -1,6 +1,8 @@ from pathlib import Path -from exasol_integration_test_docker_environment.testing.docker_registry import LocalDockerRegistryContextManager +from exasol_integration_test_docker_environment.testing.docker_registry import ( + LocalDockerRegistryContextManager, +) from exasol_script_languages_container_ci.lib.ci_push import CIPush @@ -14,14 +16,16 @@ def test(flavors_path): target_docker_repository=registry.name, target_docker_tag_prefix="tag", docker_user=None, - docker_password=None + docker_password=None, ) - expected_images = \ - {'name': 'test_ci_push', - 'tags': [ - f'tag_{flavor_name}-base_test_build_run_GUA7R5J3UM27WOHJSQPX2OJNSIEKWCM5YF5GJXKKXZI53LZPV75Q', - f'tag_{flavor_name}-flavor_test_build_run_G2OIMXJ2S3VS2EUAQNW4KWQLX3B2C27XYZ2SDMF7TQRS3UMAUWJQ', - f'tag_{flavor_name}-release_MNWZZGSSFQ6VCLBDH7CZBEZC4K35QQBSLOW5DSYHF3DFFDX2OOZQ', - ]} - assert expected_images["name"] == registry.images["name"] \ - and set(expected_images["tags"]) == set(registry.images["tags"]) + expected_images = { + "name": "test_ci_push", + "tags": [ + f"tag_{flavor_name}-base_test_build_run_GUA7R5J3UM27WOHJSQPX2OJNSIEKWCM5YF5GJXKKXZI53LZPV75Q", + f"tag_{flavor_name}-flavor_test_build_run_G2OIMXJ2S3VS2EUAQNW4KWQLX3B2C27XYZ2SDMF7TQRS3UMAUWJQ", + f"tag_{flavor_name}-release_MNWZZGSSFQ6VCLBDH7CZBEZC4K35QQBSLOW5DSYHF3DFFDX2OOZQ", + ], + } + assert expected_images["name"] == registry.images["name"] and set( + expected_images["tags"] + ) == set(registry.images["tags"]) diff --git a/test/integration_tests/test_ci_security_scan.py b/test/integration_tests/test_ci_security_scan.py index ecc9c63..70cf9bd 100644 --- a/test/integration_tests/test_ci_security_scan.py +++ b/test/integration_tests/test_ci_security_scan.py @@ -1,34 +1,34 @@ +from test.matchers import file_exists_matcher from unittest.mock import call, create_autospec import pytest from exasol_script_languages_container_ci.lib.ci_security_scan import CISecurityScan -from exasol_script_languages_container_ci.lib.ci_step_output_printer import CIStepOutputPrinterProtocol -from test.matchers import file_exists_matcher +from exasol_script_languages_container_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinterProtocol, +) def test_successful_flavor(flavors_path, test_containers_folder): flavor_path = str(flavors_path / "successful") printer_mock = create_autospec(CIStepOutputPrinterProtocol) - CISecurityScan( - printer=printer_mock - ).run_security_scan( + CISecurityScan(printer=printer_mock).run_security_scan( flavor_path=(flavor_path,), ) assert printer_mock.mock_calls == [ call.print_file(file_exists_matcher()), - call.print_exasol_docker_images()] + call.print_exasol_docker_images(), + ] def test_failing_security_scan(flavors_path): flavor_path = str(flavors_path / "failing_security_scan") printer_mock = create_autospec(CIStepOutputPrinterProtocol) with pytest.raises(AssertionError, match="Some security scans not successful."): - CISecurityScan( - printer=printer_mock - ).run_security_scan( + CISecurityScan(printer=printer_mock).run_security_scan( flavor_path=(flavor_path,), ) assert printer_mock.mock_calls == [ call.print_file(file_exists_matcher()), - call.print_exasol_docker_images()] + call.print_exasol_docker_images(), + ] diff --git a/test/integration_tests/test_ci_test.py b/test/integration_tests/test_ci_test.py index 718f027..9f36534 100644 --- a/test/integration_tests/test_ci_test.py +++ b/test/integration_tests/test_ci_test.py @@ -1,9 +1,16 @@ +from test.contract_tests.test_ci_test import ( + FailingRunDBTestFlavorDBTestsContract, + FailingRunDBTestFlavorLinkerNamespaceTestsContract, + SuccessfulFlavorDBTestsContract, + SuccessfulFlavorLinkerNamespaceTestsContract, +) + import pytest -from exasol_script_languages_container_ci.lib.ci_test import DBTestRunner, DBTestRunnerProtocol -from test.contract_tests.test_ci_test import SuccessfulFlavorDBTestsContract, \ - SuccessfulFlavorLinkerNamespaceTestsContract, \ - FailingRunDBTestFlavorDBTestsContract, FailingRunDBTestFlavorLinkerNamespaceTestsContract +from exasol_script_languages_container_ci.lib.ci_test import ( + DBTestRunner, + DBTestRunnerProtocol, +) class TestSuccessfulFlavorDBTestsContract(SuccessfulFlavorDBTestsContract): @@ -13,7 +20,9 @@ def db_test_runner(self) -> DBTestRunnerProtocol: return DBTestRunner() -class TestSuccessfulFlavorLinkerNamespaceTestsContract(SuccessfulFlavorLinkerNamespaceTestsContract): +class TestSuccessfulFlavorLinkerNamespaceTestsContract( + SuccessfulFlavorLinkerNamespaceTestsContract +): @pytest.fixture def db_test_runner(self) -> DBTestRunnerProtocol: @@ -27,7 +36,9 @@ def db_test_runner(self) -> DBTestRunnerProtocol: return DBTestRunner() -class TestFailingRunDBTestFlavorLinkerNamespaceTestsContract(FailingRunDBTestFlavorLinkerNamespaceTestsContract): +class TestFailingRunDBTestFlavorLinkerNamespaceTestsContract( + FailingRunDBTestFlavorLinkerNamespaceTestsContract +): @pytest.fixture def db_test_runner(self) -> DBTestRunnerProtocol: diff --git a/test/mock_cast.py b/test/mock_cast.py index c6fc304..3a3866e 100644 --- a/test/mock_cast.py +++ b/test/mock_cast.py @@ -1,4 +1,4 @@ -from typing import cast, Any +from typing import Any, cast from unittest.mock import Mock diff --git a/test/unit_tests/ci_calls.py b/test/unit_tests/ci_calls.py index 9a70505..588fcd2 100644 --- a/test/unit_tests/ci_calls.py +++ b/test/unit_tests/ci_calls.py @@ -1,6 +1,5 @@ -from unittest.mock import call - from test.unit_tests.test_env import test_env +from unittest.mock import call def prepare(): @@ -8,29 +7,36 @@ def prepare(): def build_ci_call(force_rebuild: bool): - return call.build(flavor_path=("flavors/TEST_FLAVOR",), - rebuild=force_rebuild, - build_docker_repository=test_env.docker_build_repo, - commit_sha=test_env.commit_sha, - docker_user=test_env.docker_user, - docker_password=test_env.docker_pwd, - test_container_folder='test_container') + return call.build( + flavor_path=("flavors/TEST_FLAVOR",), + rebuild=force_rebuild, + build_docker_repository=test_env.docker_build_repo, + commit_sha=test_env.commit_sha, + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + test_container_folder="test_container", + ) + def build_release_call(): - return call.build(flavor_path=("flavors/TEST_FLAVOR",), - rebuild=True, - build_docker_repository=None, - commit_sha="", - docker_user=None, - docker_password=None, - test_container_folder='test_container') + return call.build( + flavor_path=("flavors/TEST_FLAVOR",), + rebuild=True, + build_docker_repository=None, + commit_sha="", + docker_user=None, + docker_password=None, + test_container_folder="test_container", + ) def run_db_test_call(): - return call.execute_tests(flavor_path=("flavors/TEST_FLAVOR",), - docker_user=test_env.docker_user, - docker_password=test_env.docker_pwd, - test_container_folder='test_container') + return call.execute_tests( + flavor_path=("flavors/TEST_FLAVOR",), + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + test_container_folder="test_container", + ) def security_scan_call(): @@ -38,30 +44,38 @@ def security_scan_call(): def push_build_repo_with_sha_call(): - return call.push(flavor_path=("flavors/TEST_FLAVOR",), - target_docker_repository=test_env.docker_build_repo, - target_docker_tag_prefix=test_env.commit_sha, - docker_user=test_env.docker_user, - docker_password=test_env.docker_pwd) + return call.push( + flavor_path=("flavors/TEST_FLAVOR",), + target_docker_repository=test_env.docker_build_repo, + target_docker_tag_prefix=test_env.commit_sha, + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + ) def push_build_repo_without_sha_call(): - return call.push(flavor_path=("flavors/TEST_FLAVOR",), - target_docker_repository=test_env.docker_build_repo, - target_docker_tag_prefix="", - docker_user=test_env.docker_user, - docker_password=test_env.docker_pwd) + return call.push( + flavor_path=("flavors/TEST_FLAVOR",), + target_docker_repository=test_env.docker_build_repo, + target_docker_tag_prefix="", + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + ) def push_release_repo(): - return call.push(flavor_path=("flavors/TEST_FLAVOR",), - target_docker_repository=test_env.docker_release_repo, - target_docker_tag_prefix="", - docker_user=test_env.docker_user, - docker_password=test_env.docker_pwd) + return call.push( + flavor_path=("flavors/TEST_FLAVOR",), + target_docker_repository=test_env.docker_release_repo, + target_docker_tag_prefix="", + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + ) def release_upload(): - return call.release_upload(flavor_path=('flavors/TEST_FLAVOR',), - source_repo_url='https://github.com/test_source_repo_url', - release_id=123) + return call.release_upload( + flavor_path=("flavors/TEST_FLAVOR",), + source_repo_url="https://github.com/test_source_repo_url", + release_id=123, + ) diff --git a/test/unit_tests/test_asset_uploader.py b/test/unit_tests/test_asset_uploader.py index 349f78b..fdcc5f1 100644 --- a/test/unit_tests/test_asset_uploader.py +++ b/test/unit_tests/test_asset_uploader.py @@ -1,17 +1,22 @@ from pathlib import Path from tempfile import TemporaryDirectory +from test.mock_cast import mock_cast from typing import Union -from unittest.mock import create_autospec, MagicMock +from unittest.mock import MagicMock, create_autospec from exasol_script_languages_container_ci.lib.asset_uploader import AssetUploader -from exasol_script_languages_container_ci.lib.github_release_asset_uploader import GithubReleaseAssetUploader -from test.mock_cast import mock_cast +from exasol_script_languages_container_ci.lib.github_release_asset_uploader import ( + GithubReleaseAssetUploader, +) def test(): - github_release_asset_uploader_mock: Union[MagicMock, GithubReleaseAssetUploader] = \ + github_release_asset_uploader_mock: Union[MagicMock, GithubReleaseAssetUploader] = ( create_autospec(GithubReleaseAssetUploader) - asset_uploader = AssetUploader(release_asset_uploader=github_release_asset_uploader_mock) + ) + asset_uploader = AssetUploader( + release_asset_uploader=github_release_asset_uploader_mock + ) with TemporaryDirectory() as temp_dir: artifact_path = Path(temp_dir) release_artifact = artifact_path / "test_artifact.txt" @@ -23,12 +28,12 @@ def test(): file_suffix=".txt", content_type="test_content_type", label_prefix="test_label_prefix", - artifact_path=temp_dir + artifact_path=temp_dir, ) mock_cast(github_release_asset_uploader_mock.upload).assert_called_once_with( archive_path=str(release_artifact), - content_type='test_content_type', - label='test_label_prefix test_artifact', + content_type="test_content_type", + label="test_label_prefix test_artifact", release_id=123, - repo_id='test_repo_id' + repo_id="test_repo_id", ) diff --git a/test/unit_tests/test_ci.py b/test/unit_tests/test_ci.py index 63a9255..dbec7ed 100644 --- a/test/unit_tests/test_ci.py +++ b/test/unit_tests/test_ci.py @@ -1,3 +1,5 @@ +from test.unit_tests import ci_calls +from test.unit_tests.test_env import test_env from typing import Union from unittest.mock import Mock @@ -8,58 +10,67 @@ from exasol_script_languages_container_ci.lib.ci_push import CIPush from exasol_script_languages_container_ci.lib.ci_security_scan import CISecurityScan from exasol_script_languages_container_ci.lib.ci_test import CIExecuteTest -from test.unit_tests import ci_calls -from test.unit_tests.test_env import test_env # Testdata contain tuples of (branch, list(calls to CICommands)) # The goal is to test that for specific branches the correct list of calls (with expected arguments) is passed to the CICommands testdata_ci = [ - ("refs/heads/feature/test_branch", [ - ci_calls.prepare(), - ci_calls.build_ci_call(force_rebuild=False), - ci_calls.run_db_test_call(), - ci_calls.security_scan_call(), - ci_calls.push_build_repo_with_sha_call(), - ci_calls.push_build_repo_without_sha_call()] - ), - ("refs/heads/rebuild/feature/test_branch", [ - ci_calls.prepare(), - ci_calls.build_ci_call(force_rebuild=True), - ci_calls.run_db_test_call(), - ci_calls.security_scan_call(), - ci_calls.push_build_repo_with_sha_call(), - ci_calls.push_build_repo_without_sha_call()] - ), - ("refs/heads/master", [ - ci_calls.prepare(), - ci_calls.build_ci_call(force_rebuild=True), - ci_calls.run_db_test_call(), - ci_calls.security_scan_call(), - ci_calls.push_build_repo_with_sha_call(), - ci_calls.push_build_repo_without_sha_call(), - ci_calls.push_release_repo() - ] - ), - ("refs/heads/main", [ - ci_calls.prepare(), - ci_calls.build_ci_call(force_rebuild=True), - ci_calls.run_db_test_call(), - ci_calls.security_scan_call(), - ci_calls.push_build_repo_with_sha_call(), - ci_calls.push_build_repo_without_sha_call(), - ci_calls.push_release_repo() - ] - ), - ("refs/heads/develop", [ - ci_calls.prepare(), - ci_calls.build_ci_call(force_rebuild=True), - ci_calls.run_db_test_call(), - ci_calls.security_scan_call(), - ci_calls.push_build_repo_with_sha_call(), - ci_calls.push_build_repo_without_sha_call() - ] - ), - + ( + "refs/heads/feature/test_branch", + [ + ci_calls.prepare(), + ci_calls.build_ci_call(force_rebuild=False), + ci_calls.run_db_test_call(), + ci_calls.security_scan_call(), + ci_calls.push_build_repo_with_sha_call(), + ci_calls.push_build_repo_without_sha_call(), + ], + ), + ( + "refs/heads/rebuild/feature/test_branch", + [ + ci_calls.prepare(), + ci_calls.build_ci_call(force_rebuild=True), + ci_calls.run_db_test_call(), + ci_calls.security_scan_call(), + ci_calls.push_build_repo_with_sha_call(), + ci_calls.push_build_repo_without_sha_call(), + ], + ), + ( + "refs/heads/master", + [ + ci_calls.prepare(), + ci_calls.build_ci_call(force_rebuild=True), + ci_calls.run_db_test_call(), + ci_calls.security_scan_call(), + ci_calls.push_build_repo_with_sha_call(), + ci_calls.push_build_repo_without_sha_call(), + ci_calls.push_release_repo(), + ], + ), + ( + "refs/heads/main", + [ + ci_calls.prepare(), + ci_calls.build_ci_call(force_rebuild=True), + ci_calls.run_db_test_call(), + ci_calls.security_scan_call(), + ci_calls.push_build_repo_with_sha_call(), + ci_calls.push_build_repo_without_sha_call(), + ci_calls.push_release_repo(), + ], + ), + ( + "refs/heads/develop", + [ + ci_calls.prepare(), + ci_calls.build_ci_call(force_rebuild=True), + ci_calls.run_db_test_call(), + ci_calls.security_scan_call(), + ci_calls.push_build_repo_with_sha_call(), + ci_calls.push_build_repo_without_sha_call(), + ], + ), ] @@ -73,19 +84,23 @@ def test_branches(branch, git_access_mock, expected_calls, build_config): 4. Push to docker build repo (with and without sha) 5. Optionally: Push to docker release repo """ - ci_commands_mock: Union[CIBuild, CIPush, CIExecuteTest, CISecurityScan, Mock] = Mock() - ci(flavor="TEST_FLAVOR", - branch_name=branch, - docker_user=test_env.docker_user, - docker_password=test_env.docker_pwd, - docker_build_repository=test_env.docker_build_repo, - docker_release_repository=test_env.docker_release_repo, - commit_sha=test_env.commit_sha, - build_config=build_config, - git_access=git_access_mock, - ci_build=ci_commands_mock, - ci_push=ci_commands_mock, - ci_execute_tests=ci_commands_mock, - ci_security_scan=ci_commands_mock, - ci_prepare=ci_commands_mock) + ci_commands_mock: Union[CIBuild, CIPush, CIExecuteTest, CISecurityScan, Mock] = ( + Mock() + ) + ci( + flavor="TEST_FLAVOR", + branch_name=branch, + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + docker_build_repository=test_env.docker_build_repo, + docker_release_repository=test_env.docker_release_repo, + commit_sha=test_env.commit_sha, + build_config=build_config, + git_access=git_access_mock, + ci_build=ci_commands_mock, + ci_push=ci_commands_mock, + ci_execute_tests=ci_commands_mock, + ci_security_scan=ci_commands_mock, + ci_prepare=ci_commands_mock, + ) assert ci_commands_mock.mock_calls == expected_calls diff --git a/test/unit_tests/test_ci_prepare.py b/test/unit_tests/test_ci_prepare.py index e889cef..838e5ee 100644 --- a/test/unit_tests/test_ci_prepare.py +++ b/test/unit_tests/test_ci_prepare.py @@ -3,14 +3,13 @@ from unittest import mock import pytest -from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY +from exasol_integration_test_docker_environment.cli.options.system_options import ( + DEFAULT_OUTPUT_DIRECTORY, +) from exasol_integration_test_docker_environment.lib.base import luigi_log_config from exasol_script_languages_container_ci.lib.ci_prepare import CIPrepare - - - EXPECTED_LOG_PARENT_DIRECTORY = Path(DEFAULT_OUTPUT_DIRECTORY) / "jobs" / "logs" EXPECTED_LOG_FILE = EXPECTED_LOG_PARENT_DIRECTORY / "main.log" @@ -20,7 +19,9 @@ def test_ci_prepare_log_environment_variable_is_set(mock_settings_env_vars): assert luigi_log_config.LOG_ENV_VARIABLE_NAME in os.environ -def test_ci_prepare_log_environment_variable_is_set_to_the_correct_path(mock_settings_env_vars): +def test_ci_prepare_log_environment_variable_is_set_to_the_correct_path( + mock_settings_env_vars, +): CIPrepare().prepare() expected_path = str(EXPECTED_LOG_FILE.absolute()) assert os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME] == expected_path @@ -40,7 +41,10 @@ def test_ci_prepare_log_path_file_doesnt_exist(mock_settings_env_vars): def test_ci_prepare_log_path_parent_directory_exist(mock_settings_env_vars): EXPECTED_LOG_PARENT_DIRECTORY.mkdir(parents=True) CIPrepare().prepare() - assert Path(os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME]).parent == EXPECTED_LOG_PARENT_DIRECTORY.absolute() + assert ( + Path(os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME]).parent + == EXPECTED_LOG_PARENT_DIRECTORY.absolute() + ) def test_ci_prepare_log_path_file_exists(mock_settings_env_vars): @@ -51,4 +55,6 @@ def test_ci_prepare_log_path_file_exists(mock_settings_env_vars): CIPrepare().prepare() actual_path = Path(os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME]) actual_value = actual_path.read_text() - assert actual_value == expected_value and actual_path == EXPECTED_LOG_FILE.absolute() + assert ( + actual_value == expected_value and actual_path == EXPECTED_LOG_FILE.absolute() + ) diff --git a/test/unit_tests/test_ci_test.py b/test/unit_tests/test_ci_test.py index 1114905..b69f951 100644 --- a/test/unit_tests/test_ci_test.py +++ b/test/unit_tests/test_ci_test.py @@ -1,14 +1,19 @@ from contextlib import suppress from pathlib import Path +from test.mock_cast import mock_cast from typing import Union -from unittest.mock import call, create_autospec, MagicMock +from unittest.mock import MagicMock, call, create_autospec import pytest from exasol.slc.models.test_result import AllTestsResult -from exasol_script_languages_container_ci.lib.ci_step_output_printer import CIStepOutputPrinterProtocol -from exasol_script_languages_container_ci.lib.ci_test import CIExecuteTest, DBTestRunnerProtocol -from test.mock_cast import mock_cast +from exasol_script_languages_container_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinterProtocol, +) +from exasol_script_languages_container_ci.lib.ci_test import ( + CIExecuteTest, + DBTestRunnerProtocol, +) class BaseCIExecuteTest: @@ -24,13 +29,14 @@ def base_setup(self, db_test_runner): self.flavor_path = "test_flavor" self.test_container_folder = "test_container_folder" self.ci_execute_test = CIExecuteTest( - printer=self.printer_mock, - db_test_runner=db_test_runner + printer=self.printer_mock, db_test_runner=db_test_runner ) @staticmethod def create_all_tests_result_mock(tests_are_ok: bool): - all_tests_result: Union[MagicMock, AllTestsResult] = create_autospec(AllTestsResult) + all_tests_result: Union[MagicMock, AllTestsResult] = create_autospec( + AllTestsResult + ) all_tests_result.tests_are_ok = tests_are_ok all_tests_result.command_line_output_path = create_autospec(Path) return all_tests_result @@ -46,12 +52,24 @@ def execute_tests(self): @pytest.fixture() def run_db_tests_calls(self): return [ - call.run(flavor_path=(self.flavor_path,), test_folder=(), release_goal=('release',), workers=7, - docker_username=None, docker_password=None, - test_container_folder=self.test_container_folder), - call.run(flavor_path=(self.flavor_path,), workers=7, test_folder=('test/linker_namespace_sanity',), - release_goal=('base_test_build_run',), docker_username=None, docker_password=None, - test_container_folder=self.test_container_folder) + call.run( + flavor_path=(self.flavor_path,), + test_folder=(), + release_goal=("release",), + workers=7, + docker_username=None, + docker_password=None, + test_container_folder=self.test_container_folder, + ), + call.run( + flavor_path=(self.flavor_path,), + workers=7, + test_folder=("test/linker_namespace_sanity",), + release_goal=("base_test_build_run",), + docker_username=None, + docker_password=None, + test_container_folder=self.test_container_folder, + ), ] @@ -59,17 +77,26 @@ class TestSuccessfulFlavor(BaseCIExecuteTest): @pytest.fixture def complete_setup(self, base_setup): - self.db_tests_all_tests_result = self.create_all_tests_result_mock(tests_are_ok=True) - self.linker_namespace_tests_all_tests_result = self.create_all_tests_result_mock(tests_are_ok=True) - mock_cast(self.db_test_runner_mock.run).side_effect = [self.db_tests_all_tests_result, - self.linker_namespace_tests_all_tests_result] + self.db_tests_all_tests_result = self.create_all_tests_result_mock( + tests_are_ok=True + ) + self.linker_namespace_tests_all_tests_result = ( + self.create_all_tests_result_mock(tests_are_ok=True) + ) + mock_cast(self.db_test_runner_mock.run).side_effect = [ + self.db_tests_all_tests_result, + self.linker_namespace_tests_all_tests_result, + ] def test_ci_step_output_printer_call(self, complete_setup): self.execute_tests() assert self.printer_mock.mock_calls == [ call.print_file(self.db_tests_all_tests_result.command_line_output_path), - call.print_file(self.linker_namespace_tests_all_tests_result.command_line_output_path), - call.print_exasol_docker_images()] + call.print_file( + self.linker_namespace_tests_all_tests_result.command_line_output_path + ), + call.print_exasol_docker_images(), + ] def test_db_test_runner_calls(self, complete_setup, run_db_tests_calls): self.execute_tests() @@ -78,18 +105,18 @@ def test_db_test_runner_calls(self, complete_setup, run_db_tests_calls): class TestFailingRunDBTestFlavor(BaseCIExecuteTest): - @pytest.fixture( - params=[ - (False, True), - (True, False), - (False, False) - ] - ) + @pytest.fixture(params=[(False, True), (True, False), (False, False)]) def complete_setup(self, base_setup, request): - self.db_tests_all_tests_result = self.create_all_tests_result_mock(tests_are_ok=request.param[0]) - self.linker_namespace_tests_all_tests_result = self.create_all_tests_result_mock(tests_are_ok=request.param[1]) - mock_cast(self.db_test_runner_mock.run).side_effect = [self.db_tests_all_tests_result, - self.linker_namespace_tests_all_tests_result] + self.db_tests_all_tests_result = self.create_all_tests_result_mock( + tests_are_ok=request.param[0] + ) + self.linker_namespace_tests_all_tests_result = ( + self.create_all_tests_result_mock(tests_are_ok=request.param[1]) + ) + mock_cast(self.db_test_runner_mock.run).side_effect = [ + self.db_tests_all_tests_result, + self.linker_namespace_tests_all_tests_result, + ] @pytest.fixture def run_suppress_exception(self, complete_setup): @@ -103,8 +130,11 @@ def test_raises(self, complete_setup): def test_ci_step_output_printer_call(self, run_suppress_exception): assert self.printer_mock.mock_calls == [ call.print_file(self.db_tests_all_tests_result.command_line_output_path), - call.print_file(self.linker_namespace_tests_all_tests_result.command_line_output_path), - call.print_exasol_docker_images()] + call.print_file( + self.linker_namespace_tests_all_tests_result.command_line_output_path + ), + call.print_exasol_docker_images(), + ] def test_db_test_runner_calls(self, run_suppress_exception, run_db_tests_calls): assert self.db_test_runner_mock.mock_calls == run_db_tests_calls diff --git a/test/unit_tests/test_config_data_model.py b/test/unit_tests/test_config_data_model.py index 9ac66ad..dfea23d 100644 --- a/test/unit_tests/test_config_data_model.py +++ b/test/unit_tests/test_config_data_model.py @@ -1,25 +1,18 @@ - - import pytest -from exasol_script_languages_container_ci.lib.config.config_data_model import Config, Build, Ignore, Release +from exasol_script_languages_container_ci.lib.config.config_data_model import ( + Build, + Config, + Ignore, + Release, +) @pytest.fixture def expected_config() -> Config: config = Config( - build=Build( - ignore=Ignore( - paths=[ - "a/b/c", - "e/f/g" - ] - ), - base_branch="" - ), - release=Release( - timeout_in_minutes=1 - ) + build=Build(ignore=Ignore(paths=["a/b/c", "e/f/g"]), base_branch=""), + release=Release(timeout_in_minutes=1), ) return config diff --git a/test/unit_tests/test_config_data_model_generator.py b/test/unit_tests/test_config_data_model_generator.py index 9825f77..2aad540 100644 --- a/test/unit_tests/test_config_data_model_generator.py +++ b/test/unit_tests/test_config_data_model_generator.py @@ -1,14 +1,18 @@ import sys -from exasol_script_languages_container_ci.lib.config.data_model_generator import CONFIG_DATA_MODEL_FILE_NAME, \ - generate_config_data_model +from exasol_script_languages_container_ci.lib.config.data_model_generator import ( + CONFIG_DATA_MODEL_FILE_NAME, + generate_config_data_model, +) def test_loading_generated_module(tmp_path): config_data_model_file = tmp_path / CONFIG_DATA_MODEL_FILE_NAME generate_config_data_model(config_data_model_file) module = load_module(config_data_model_file) - assert {"Config", "Build", "Release", "Ignore", "Release"}.issubset(module.__dict__.keys()) + assert {"Config", "Build", "Release", "Ignore", "Release"}.issubset( + module.__dict__.keys() + ) def test_using_generated_module(tmp_path, expected_json_config): @@ -23,23 +27,16 @@ def test_using_generated_module(tmp_path, expected_json_config): def create_config(module): config = module.Config( build=module.Build( - ignore=module.Ignore( - paths=[ - "a/b/c", - "e/f/g" - ] - ), - base_branch="" + ignore=module.Ignore(paths=["a/b/c", "e/f/g"]), base_branch="" ), - release=module.Release( - timeout_in_minutes=1 - ) + release=module.Release(timeout_in_minutes=1), ) return config def load_module(config_data_model_file): import importlib.util + module_name = "test_create_model_can_be_imported" spec = importlib.util.spec_from_file_location(module_name, config_data_model_file) module = importlib.util.module_from_spec(spec) diff --git a/test/unit_tests/test_env.py b/test/unit_tests/test_env.py index b620ef4..1c623d6 100644 --- a/test/unit_tests/test_env.py +++ b/test/unit_tests/test_env.py @@ -1,4 +1,3 @@ - class TestEnv: docker_user = "test_docker_user" docker_pwd = "test_docker_pwd" diff --git a/test/unit_tests/test_ignore_folders.py b/test/unit_tests/test_ignore_folders.py index eaf2b2e..e670062 100644 --- a/test/unit_tests/test_ignore_folders.py +++ b/test/unit_tests/test_ignore_folders.py @@ -2,13 +2,18 @@ from pathlib import Path from typing import List +import git import pytest from _pytest.tmpdir import TempPathFactory from exasol_script_languages_container_ci.lib.ci import check_if_need_to_build -from exasol_script_languages_container_ci.lib.config.config_data_model import Config, Build, Release, Ignore +from exasol_script_languages_container_ci.lib.config.config_data_model import ( + Build, + Config, + Ignore, + Release, +) from exasol_script_languages_container_ci.lib.git_access import GitAccess -import git def commit_base(repo: git.Repo, repo_path: Path) -> None: @@ -16,7 +21,7 @@ def commit_base(repo: git.Repo, repo_path: Path) -> None: Create dummy commit on base branch with "something" """ (repo_path / "something").parent.mkdir(parents=True, exist_ok=True) - open(repo_path / "something", 'w').close() + open(repo_path / "something", "w").close() repo.index.add([str(repo_path / "something")]) repo.index.commit("Base commit") assert repo.active_branch.name == "master" or repo.active_branch.name == "main" @@ -24,8 +29,13 @@ def commit_base(repo: git.Repo, repo_path: Path) -> None: repo.active_branch.rename("master") -def commit_files(branch_name: str, repo: git.Repo, repo_path: Path, - files_to_commit: List[List[str]], commit_message: str) -> None: +def commit_files( + branch_name: str, + repo: git.Repo, + repo_path: Path, + files_to_commit: List[List[str]], + commit_message: str, +) -> None: """ Create empty given files (param files_to_commit) and commit them to a "Dummy commit" """ @@ -35,15 +45,17 @@ def commit_files(branch_name: str, repo: git.Repo, repo_path: Path, for file_list in files_to_commit: for file in file_list: (repo_path / file).parent.mkdir(parents=True, exist_ok=True) - open(repo_path / file, 'w').close() + open(repo_path / file, "w").close() repo.index.add([str(repo_path / file)]) repo.index.commit(commit_message) @pytest.fixture def build_config() -> Config: - config = Config(build=Build(base_branch="master", ignore=Ignore(paths=["doc", "githooks"])), - release=Release(timeout_in_minutes=10)) + config = Config( + build=Build(base_branch="master", ignore=Ignore(paths=["doc", "githooks"])), + release=Release(timeout_in_minutes=10), + ) return config @@ -51,44 +63,102 @@ def build_config() -> Config: TEST_DATA = [ # If the last commit contains files not included in the ignore-path list, the build must run - ("last_commit_not_ignore_path_build_must_run", "refs/heads/feature_branch", - [["flavors/flavor_abc/build_steps.py", "doc/something", "src/udfclient.cpp"]], "message", True), + ( + "last_commit_not_ignore_path_build_must_run", + "refs/heads/feature_branch", + [["flavors/flavor_abc/build_steps.py", "doc/something", "src/udfclient.cpp"]], + "message", + True, + ), # If there are 2 commits, and the last only contains files in the ignore-list, but the first contains # files not included in the ignore-path list, the build must run - ("commit_before_last_commit_not_ignore_path_build_must_run", "refs/heads/feature_branch", - [["flavors/flavor_abc/build_steps.py", "doc/something", "src/udfclient.cpp"], ["doc/something"]], "message", True), + ( + "commit_before_last_commit_not_ignore_path_build_must_run", + "refs/heads/feature_branch", + [ + ["flavors/flavor_abc/build_steps.py", "doc/something", "src/udfclient.cpp"], + ["doc/something"], + ], + "message", + True, + ), # If last commit(s) contain only files included in the ignore-path-list or another flavor the build must not run - ("last_commit_ignore_path_or_another_flavor_build_must_not_run", "refs/heads/feature_branch", - [["flavors/flavor_abc/build_steps.py", "doc/something"]], "message", False), + ( + "last_commit_ignore_path_or_another_flavor_build_must_not_run", + "refs/heads/feature_branch", + [["flavors/flavor_abc/build_steps.py", "doc/something"]], + "message", + False, + ), # If last commit message contains "[rebuild]" the build should always trigger - ("rebuild_in_last_commit_msg_build_must_run", "refs/heads/feature_branch", - [["flavors/flavor_abc/build_steps.py", "doc/something"]], "message [rebuild]", True), + ( + "rebuild_in_last_commit_msg_build_must_run", + "refs/heads/feature_branch", + [["flavors/flavor_abc/build_steps.py", "doc/something"]], + "message [rebuild]", + True, + ), # Affected files on current flavor should trigger a build - ("changes_in_current_flavor_build_must_run", "refs/heads/feature_branch", - [[f"flavors/{TEST_FLAVOR}/build_steps.py", "doc/something"]], "message", True), + ( + "changes_in_current_flavor_build_must_run", + "refs/heads/feature_branch", + [[f"flavors/{TEST_FLAVOR}/build_steps.py", "doc/something"]], + "message", + True, + ), # If there are 2 commits, and the last only contains files in the ignore-list, but the first contains # files of the current flavor, the build must run - ("changes_in_current_flavor_before_last_commit_build_must_run", "refs/heads/feature_branch", - [[f"flavors/{TEST_FLAVOR}/build_steps.py"], ["flavors/flavor_abc/build_steps.py"]], "message", True), - ("develop_must_always_run", "refs/heads/develop", [["doc/something"]], "message", True), + ( + "changes_in_current_flavor_before_last_commit_build_must_run", + "refs/heads/feature_branch", + [ + [f"flavors/{TEST_FLAVOR}/build_steps.py"], + ["flavors/flavor_abc/build_steps.py"], + ], + "message", + True, + ), + ( + "develop_must_always_run", + "refs/heads/develop", + [["doc/something"]], + "message", + True, + ), # Even if folder should be ignored, in case of develop branch we always expect to run - ("master_must_always_run", "refs/heads/master", [["doc/something"]], "message", True), + ( + "master_must_always_run", + "refs/heads/master", + [["doc/something"]], + "message", + True, + ), # Even if folder should be ignored, in case of master branch we always expect to run ("main_must_always_run", "refs/heads/main", [["doc/something"]], "message", True), # Even if folder should be ignored, in case of main branch we always expect to run - ("rebuild_must_always_run", "refs/heads/rebuild/feature_branch", [["doc/something"]], "message", True), + ( + "rebuild_must_always_run", + "refs/heads/rebuild/feature_branch", + [["doc/something"]], + "message", + True, + ), # Even if folder should be ignored, in case of rebuild/* branch we always expect to run ] -@pytest.mark.parametrize("test_name, branch_name, files_to_commit,commit_message, expected_result", TEST_DATA) -def test_ignore_folder_should_run_ci(test_name: str, - branch_name: str, - tmp_test_dir, - build_config: Config, - files_to_commit, - commit_message: str, - expected_result: bool): +@pytest.mark.parametrize( + "test_name, branch_name, files_to_commit,commit_message, expected_result", TEST_DATA +) +def test_ignore_folder_should_run_ci( + test_name: str, + branch_name: str, + tmp_test_dir, + build_config: Config, + files_to_commit, + commit_message: str, + expected_result: bool, +): """ This test creates a temporary git repository, commits the given file list (files_for_commit), then runs ci.check_if_need_to_build() and checks if it returned the expected result @@ -96,4 +166,7 @@ def test_ignore_folder_should_run_ci(test_name: str, repo_path = Path(tmp_test_dir) tmp_repo = git.Repo.init(repo_path) commit_files(branch_name, tmp_repo, repo_path, files_to_commit, commit_message) - assert check_if_need_to_build(branch_name, build_config, TEST_FLAVOR, GitAccess()) == expected_result + assert ( + check_if_need_to_build(branch_name, build_config, TEST_FLAVOR, GitAccess()) + == expected_result + ) diff --git a/test/unit_tests/test_release.py b/test/unit_tests/test_release.py index d97c7bc..6e34dfe 100644 --- a/test/unit_tests/test_release.py +++ b/test/unit_tests/test_release.py @@ -1,5 +1,7 @@ +from test.unit_tests import ci_calls +from test.unit_tests.test_env import test_env from typing import Union -from unittest.mock import create_autospec, MagicMock, Mock +from unittest.mock import MagicMock, Mock, create_autospec import pytest @@ -12,29 +14,29 @@ from exasol_script_languages_container_ci.lib.release import release from exasol_script_languages_container_ci.lib.release_uploader import ReleaseUploader -from test.unit_tests import ci_calls - -from test.unit_tests.test_env import test_env - # Testdata contain tuples of (dry_run, list(calls to CICommands)) testdata_ci = [ - (True, [ - ci_calls.prepare(), - ci_calls.build_release_call(), - ci_calls.run_db_test_call(), - ci_calls.security_scan_call(), - ci_calls.release_upload() - ] - ), - (False, [ - ci_calls.prepare(), - ci_calls.build_release_call(), - ci_calls.run_db_test_call(), - ci_calls.security_scan_call(), - ci_calls.push_release_repo(), - ci_calls.release_upload() - ] - ), + ( + True, + [ + ci_calls.prepare(), + ci_calls.build_release_call(), + ci_calls.run_db_test_call(), + ci_calls.security_scan_call(), + ci_calls.release_upload(), + ], + ), + ( + False, + [ + ci_calls.prepare(), + ci_calls.build_release_call(), + ci_calls.run_db_test_call(), + ci_calls.security_scan_call(), + ci_calls.push_release_repo(), + ci_calls.release_upload(), + ], + ), ] @@ -48,19 +50,23 @@ def test(is_dry_run: bool, expected_calls, build_config: Config): 4. Push to docker release repo (only without dry-run) 5. Upload release to GitHub """ - ci_commands_mock: Union[CIBuild, CIPush, CIExecuteTest, CISecurityScan, ReleaseUploader, Mock] = Mock() - release(flavor="TEST_FLAVOR", - docker_user=test_env.docker_user, - docker_password=test_env.docker_pwd, - docker_release_repository=test_env.docker_release_repo, - source_repo_url="https://github.com/test_source_repo_url", - build_config=build_config, - release_id=123, - is_dry_run=is_dry_run, - release_uploader=ci_commands_mock, - ci_build=ci_commands_mock, - ci_push=ci_commands_mock, - ci_execute_tests=ci_commands_mock, - ci_security_scan=ci_commands_mock, - ci_prepare=ci_commands_mock) + ci_commands_mock: Union[ + CIBuild, CIPush, CIExecuteTest, CISecurityScan, ReleaseUploader, Mock + ] = Mock() + release( + flavor="TEST_FLAVOR", + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + docker_release_repository=test_env.docker_release_repo, + source_repo_url="https://github.com/test_source_repo_url", + build_config=build_config, + release_id=123, + is_dry_run=is_dry_run, + release_uploader=ci_commands_mock, + ci_build=ci_commands_mock, + ci_push=ci_commands_mock, + ci_execute_tests=ci_commands_mock, + ci_security_scan=ci_commands_mock, + ci_prepare=ci_commands_mock, + ) assert ci_commands_mock.mock_calls == expected_calls diff --git a/test/unit_tests/test_release_uploader.py b/test/unit_tests/test_release_uploader.py index 25394b5..05e64ab 100644 --- a/test/unit_tests/test_release_uploader.py +++ b/test/unit_tests/test_release_uploader.py @@ -1,16 +1,20 @@ +from test.mock_cast import mock_cast from typing import Union -from unittest.mock import create_autospec, MagicMock, call, patch, Mock +from unittest.mock import MagicMock, Mock, call, create_autospec, patch from exasol_script_languages_container_ci.lib.asset_uploader import AssetUploader from exasol_script_languages_container_ci.lib.ci_export import CIExport from exasol_script_languages_container_ci.lib.release_uploader import ReleaseUploader -from test.mock_cast import mock_cast -@patch("exasol_script_languages_container_ci.lib.release_uploader.TemporaryDirectory", - autospec=True) +@patch( + "exasol_script_languages_container_ci.lib.release_uploader.TemporaryDirectory", + autospec=True, +) def test(temp_dir_mock): - asset_uploader_mock: Union[MagicMock, AssetUploader] = create_autospec(AssetUploader) + asset_uploader_mock: Union[MagicMock, AssetUploader] = create_autospec( + AssetUploader + ) ci_export_mock: Union[MagicMock, CIExport] = create_autospec(CIExport) release_uploader = ReleaseUploader(asset_uploader_mock, ci_export_mock) release_uploader.release_upload( @@ -19,26 +23,23 @@ def test(temp_dir_mock): source_repo_url="https://github.com/test_source_repo_url", ) expected_artifact_path = temp_dir_mock().__enter__() - assert \ - mock_cast(asset_uploader_mock.upload_assets).mock_calls == [ - call( - repo_id='test_source_repo_url', - release_id=123, - content_type='application/gzip', - artifact_path=expected_artifact_path, - file_suffix='.tar.gz', - label_prefix='Flavor' - ), - call( - repo_id='test_source_repo_url', - release_id=123, - content_type='text/plain', - artifact_path=expected_artifact_path, - file_suffix='.tar.gz.sha512sum', - label_prefix='Checksum' - ), - ] \ - and mock_cast(ci_export_mock.export).mock_calls == [ - call(flavor_path=('test_flavor_path',), - export_path=expected_artifact_path) - ] + assert mock_cast(asset_uploader_mock.upload_assets).mock_calls == [ + call( + repo_id="test_source_repo_url", + release_id=123, + content_type="application/gzip", + artifact_path=expected_artifact_path, + file_suffix=".tar.gz", + label_prefix="Flavor", + ), + call( + repo_id="test_source_repo_url", + release_id=123, + content_type="text/plain", + artifact_path=expected_artifact_path, + file_suffix=".tar.gz.sha512sum", + label_prefix="Checksum", + ), + ] and mock_cast(ci_export_mock.export).mock_calls == [ + call(flavor_path=("test_flavor_path",), export_path=expected_artifact_path) + ]