#1: Implement codebuild deployment and buildspec generation (#2)

Fixes #1
* Initial setup
* Initial implementation for `deploy-ci-build`, `deploy-source-credentials`, `generate-buildspec`, `validate-ci-build`, `validate-source-credentials`

Co-authored-by: Nicola Coretti <[email protected]>

.github/workflows/check_ci.yaml

@@ -0,0 +1,30 @@
+name: Run Unit Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  check_setup_py:
+    environment: AWS
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - uses: abatilo/[email protected]
+      with:
+        poetry-version: 1.1.13
+    - name: Setup poetry env
+      run:  bash scripts/build/setup_poetry_env.sh "python3.8"
+
+    - name: Run pytest
+      run: poetry run pytest
+      env: # Set the secret as an env variable
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
+        AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}

.github/workflows/check_setup_py.yaml

@@ -0,0 +1,20 @@
+name: Check packaging is up to date
+
+on: [push]
+
+jobs:
+  check_setup_py:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - uses: abatilo/[email protected]
+      with:
+        poetry-version: 1.1.13
+    - name: Run packaging update
+      run: bash githooks/update_packaging.sh "python3.8"
+    - name: Check git status
+      run: bash scripts/build/check_git_status.sh

.github/workflows/check_version.yaml

@@ -0,0 +1,21 @@
+name: Check if versions are consistent
+
+on: [push]
+
+jobs:
+  check-version-numbers:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Setup Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install Poetry
+        uses: abatilo/[email protected]
+        with:
+          poetry-version: 1.1.13
+      - name: Check Release
+        run: ./scripts/build/check_release.sh "python3.8"

.github/workflows/release_droid_upload_github_release_assets.yml

@@ -0,0 +1,52 @@
+name: Release Droid - Upload GitHub Release Assets
+
+on:
+  workflow_dispatch:
+    inputs:
+      upload_url:
+        description: 'Upload URL'
+        required: true
+
+jobs:
+  check-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Setup Python 3.8 for integration-test-docker-environment
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install Poetry
+        uses: abatilo/[email protected]
+        with:
+          poetry-version: 1.1.13
+      - name: Build Release
+        run: ./scripts/build/check_release.sh "python3.8"
+
+  upload:
+    needs: check-release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install Poetry
+        uses: abatilo/[email protected]
+        with:
+          poetry-version: 1.1.13
+
+      - name: Build Release
+        run: bash ./scripts/build/build_release.sh "python3.8"
+      - name: Generate sha512sum files
+        run: find ./dist -maxdepth 1 -type f -exec bash -c 'sha512sum {} > {}.sha512' \;
+        shell: bash
+      - name: Upload assets to the GitHub release draft
+        uses: shogo82148/actions-upload-release-asset@v1
+        with:
+          upload_url: ${{ github.event.inputs.upload_url }}
+          asset_path: dist/*

.github/workflows/shellcheck.yaml

@@ -0,0 +1,15 @@
+name: Check bash scripts
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run shellcheck
+        run: ./scripts/build/shellcheck.sh

.gitignore

@@ -0,0 +1,2 @@
+.idea
+.pytest_cache

README.md

@@ -2,4 +2,10 @@
 # Overview
 
 This project contains the AWS Cloudformation YAML files to deploy the AWS infrastructure for the ScriptLanguages-Container CI jobs.
-Those YAML files are not static, but generated, based on the provided Script-Languages container flavors.
+Those YAML files are not static, but generated, based on the provided Script-Languages container flavors.
+
+Also it contains a command to create the Buildspec for AWS Code Build, based on the list of flavors.
+
+## Links
+
+* [User Guide](./user_guide/user_guide.md)

doc/changes_0.1.0.md → doc/changes/changes_0.1.0.md

@@ -13,7 +13,7 @@ n/a
 
 ## Features / Enhancements
 
-n/a
+ - #1: Implement codebuild deployment and buildspec generation
 
 ## Documentation
 

exasol_script_languages_container_ci_setup/__init__.py

@@ -0,0 +1,8 @@
+from exasol_script_languages_container_ci_setup.cli.commands import (
+    health,
+    generate_buildspec,
+    deploy_source_credentials,
+    deploy_ci_build,
+    validate_ci_build,
+    validate_source_credentials
+)

exasol_script_languages_container_ci_setup/cli/cli.py

@@ -0,0 +1,6 @@
+import click
+
+
+@click.group()
+def cli():
+    pass

exasol_script_languages_container_ci_setup/cli/commands/deploy_ci_build.py

@@ -0,0 +1,32 @@
+import logging
+import sys
+
+import click
+
+from exasol_script_languages_container_ci_setup.cli.cli import cli
+from exasol_script_languages_container_ci_setup.cli.common import add_options
+from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
+from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.ci_build import run_deploy_ci_build
+from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
+
+
+@cli.command()
+@add_options(aws_options)
+@add_options(logging_options)
+@click.option('--project', type=str, required=True,
+              help="""The project for which the stack will be created.""")
+@click.option('--project-url', type=str, required=True,
+              help="""The URL of the project on Github.""")
+def deploy_ci_build(
+        aws_profile: str,
+        log_level: str,
+        project: str,
+        project_url: str):
+    set_log_level(log_level)
+    try:
+        run_deploy_ci_build(AwsAccess(aws_profile), project,
+                            project_url)
+    except Exception:
+        logging.error("run_deploy_ci_build failed.")
+        sys.exit(1)

exasol_script_languages_container_ci_setup/cli/commands/deploy_source_credentials.py

@@ -0,0 +1,35 @@
+import logging
+import sys
+
+import click
+
+from exasol_script_languages_container_ci_setup.cli.cli import cli
+from exasol_script_languages_container_ci_setup.cli.common import add_options
+from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
+from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.source_credentials import run_deploy_source_credentials
+from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
+
+
+@cli.command()
+@add_options(aws_options)
+@add_options(logging_options)
+@click.option('--secret-name', required=True, type=str,
+              help="Secret stored in AWS Secret Manager.")
+@click.option('--secret-user-key', required=True, type=str,
+              help="User key stored as secret in AWS Secret Manager.")
+@click.option('--secret-token-key', required=True, type=str,
+              help="Token key stored as secret in AWS Secret Manager.")
+def deploy_source_credentials(
+        aws_profile: str,
+        log_level: str,
+        secret_name: str,
+        secret_user_key: str,
+        secret_token_key: str):
+    set_log_level(log_level)
+    try:
+        run_deploy_source_credentials(AwsAccess(aws_profile), secret_name, secret_user_key, secret_token_key)
+    except Exception:
+        logging.error("deploy_source_credentials failed.")
+        sys.exit(1)
+

exasol_script_languages_container_ci_setup/cli/commands/generate_buildspec.py

@@ -0,0 +1,33 @@
+from pathlib import Path
+from typing import Tuple, Optional
+
+import click
+
+from exasol_script_languages_container_ci_setup.cli.cli import cli
+from exasol_script_languages_container_ci_setup.cli.common import add_options
+from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
+from exasol_script_languages_container_ci_setup.lib.run_generate_buildspec import run_generate_buildspec
+
+
+@cli.command()
+@add_options(logging_options)
+@click.option('--flavor-root-path', required=True, multiple=True,
+              type=click.Path(file_okay=False, dir_okay=True, exists=True),
+              help="Path where script language container flavors are located.")
+@click.option('--output-path', type=click.Path(file_okay=False, dir_okay=True, exists=True, writable=True),
+              default="./aws-code-build/ci", show_default=True,
+              help="Path where buildspec files will be deployed.")
+@click.option('--config-file', type=click.Path(file_okay=True, dir_okay=False, exists=True),
+              help="Configuration file for build (project specific).")
+def generate_buildspecs(
+        flavor_root_path: Tuple[str, ...],
+        log_level: str,
+        output_path: str,
+        config_file: Optional[str]
+        ):
+    """
+    This command generates the buildspec file(s) for AWS CodeBuild based on the flavors located in path "flavor_root_path".
+    """
+    set_log_level(log_level)
+    run_generate_buildspec(flavor_root_path, output_path, config_file)
+

exasol_script_languages_container_ci_setup/cli/commands/health.py

@@ -0,0 +1,44 @@
+import sys
+from inspect import cleandoc
+
+from exasol_script_languages_container_ci_setup.cli.cli import cli
+from exasol_script_languages_container_ci_setup.cli.common import add_options
+from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
+from exasol_script_languages_container_ci_setup.health_check import (
+    health_checkup,
+)
+
+
+@cli.command()
+@add_options(aws_options)
+def health(aws_profile: str):
+    """
+    Check the health of the execution environment.
+
+    If no issues have been found, using the library or executing the test should work just fine.
+    For all found issues there will be a proposed fix/solution.
+
+    If the environment was found to be healthy the exit code will be 0.
+    """
+    success, failure = 0, -1
+
+    problems = set(health_checkup(aws_profile=aws_profile))
+    if not problems:
+        sys.exit(success)
+
+    message = cleandoc(
+        """
+        {count} problem(s) have been identified.
+        
+        {problems}
+        """
+    ).format(
+        count=len(problems),
+        problems="\n".join(
+            (
+                str(p) for p in problems
+            )
+        ),
+    )
+    print(message)
+    sys.exit(failure)

exasol_script_languages_container_ci_setup/cli/commands/validate_ci_build.py

@@ -0,0 +1,26 @@
+import logging
+
+import click
+
+from exasol_script_languages_container_ci_setup.cli.cli import cli
+from exasol_script_languages_container_ci_setup.cli.common import add_options
+from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
+from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.ci_build import run_deploy_ci_build, run_validate_ci_build
+from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
+
+
+@cli.command()
+@add_options(aws_options)
+@add_options(logging_options)
+@click.option('--project', type=str, required=True,
+              help="""The project for which the stack will be created.""")
+@click.option('--project-url', type=str, required=True,
+              help="""The URL of the project on Github.""")
+def validate_ci_build(
+        aws_profile: str,
+        log_level: str,
+        project: str,
+        project_url: str):
+    set_log_level(log_level)
+    run_validate_ci_build(AwsAccess(aws_profile), project, project_url)

exasol_script_languages_container_ci_setup/cli/commands/validate_source_credentials.py

@@ -0,0 +1,27 @@
+import click
+
+from exasol_script_languages_container_ci_setup.cli.cli import cli
+from exasol_script_languages_container_ci_setup.cli.common import add_options
+from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
+from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.source_credentials import run_validate_source_credentials
+from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
+
+
+@cli.command()
+@add_options(aws_options)
+@add_options(logging_options)
+@click.option('--secret-name', required=True, type=str,
+              help="Secret name for the Github user credentials stored in AWS Secret Manager.")
+@click.option('--secret-user-key', required=True, type=str,
+              help="Github user key stored as secret in AWS Secret Manager under the respective secret name.")
+@click.option('--secret-token-key', required=True, type=str,
+              help="Github user token key stored as secret in AWS Secret Manager under the respective secret name.")
+def validate_source_credentials(
+        aws_profile: str,
+        log_level: str,
+        secret_name: str,
+        secret_user_key: str,
+        secret_token_key: str):
+    set_log_level(log_level)
+    run_validate_source_credentials(AwsAccess(aws_profile), secret_name, secret_user_key, secret_token_key)

exasol_script_languages_container_ci_setup/cli/common.py

@@ -0,0 +1,8 @@
+
+def add_options(options):
+    def _add_options(func):
+        for option in reversed(options):
+            func = option(func)
+        return func
+
+    return _add_options