From 096c2899d53ef0bc8b6d02a3d49dfc2a04188fa7 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Fri, 17 May 2024 14:54:01 -0700 Subject: [PATCH] Removing the black autocommit action and doing some general housekeeping on the other GH actions --- .github/workflows/black.yaml | 20 --- .github/workflows/codacy.yaml | 64 ++++---- .github/workflows/codeql.yaml | 60 ++++---- .github/workflows/on_branch_merge.yaml | 24 +-- .github/workflows/on_pr.yaml | 32 ++-- .github/workflows/run_tests.yaml | 200 ++++++++++++------------- .github/workflows/trivy.yaml | 39 +++-- 7 files changed, 204 insertions(+), 235 deletions(-) delete mode 100644 .github/workflows/black.yaml diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml deleted file mode 100644 index ab4139e..0000000 --- a/.github/workflows/black.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: Python autoformatting - -on: - workflow_call: - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - - uses: actions/checkout@v3 - - - uses: psf/black@stable - with: - options: --verbose - - - - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: Auto-commit black formatting changes diff --git a/.github/workflows/codacy.yaml b/.github/workflows/codacy.yaml index e98073f..cc551f3 100644 --- a/.github/workflows/codacy.yaml +++ b/.github/workflows/codacy.yaml @@ -9,41 +9,41 @@ name: Codacy Security Scan on: - workflow_call: + workflow_call: permissions: - contents: read + contents: read jobs: - codacy-security-scan: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - name: Codacy Security Scan - runs-on: ubuntu-latest - steps: - # Checkout the repository to the GitHub Actions runner - - name: Checkout code - uses: actions/checkout@v3 + codacy-security-scan: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + name: Codacy Security Scan + runs-on: ubuntu-latest + steps: + # Checkout the repository to the GitHub Actions runner + - name: Checkout code + uses: actions/checkout@v4 - # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis - - name: Run Codacy Analysis CLI - uses: codacy/codacy-analysis-cli-action@5cc54a75f9ad88159bb54046196d920e40e367a5 - with: - # Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository - # You can also omit the token and run the tools that support default configurations - project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} - verbose: true - output: results.sarif - format: sarif - # Adjust severity of non-security issues - gh-code-scanning-compat: true - # Force 0 exit code to allow SARIF file generation - # This will handover control about PR rejection to the GitHub side - max-allowed-issues: 2147483647 + # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis + - name: Run Codacy Analysis CLI + uses: codacy/codacy-analysis-cli-action@5cc54a75f9ad88159bb54046196d920e40e367a5 + with: + # Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository + # You can also omit the token and run the tools that support default configurations + project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} + verbose: true + output: results.sarif + format: sarif + # Adjust severity of non-security issues + gh-code-scanning-compat: true + # Force 0 exit code to allow SARIF file generation + # This will handover control about PR rejection to the GitHub side + max-allowed-issues: 2147483647 - # Upload the SARIF file generated in the previous step - - name: Upload SARIF results file - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: results.sarif + # Upload the SARIF file generated in the previous step + - name: Upload SARIF results file + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: results.sarif diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 30aa488..8aad451 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -1,40 +1,40 @@ name: "CodeQL" on: - workflow_call: + workflow_call: jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write - strategy: - fail-fast: false - matrix: - language: [ 'python' ] - # python-version: ["3.9", "3.10"] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + strategy: + fail-fast: false + matrix: + language: ["python"] + python-version: ["3.10", "3.11", "3.12"] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - steps: - - name: Checkout repository - uses: actions/checkout@v3 + steps: + - name: Checkout repository + uses: actions/checkout@v4 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. - # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/on_branch_merge.yaml b/.github/workflows/on_branch_merge.yaml index 0567096..db29666 100644 --- a/.github/workflows/on_branch_merge.yaml +++ b/.github/workflows/on_branch_merge.yaml @@ -1,20 +1,20 @@ name: Merge to main / develop on: - push: - branches: - - main - - develop + push: + branches: + - main + - develop jobs: - run_unit_tests: - uses: ./.github/workflows/run_tests.yaml + run_unit_tests: + uses: ./.github/workflows/run_tests.yaml - run_code_ql: - uses: ./.github/workflows/codeql.yaml + run_code_ql: + uses: ./.github/workflows/codeql.yaml - run_codacy: - uses: ./.github/workflows/codacy.yaml + run_codacy: + uses: ./.github/workflows/codacy.yaml - run_trivy: - uses: ./.github/workflows/trivy.yaml + run_trivy: + uses: ./.github/workflows/trivy.yaml diff --git a/.github/workflows/on_pr.yaml b/.github/workflows/on_pr.yaml index d76f6ab..6956bd7 100644 --- a/.github/workflows/on_pr.yaml +++ b/.github/workflows/on_pr.yaml @@ -1,26 +1,22 @@ name: Standard PR workflow on: - pull_request: - types: - - opened - - reopened - - synchronize - - ready_for_review + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review jobs: - format_files: - uses: ./.github/workflows/black.yaml + run_unit_tests: + uses: ./.github/workflows/run_tests.yaml - run_unit_tests: - uses: ./.github/workflows/run_tests.yaml - needs: format_files + run_code_ql: + uses: ./.github/workflows/codeql.yaml - run_code_ql: - uses: ./.github/workflows/codeql.yaml + run_codacy: + uses: ./.github/workflows/codacy.yaml - run_codacy: - uses: ./.github/workflows/codacy.yaml - - run_trivy: - uses: ./.github/workflows/trivy.yaml + run_trivy: + uses: ./.github/workflows/trivy.yaml diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index a4c51ed..541aa80 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -1,107 +1,103 @@ name: Lint and Test on: - workflow_call: + workflow_call: jobs: - run_unit_tests: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11"] - - steps: - - - name: Repo checkout - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Poetry - uses: snok/install-poetry@v1 - - - name: Install dependencies - id: install_deps - run: poetry install - - - name: poetry run pytest -rP tests/ - id: run_pytest - run: | - poetry run pytest --cov=credit_engine --cov-report=xml -rP tests/ - continue-on-error: true - - - name: Send to Codecov - id: send_to_codecov - uses: codecov/codecov-action@v3 - continue-on-error: true - with: - files: ./coverage.xml - fail_ci_if_error: true - - - name: outcome failure - if: steps.run_pytest.outcome != 'success' || steps.send_to_codecov.outcome != 'success' - run: | - echo "Python tests: ${{ steps.run_pytest.outcome }}" - echo "upload coverage: ${{ steps.send_to_codecov.outcome }}" - exit 1 - - - name: outcome success - run: | - echo "Python tests: ${{ steps.run_pytest.outcome }}" - echo "upload coverage: ${{ steps.send_to_codecov.outcome }}" - exit 0 - - run_linkml_tasks: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11"] - - steps: - - - name: Repo checkout - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Poetry - uses: snok/install-poetry@v1 - - - name: Install dependencies - id: install_deps - run: poetry install -E docs - - - name: Lint linkml file - id: lint_linkml - run: | - poetry run linkml-lint ./schema/kbase/linkml/credit_metadata.yaml --ignore-warnings - continue-on-error: true - - - name: Validate kbase sample data against the schema - id: validate_kbase_data - run: | - poetry run linkml-validate -s schema/kbase/linkml/credit_metadata.yaml sample_data/kbase/10.25982_1722943.yaml - poetry run linkml-validate -s schema/kbase/linkml/credit_metadata.yaml sample_data/kbase/10.6084_m9.figshare.12644081.v1.yaml - continue-on-error: true - - - name: outcome failure - if: steps.lint_linkml.outcome != 'success' # || steps.validate_kbase_data.outcome != 'success' - run: | - echo "linkml linting: ${{ steps.lint_linkml.outcome }}" - echo "KBase sample data validation: ${{ steps.validate_kbase_data.outcome }}" - exit 1 - - - name: outcome success - if: steps.lint_linkml.outcome == 'success' # || steps.validate_kbase_data.outcome != 'success' - run: | - echo "linkml linting: ${{ steps.lint_linkml.outcome }}" - echo "KBase sample data validation: ${{ steps.validate_kbase_data.outcome }}" - exit 0 + run_unit_tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - name: Repo checkout + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + uses: snok/install-poetry@v1 + + - name: Install dependencies + id: install_deps + run: poetry install + + - name: poetry run pytest -rP tests/ + id: run_pytest + run: | + poetry run pytest --cov=credit_engine --cov-report=xml -rP tests/ + continue-on-error: true + + - name: Send to Codecov + id: send_to_codecov + uses: codecov/codecov-action@v4 + continue-on-error: true + with: + files: ./coverage.xml + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + + - name: outcome failure + if: steps.run_pytest.outcome != 'success' || steps.send_to_codecov.outcome != 'success' + run: | + echo "Python tests: ${{ steps.run_pytest.outcome }}" + echo "upload coverage: ${{ steps.send_to_codecov.outcome }}" + exit 1 + + - name: outcome success + run: | + echo "Python tests: ${{ steps.run_pytest.outcome }}" + echo "upload coverage: ${{ steps.send_to_codecov.outcome }}" + exit 0 + + run_linkml_tasks: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - name: Repo checkout + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + id: install_deps + run: pip install linkml + + - name: Lint linkml file + id: lint_linkml + run: | + linkml-lint ./schema/kbase/linkml/credit_metadata.yaml --ignore-warnings + continue-on-error: true + + - name: Validate kbase sample data against the schema + id: validate_kbase_data + run: | + linkml-validate -s schema/kbase/linkml/credit_metadata.yaml sample_data/kbase/10.25982_1722943.yaml + linkml-validate -s schema/kbase/linkml/credit_metadata.yaml sample_data/kbase/10.6084_m9.figshare.12644081.v1.yaml + continue-on-error: true + + - name: outcome failure + if: steps.lint_linkml.outcome != 'success' # || steps.validate_kbase_data.outcome != 'success' + run: | + echo "linkml linting: ${{ steps.lint_linkml.outcome }}" + echo "KBase sample data validation: ${{ steps.validate_kbase_data.outcome }}" + exit 1 + + - name: outcome success + if: steps.lint_linkml.outcome == 'success' # || steps.validate_kbase_data.outcome != 'success' + run: | + echo "linkml linting: ${{ steps.lint_linkml.outcome }}" + echo "KBase sample data validation: ${{ steps.validate_kbase_data.outcome }}" + exit 0 diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index b54bba9..b76866e 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -1,26 +1,23 @@ name: Trivy scan on: - workflow_call: + workflow_call: jobs: - run_trivy: - runs-on: ubuntu-latest - steps: - - - name: Repo checkout - uses: actions/checkout@v3 - - - name: Run Trivy vulnerability scanner in repo mode - uses: aquasecurity/trivy-action@master - with: - scan-type: 'fs' - ignore-unfixed: true - format: 'sarif' - output: 'trivy-results.sarif' - severity: 'CRITICAL' + run_trivy: + runs-on: ubuntu-latest + steps: + - name: Repo checkout + uses: actions/checkout@v4 + - name: Run Trivy vulnerability scanner in repo mode + uses: aquasecurity/trivy-action@master + with: + scan-type: "fs" + ignore-unfixed: true + format: "sarif" + output: "trivy-results.sarif" + severity: "CRITICAL" - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: 'trivy-results.sarif' + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif"