From e912ee2f15bf04138c6683538d8dfcb79a940d49 Mon Sep 17 00:00:00 2001 From: Christophe Oudar Date: Sun, 15 Dec 2024 22:01:27 +0100 Subject: [PATCH] Use docker for CI --- .github/workflows/pr_lint_models.yml | 31 +++++++++------ .github/workflows/pr_run_models.yml | 57 ++++++++++++++++++---------- Dockerfile | 2 +- packages.yml | 2 +- 4 files changed, 58 insertions(+), 34 deletions(-) diff --git a/.github/workflows/pr_lint_models.yml b/.github/workflows/pr_lint_models.yml index 719a02b..55518c6 100644 --- a/.github/workflows/pr_lint_models.yml +++ b/.github/workflows/pr_lint_models.yml @@ -35,14 +35,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install Python packages - run: python -m pip install dbt-bigquery~=1.8.2 sqlfluff-templater-dbt - - name: Write keyfile if secret is defined run: | if [ -z "${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT }}" ]; then @@ -52,10 +44,21 @@ jobs: fi - name: Test database connection - run: dbt debug + run: | + docker image ls \ + && docker run -i \ + -v $(pwd)/integration_tests:/usr/app/ \ + -v $(pwd)/keyfile.json:/usr/app/keyfile.json \ + dbt-bigquery-monitoring-base:main \ + -c "dbt debug" - name: Install dbt packages - run: dbt deps + run: | + docker run -i \ + -v $(pwd)/integration_tests:/usr/app/ \ + -v $(pwd)/keyfile.json:/usr/app/keyfile.json \ + dbt-bigquery-monitoring-base:main \ + -c "dbt deps" - name: Get changed files id: get_file_changes @@ -86,7 +89,13 @@ jobs: if: steps.get_files_to_lint.outputs.lintees != '' shell: bash -l {0} run: | - sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json + touch annotations.json && \ + docker run -i \ + -v $(pwd)/integration_tests:/usr/app/ \ + -v $(pwd)/keyfile.json:/usr/app/keyfile.json \ + -v rw:$(pwd)/annotations.json:/usr/app/annotations.json \ + dbt-bigquery-monitoring-base:main \ + -c "sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json" sed -i '/^\[/!d' annotations.json # see https://github.com/sqlfluff/sqlfluff/issues/2244 - name: Annotate diff --git a/.github/workflows/pr_run_models.yml b/.github/workflows/pr_run_models.yml index e1cc408..247bd06 100644 --- a/.github/workflows/pr_run_models.yml +++ b/.github/workflows/pr_run_models.yml @@ -35,11 +35,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install Python packages run: python -m pip install dbt-bigquery~=1.8.2 sqlfluff-templater-dbt @@ -53,32 +48,52 @@ jobs: - name: Setup dbt dependencies run: | - cd integration_tests - dbt deps + docker run -i \ + -v $(pwd)/integration_tests:/usr/app/ \ + -v $(pwd)/keyfile.json:/usr/app/keyfile.json \ + dbt-bigquery-monitoring-base:main \ + -c "dbt deps" - name: Setup dbt fixtures - run: | - cd integration_tests - dbt run -s tag:fixtures + run: | + docker run -i \ + -v $(pwd)/integration_tests:/usr/app/ \ + -v $(pwd)/keyfile.json:/usr/app/keyfile.json \ + dbt-bigquery-monitoring-base:main \ + -c "dbt run -s tag:fixtures" - name: Run all models once run: | - cd integration_tests - dbt build -s dbt_bigquery_monitoring --full-refresh --empty + docker run -i \ + -v $(pwd)/integration_tests:/usr/app/ \ + -v $(pwd)/keyfile.json:/usr/app/keyfile.json \ + dbt-bigquery-monitoring-base:main \ + -c "dbt build -s dbt_bigquery_monitoring --full-refresh --empty" - name: Run all models again to test incremental run: | - cd integration_tests - dbt build -s dbt_bigquery_monitoring --empty + docker run -i \ + -v $(pwd)/integration_tests:/usr/app/ \ + -v $(pwd)/keyfile.json:/usr/app/keyfile.json \ + dbt-bigquery-monitoring-base:main \ + -c "dbt build -s dbt_bigquery_monitoring --empty" - name: Run all models again with cloud audit logs - run: | - cd integration_tests - DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \ - dbt run -s jobs_from_audit_logs+ --full-refresh --empty + run: | + docker run -i \ + -e DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true \ + -e DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \ + -v $(pwd)/integration_tests:/usr/app/ \ + -v $(pwd)/keyfile.json:/usr/app/keyfile.json \ + dbt-bigquery-monitoring-base:main \ + -c "dbt run -s jobs_from_audit_logs+ --full-refresh --empty" - name: Run all models again to test incremental with cloud audit logs run: | - cd integration_tests - DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \ - dbt run -s jobs_from_audit_logs+ --empty + docker run -i \ + -e DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true \ + -e DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \ + -v $(pwd)/integration_tests:/usr/app/ \ + -v $(pwd)/keyfile.json:/usr/app/keyfile.json \ + dbt-bigquery-monitoring-base:main \ + -c "dbt run -s jobs_from_audit_logs+ --empty" diff --git a/Dockerfile b/Dockerfile index b262793..002b7f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,4 +53,4 @@ RUN pip install --no-cache-dir \ && rm -rf /usr/local/share/doc /root/.cache/ WORKDIR /usr/app/ -ENTRYPOINT ["zsh"] +ENTRYPOINT ["bash"] diff --git a/packages.yml b/packages.yml index 6f9c24f..9c3d03e 100644 --- a/packages.yml +++ b/packages.yml @@ -2,5 +2,5 @@ packages: - package: dbt-labs/dbt_utils version: [">=0.8.0", "<2.0.0"] # - package: dbt-labs/codegen - # version: 0.12.1 + # version: 0.13.1