Use docker for CI

bqbooster · Dec 15, 2024 · e912ee2 · e912ee2
1 parent 48985b6
commit e912ee2
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 34 deletions.
diff --git a/.github/workflows/pr_lint_models.yml b/.github/workflows/pr_lint_models.yml
@@ -35,14 +35,6 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'    
-
-      - name: Install Python packages
-        run: python -m pip install dbt-bigquery~=1.8.2 sqlfluff-templater-dbt
-
       - name: Write keyfile if secret is defined
         run: |
           if [ -z "${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT }}" ]; then
@@ -52,10 +44,21 @@ jobs:
           fi
 
       - name: Test database connection
-        run: dbt debug
+        run: |
+          docker image ls \
+          && docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          dbt-bigquery-monitoring-base:main \
+          -c "dbt debug"
 
       - name: Install dbt packages
-        run: dbt deps
+        run: |
+          docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          dbt-bigquery-monitoring-base:main \
+          -c "dbt deps"
 
       - name: Get changed files
         id: get_file_changes
@@ -86,7 +89,13 @@ jobs:
         if: steps.get_files_to_lint.outputs.lintees != ''
         shell: bash -l {0}
         run: |
-          sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json
+          touch annotations.json && \
+          docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          -v rw:$(pwd)/annotations.json:/usr/app/annotations.json \
+          dbt-bigquery-monitoring-base:main \
+          -c "sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json"
           sed -i '/^\[/!d' annotations.json # see https://github.com/sqlfluff/sqlfluff/issues/2244
 
       - name: Annotate

diff --git a/.github/workflows/pr_run_models.yml b/.github/workflows/pr_run_models.yml
@@ -35,11 +35,6 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
       - name: Install Python packages
         run: python -m pip install dbt-bigquery~=1.8.2 sqlfluff-templater-dbt
 
@@ -53,32 +48,52 @@ jobs:
 
       - name: Setup dbt dependencies
         run: | 
-          cd integration_tests
-          dbt deps
+          docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          dbt-bigquery-monitoring-base:main \
+          -c "dbt deps"
 
       - name: Setup dbt fixtures
-        run: | 
-          cd integration_tests
-          dbt run -s tag:fixtures
+        run: |
+          docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          dbt-bigquery-monitoring-base:main \
+          -c "dbt run -s tag:fixtures"
 
       - name: Run all models once
         run: | 
-          cd integration_tests
-          dbt build -s dbt_bigquery_monitoring --full-refresh --empty
+          docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          dbt-bigquery-monitoring-base:main \
+          -c "dbt build -s dbt_bigquery_monitoring --full-refresh --empty"
 
       - name: Run all models again to test incremental
         run: |       
-          cd integration_tests
-          dbt build -s dbt_bigquery_monitoring --empty
+          docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          dbt-bigquery-monitoring-base:main \
+          -c "dbt build -s dbt_bigquery_monitoring --empty"
 
       - name: Run all models again with cloud audit logs
-        run: |       
-          cd integration_tests
-          DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \
-          dbt run -s jobs_from_audit_logs+ --full-refresh --empty
+        run: |
+          docker run -i \
+          -e DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true \
+          -e DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          dbt-bigquery-monitoring-base:main \
+          -c "dbt run -s jobs_from_audit_logs+ --full-refresh --empty"
 
       - name: Run all models again to test incremental with cloud audit logs
         run: |       
-          cd integration_tests
-          DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \
-          dbt run -s jobs_from_audit_logs+ --empty
+          docker run -i \
+          -e DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true \
+          -e DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          dbt-bigquery-monitoring-base:main \
+          -c "dbt run -s jobs_from_audit_logs+ --empty"
diff --git a/Dockerfile b/Dockerfile
@@ -53,4 +53,4 @@ RUN pip install --no-cache-dir \
     && rm -rf /usr/local/share/doc /root/.cache/
 
 WORKDIR /usr/app/
-ENTRYPOINT ["zsh"]
+ENTRYPOINT ["bash"]
diff --git a/packages.yml b/packages.yml
@@ -2,5 +2,5 @@ packages:
   - package: dbt-labs/dbt_utils
     version: [">=0.8.0", "<2.0.0"]
   # - package: dbt-labs/codegen
-  #   version: 0.12.1
+  #   version: 0.13.1