Use docker for CI

bqbooster · Dec 15, 2024 · 8a14974 · 8a14974
1 parent 48985b6
commit 8a14974
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 37 deletions.
diff --git a/.github/workflows/build_base_image.yml b/.github/workflows/build_base_image.yml
@@ -9,6 +9,10 @@ on:
       - 'Dockerfile'
       - 'Makefile'
 
+env:
+  DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
+  IMAGE_NAME: dbt-bigquery-monitoring-base
+
 jobs:
   build-base-image:
     name: Build base image
@@ -25,5 +29,10 @@ jobs:
 
       - name: Build base image
         run: |
-          make build
+          docker build -t $(IMAGE_NAME):{{ github.branch }} .
+
+      - name: Push base image
+        run: |
+          docker tag $IMAGE_NAME:{{ github.branch }} $DOCKER_REGISTRY/$IMAGE_NAME:{{ github.branch }}
+	        docker push $DOCKER_REGISTRY/$IMAGE_NAME:main
 
diff --git a/.github/workflows/pr_lint_models.yml b/.github/workflows/pr_lint_models.yml
@@ -12,13 +12,13 @@ on:
 
 # GitHub secrets
 env:
-  DBT_PROFILES_DIR: ./integration_tests
   GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger
   DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT }}
   DBT_ENV_SECRET_BIGQUERY_TEST_STORAGE_PROJECT: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_STORAGE_PROJECT }}
   DBT_ENV_SECRET_BIGQUERY_TEST_EXECUTION_PROJECT: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_EXECUTION_PROJECT }}
   DBT_ENV_SECRET_BIGQUERY_TEST_LOCATION: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_LOCATION }}
   DBT_BQ_MONITORING_GCP_PROJECTS: ${{ secrets.DBT_BQ_MONITORING_GCP_PROJECTS }}
+  DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
 
 concurrency:
   cancel-in-progress: true
@@ -35,14 +35,6 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'    
-
-      - name: Install Python packages
-        run: python -m pip install dbt-bigquery~=1.8.2 sqlfluff-templater-dbt
-
       - name: Write keyfile if secret is defined
         run: |
           if [ -z "${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT }}" ]; then
@@ -52,10 +44,21 @@ jobs:
           fi
 
       - name: Test database connection
-        run: dbt debug
+        run: |
+          docker image ls \
+          && docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          $DOCKER_REGISTRY/dbt-bigquery-monitoring-base:main \
+          -c "dbt debug"
 
       - name: Install dbt packages
-        run: dbt deps
+        run: |
+          docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          $DOCKER_REGISTRY/dbt-bigquery-monitoring-base:main \
+          -c "dbt deps"
 
       - name: Get changed files
         id: get_file_changes
@@ -86,7 +89,13 @@ jobs:
         if: steps.get_files_to_lint.outputs.lintees != ''
         shell: bash -l {0}
         run: |
-          sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json
+          touch annotations.json && \
+          docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          -v rw:$(pwd)/annotations.json:/usr/app/annotations.json \
+          $DOCKER_REGISTRY/dbt-bigquery-monitoring-base:main \
+          -c "sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json"
           sed -i '/^\[/!d' annotations.json # see https://github.com/sqlfluff/sqlfluff/issues/2244
 
       - name: Annotate

diff --git a/.github/workflows/pr_run_models.yml b/.github/workflows/pr_run_models.yml
@@ -12,13 +12,13 @@ on:
 
 # GitHub secrets
 env:
-  DBT_PROFILES_DIR: .
   GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger
   DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT }}
   DBT_ENV_SECRET_BIGQUERY_TEST_STORAGE_PROJECT: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_STORAGE_PROJECT }}
   DBT_ENV_SECRET_BIGQUERY_TEST_EXECUTION_PROJECT: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_EXECUTION_PROJECT }}
   DBT_ENV_SECRET_BIGQUERY_TEST_LOCATION: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_LOCATION }}
   DBT_BQ_MONITORING_GCP_PROJECTS: ${{ secrets.DBT_BQ_MONITORING_GCP_PROJECTS }}
+  DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
 
 concurrency:
   cancel-in-progress: true
@@ -35,11 +35,6 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
       - name: Install Python packages
         run: python -m pip install dbt-bigquery~=1.8.2 sqlfluff-templater-dbt
 
@@ -53,32 +48,52 @@ jobs:
 
       - name: Setup dbt dependencies
         run: | 
-          cd integration_tests
-          dbt deps
+          docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          $DOCKER_REGISTRY/dbt-bigquery-monitoring-base:main \
+          -c "dbt deps"
 
       - name: Setup dbt fixtures
-        run: | 
-          cd integration_tests
-          dbt run -s tag:fixtures
+        run: |
+          docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          $DOCKER_REGISTRY/dbt-bigquery-monitoring-base:main \
+          -c "dbt run -s tag:fixtures"
 
       - name: Run all models once
         run: | 
-          cd integration_tests
-          dbt build -s dbt_bigquery_monitoring --full-refresh --empty
+          docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          $DOCKER_REGISTRY/dbt-bigquery-monitoring-base:main \
+          -c "dbt build -s dbt_bigquery_monitoring --full-refresh --empty"
 
       - name: Run all models again to test incremental
         run: |       
-          cd integration_tests
-          dbt build -s dbt_bigquery_monitoring --empty
+          docker run -i \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          $DOCKER_REGISTRY/dbt-bigquery-monitoring-base:main \
+          -c "dbt build -s dbt_bigquery_monitoring --empty"
 
       - name: Run all models again with cloud audit logs
-        run: |       
-          cd integration_tests
-          DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \
-          dbt run -s jobs_from_audit_logs+ --full-refresh --empty
+        run: |
+          docker run -i \
+          -e DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true \
+          -e DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          $DOCKER_REGISTRY/dbt-bigquery-monitoring-base:main \
+          -c "dbt run -s jobs_from_audit_logs+ --full-refresh --empty"
 
       - name: Run all models again to test incremental with cloud audit logs
         run: |       
-          cd integration_tests
-          DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \
-          dbt run -s jobs_from_audit_logs+ --empty
+          docker run -i \
+          -e DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true \
+          -e DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \
+          -v $(pwd)/integration_tests:/usr/app/ \
+          -v $(pwd)/keyfile.json:/usr/app/keyfile.json \
+          $DOCKER_REGISTRY/dbt-bigquery-monitoring-base:main \
+          -c "dbt run -s jobs_from_audit_logs+ --empty"
diff --git a/Dockerfile b/Dockerfile
@@ -53,4 +53,4 @@ RUN pip install --no-cache-dir \
     && rm -rf /usr/local/share/doc /root/.cache/
 
 WORKDIR /usr/app/
-ENTRYPOINT ["zsh"]
+ENTRYPOINT ["bash"]
diff --git a/packages.yml b/packages.yml
@@ -2,5 +2,5 @@ packages:
   - package: dbt-labs/dbt_utils
     version: [">=0.8.0", "<2.0.0"]
   # - package: dbt-labs/codegen
-  #   version: 0.12.1
+  #   version: 0.13.1