pr_run_models.yml

name: Pull Request testing - Run dbt models on BigQuery
on:
  pull_request:
    paths:
      - 'macros/**'
      - 'models/**' 
      - 'integration_tests/**'
      - 'dbt_project.yml'
      - 'packages.yml'
      - 'config.py'
  workflow_dispatch:

# GitHub secrets
env:
  DBT_PROFILES_DIR: .
  GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger
  DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT }}
  DBT_ENV_SECRET_BIGQUERY_TEST_STORAGE_PROJECT: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_STORAGE_PROJECT }}
  DBT_ENV_SECRET_BIGQUERY_TEST_EXECUTION_PROJECT: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_EXECUTION_PROJECT }}
  DBT_ENV_SECRET_BIGQUERY_TEST_LOCATION: ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_LOCATION }}
  DBT_BQ_MONITORING_GCP_PROJECTS: ${{ secrets.DBT_BQ_MONITORING_GCP_PROJECTS }}

concurrency:
  cancel-in-progress: true
  group: pr-run-models-${{ github.workflow }}-${{ github.ref }}

jobs:
  integration-bigquery:
    name: Run dbt models on BigQuery to test model validity
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR
      
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install Python packages
        run: python -m pip install dbt-bigquery~=1.9.1 sqlfluff-templater-dbt

      - name: Write keyfile if secret is defined
        run: |
          if [ -z "${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT }}" ]; then
            echo "Error: DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT is not defined."
          else
            echo ${{ secrets.DBT_ENV_SECRET_BIGQUERY_TEST_SERVICE_ACCOUNT }} | base64 -d > ./integration_tests/keyfile.json  
          fi

      - name: Setup dbt dependencies
        run: | 
          cd integration_tests
          dbt deps

      - name: Setup dbt fixtures
        run: | 
          cd integration_tests
          dbt run -s tag:fixtures

      - name: Run all models once
        run: | 
          cd integration_tests
          dbt build -s dbt_bigquery_monitoring --full-refresh --empty

      - name: Run all models again to test incremental
        run: |       
          cd integration_tests
          dbt build -s dbt_bigquery_monitoring --empty

      - name: Run all models again with cloud audit logs
        run: |       
          cd integration_tests
          DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \
          dbt run -s jobs_from_audit_logs+ --full-refresh --empty

      - name: Run all models again to test incremental with cloud audit logs
        run: |       
          cd integration_tests
          DBT_BQ_MONITORING_SHOULD_COMBINE_AUDIT_LOGS_AND_INFORMATION_SCHEMA=true DBT_BQ_MONITORING_GCP_BIGQUERY_AUDIT_LOGS=true \
          dbt run -s jobs_from_audit_logs+ --empty