schema-check.yml

# **what?**
# Compares the schema of the dbt version of the given ref vs
# the latest official schema releases found in schemas.getdbt.com.
# If there are differences, the workflow will fail and upload the
# diff as an artifact. The metadata team should be alerted to the change.
#
# **why?**
# Reaction work may need to be done if artifact schema changes
# occur so we want to proactively alert to it.
#
# **when?**
# On pushes to `develop` and release branches. Manual runs are also enabled.
name: Artifact Schema Check

on:
  workflow_dispatch:
  pull_request: #TODO: remove before merging
  push:
    branches:
      - "develop"
      - "*.latest"
      - "releases/*"

# no special access is needed
permissions: read-all

env:
  LATEST_SCHEMA_PATH: ${{ github.workspace }}/new_schemas
  SCHEMA_DIFF_ARTIFACT: ${{ github.workspace }}//schema_schanges.txt
  DBT_REPO_DIRECTORY: ${{ github.workspace }}/dbt
  SCHEMA_REPO_DIRECTORY: ${{ github.workspace }}/schemas.getdbt.com

jobs:
  checking-schemas:
    name: "Checking schemas"
    runs-on: ubuntu-latest

    steps:
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.8

      - name: Checkout dbt repo
        uses: actions/checkout@v4
        with:
            path: ${{ env.DBT_REPO_DIRECTORY }}

      - name: Checkout schemas.getdbt.com repo
        uses: actions/checkout@v4
        with:
          repository: dbt-labs/schemas.getdbt.com
          ref: 'main'
          path: ${{ env.SCHEMA_REPO_DIRECTORY }}

      - name: Generate current schema
        run: |
          cd ${{ env.DBT_REPO_DIRECTORY }}
          python3 -m venv env
          source env/bin/activate
          pip install --upgrade pip
          pip install -r dev-requirements.txt -r editable-requirements.txt
          python scripts/collect-artifact-schema.py --path ${{ env.LATEST_SCHEMA_PATH }}

      # Copy generated schema files into the schemas.getdbt.com repo
      # Do a git diff to find any changes
      # Ignore any date or version changes though
      - name: Compare schemas
        run: |
          cp -r ${{ env.LATEST_SCHEMA_PATH }}/dbt ${{ env.SCHEMA_REPO_DIRECTORY }}
          cd ${{ env.SCHEMA_REPO_DIRECTORY }}
          diff_results=$(git diff -I='*[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T' \
          -I='*[0-9]{1}.[0-9]{2}.[0-9]{1}(rc[0-9]|b[0-9]| )' --compact-summary)
          if [[ $(echo diff_results) ]]; then
            echo $diff_results
            echo "Schema changes detected!"
            git diff -I='*[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T' \
              -I='*[0-9]{1}.[0-9]{2}.[0-9]{1}(rc[0-9]|b[0-9]| )' > ${{ env.SCHEMA_DIFF_ARTIFACT }}
            exit 1
          else
            echo "No schema changes detected"
          fi

      - name: Upload schema diff
        uses: actions/upload-artifact@v4
        if: ${{ failure() }}
        with:
          name: 'schema_schanges.txt'
          path: '${{ env.SCHEMA_DIFF_ARTIFACT }}'