Recognize when loaded_at_field is explicitly set to null #137
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # **what?** | |
| # Compares the schema of the dbt version of the given ref vs | |
| # the latest official schema releases found in schemas.getdbt.com. | |
| # If there are differences, the workflow will fail and upload the | |
| # diff as an artifact. The metadata team should be alerted to the change. | |
| # | |
| # **why?** | |
| # Reaction work may need to be done if artifact schema changes | |
| # occur so we want to proactively alert to it. | |
| # | |
| # **when?** | |
| # On pushes to `develop` and release branches. Manual runs are also enabled. | |
| name: Artifact Schema Check | |
| on: | |
| workflow_dispatch: | |
| pull_request: #TODO: remove before merging | |
| push: | |
| branches: | |
| - "develop" | |
| - "*.latest" | |
| - "releases/*" | |
| # no special access is needed | |
| permissions: read-all | |
| env: | |
| LATEST_SCHEMA_PATH: ${{ github.workspace }}/new_schemas | |
| SCHEMA_DIFF_ARTIFACT: ${{ github.workspace }}/schema_schanges.txt | |
| DBT_REPO_DIRECTORY: ${{ github.workspace }}/dbt | |
| SCHEMA_REPO_DIRECTORY: ${{ github.workspace }}/schemas.getdbt.com | |
| jobs: | |
| checking-schemas: | |
| name: "Checking schemas" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.8 | |
| - name: Checkout dbt repo | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ${{ env.DBT_REPO_DIRECTORY }} | |
| - name: Checkout schemas.getdbt.com repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: dbt-labs/schemas.getdbt.com | |
| ref: 'main' | |
| ssh-key: ${{ secrets.SCHEMA_SSH_PRIVATE_KEY }} | |
| path: ${{ env.SCHEMA_REPO_DIRECTORY }} | |
| - name: Generate current schema | |
| run: | | |
| cd ${{ env.DBT_REPO_DIRECTORY }} | |
| python3 -m venv env | |
| source env/bin/activate | |
| pip install --upgrade pip | |
| pip install -r dev-requirements.txt -r editable-requirements.txt | |
| python scripts/collect-artifact-schema.py --path ${{ env.LATEST_SCHEMA_PATH }} | |
| # Copy generated schema files into the schemas.getdbt.com repo | |
| # Do a git diff to find any changes | |
| # Ignore any date or version changes though | |
| - name: Compare schemas | |
| run: | | |
| cp -r ${{ env.LATEST_SCHEMA_PATH }}/dbt ${{ env.SCHEMA_REPO_DIRECTORY }} | |
| cd ${{ env.SCHEMA_REPO_DIRECTORY }} | |
| diff_results=$(git diff -I='*[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T' \ | |
| -I='*[0-9]{1}.[0-9]{2}.[0-9]{1}(rc[0-9]|b[0-9]| )' --compact-summary) | |
| if [[ $(echo diff_results) ]]; then | |
| echo $diff_results | |
| echo "Schema changes detected!" | |
| git diff -I='*[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T' \ | |
| -I='*[0-9]{1}.[0-9]{2}.[0-9]{1}(rc[0-9]|b[0-9]| )' > ${{ env.SCHEMA_DIFF_ARTIFACT }} | |
| exit 1 | |
| else | |
| echo "No schema changes detected" | |
| fi | |
| - name: Upload schema diff | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ failure() }} | |
| with: | |
| name: 'schema_schanges.txt' | |
| path: '${{ env.SCHEMA_DIFF_ARTIFACT }}' |