dbt-core@main with dbt-common branch #58
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # **what?** | |
| # Runs all tests in dbt-core with this branch of dbt-common to ensure nothing is broken | |
| # **why?** | |
| # Ensure dbt-common changes do not break dbt-core | |
| # **when?** | |
| # This will run when trying to merge a PR into main. | |
| # It can also be manually triggered. | |
| # This workflow can be skipped by adding the "Skip Core Testing" label to the PR. This is | |
| # useful when making a change in both `dbt-core` and `dbt-common` where the changes are dependant | |
| # and cause the other repository to break. | |
| name: "Test Branches of dbt-core + dbt-common" | |
| run-name: >- | |
| ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') | |
| && format('dbt-core@{0} with dbt-common@{1}', inputs.dbt-core-ref, inputs.dbt-common-ref) | |
| || 'dbt-core@main with dbt-common branch' }} | |
| on: | |
| merge_group: | |
| types: [checks_requested] | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| dbt-core-ref: | |
| description: "The branch of dbt-core to test against" | |
| default: "main" | |
| dbt-common-ref: | |
| description: "The branch of dbt-common to test against" | |
| default: "main" | |
| workflow_call: | |
| inputs: | |
| dbt-core-ref: | |
| description: "The branch of dbt-core to test against" | |
| type: string | |
| required: true | |
| default: "main" | |
| dbt-common-ref: | |
| description: "The branch of dbt-common to test against" | |
| type: string | |
| required: true | |
| default: "main" | |
| permissions: read-all | |
| # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| # top-level adjustments can be made here | |
| env: | |
| # number of parallel processes to spawn for python integration testing | |
| PYTHON_INTEGRATION_TEST_WORKERS: 5 | |
| jobs: | |
| job-prep: | |
| # This allow us to run the workflow on pull_requests as well so we can always run unit tests | |
| # and only run integration tests on merge for time purposes | |
| name: Setup Repo Refs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| dbt-core-ref: ${{ steps.core-ref.outputs.ref }} | |
| dbt-common-ref: ${{ steps.common-ref.outputs.ref }} | |
| steps: | |
| - name: "[[DEBUG]] Input Refs" | |
| id: job-inputs | |
| run: | | |
| echo "inputs.dbt-core-ref=${{ inputs.dbt-core-ref }}" | |
| echo "inputs.dbt-common-ref=${{ inputs.dbt-common-ref }}" | |
| - name: "Determine dbt-core ref" | |
| id: core-ref | |
| run: | | |
| if [[ -z "${{ inputs.dbt-core-ref }}" ]]; then | |
| REF="main" | |
| else | |
| REF=${{ inputs.dbt-core-ref }} | |
| fi | |
| echo "ref=$REF" >> $GITHUB_OUTPUT | |
| - name: "Determine dbt-common ref" | |
| id: common-ref | |
| run: | | |
| if [[ -z "${{ inputs.dbt-common-ref }}" ]]; then | |
| # these will be commits instead of branches | |
| if [[ "${{ github.event_name }}" == "merge_group" ]]; then | |
| REF=${{ github.event.pull_request.merge_commit_sha }} | |
| else | |
| REF=${{ github.event.pull_request.base.sha }} | |
| fi | |
| else | |
| REF=${{ inputs.dbt-common-ref }} | |
| fi | |
| echo "ref=$REF" >> $GITHUB_OUTPUT | |
| - name: "[[DEBUG]] Final Refs" | |
| run: | | |
| echo "dbt-core-ref=${{ steps.core-ref.outputs.ref }}" | |
| echo "dbt-common-ref=${{ steps.common-ref.outputs.ref }}" | |
| dbt-core-unit-test: | |
| name: "Run unit tests for dbt-core" | |
| needs: [job-prep] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: "Check out dbt-core@${{ needs.job-prep.outputs.dbt-core-ref }}" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: dbt-labs/dbt-core | |
| ref: ${{ needs.job-prep.outputs.dbt-core-ref }} | |
| - name: "Set up Python 3.11" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: "Install python tools" | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip --version | |
| # tox takes care of installing the correct version of dbt-core dependencies but we need to | |
| # install them first s that we can override the dbt-common branch | |
| - name: "Manually install dbt-core dependencies" | |
| run: | | |
| python -m pip uninstall dbt-common -y | |
| python -m pip install -r dev-requirements.txt -r editable-requirements.txt | |
| # Since the dbt-common dependency is pinned in dev-requirements.txt we need to force update it | |
| # Since tox installs dependencies but doesn't force update, it won't get overridden in the next | |
| # step since the requirements will already be met | |
| - name: "Force update the version of dbt-common@${{ needs.job-prep.outputs.dbt-common-ref }}" | |
| run: | | |
| python -m pip install pip install git+https://github.com/dbt-labs/dbt-common.git@${{ needs.job-prep.outputs.dbt-common-ref }} --force-reinstall | |
| - name: "Run unit tests" | |
| # Doing the check here instead of the top level because this is job a required check, the | |
| # label just means we should skip the tests | |
| if: ${{ !contains(github.event.label.name, 'Skip Core Testing')}} | |
| run: tox | |
| env: | |
| TOXENV: unit | |
| - name: "Check installed versions" | |
| run: pip freeze | |
| dbt-core-integration-metadata: | |
| name: integration test metadata generation | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name != 'pull_request' }} && ${{ !contains(github.event.label.name, 'Skip Core Testing')}} | |
| outputs: | |
| split-groups: ${{ steps.generate-split-groups.outputs.split-groups }} | |
| include: ${{ steps.generate-include.outputs.include }} | |
| steps: | |
| - name: "generate split-groups" | |
| id: generate-split-groups | |
| run: | | |
| MATRIX_JSON="[" | |
| for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do | |
| MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}") | |
| done | |
| MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}" | |
| MATRIX_JSON+="]" | |
| echo "split-groups=${MATRIX_JSON}" | |
| echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT | |
| - name: "generate include" | |
| id: generate-include | |
| run: | | |
| INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-latest"' ) | |
| INCLUDE_GROUPS="[" | |
| for include in ${INCLUDE[@]}; do | |
| for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do | |
| INCLUDE_GROUPS+=$(sed 's/$/, /' <<< "{\"split-group\":\"${group}\",${include}}") | |
| done | |
| done | |
| INCLUDE_GROUPS=$(echo $INCLUDE_GROUPS | sed 's/,*$//g') | |
| INCLUDE_GROUPS+="]" | |
| echo "include=${INCLUDE_GROUPS}" | |
| echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT | |
| dbt-core-integration-tests: | |
| name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }} | |
| if: ${{ github.event_name != 'pull_request' }} && ${{ !contains(github.event.label.name, 'Skip Core Testing')}} | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| needs: [job-prep, dbt-core-integration-metadata] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| os: [ubuntu-20.04] | |
| split-group: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.split-groups) }} | |
| include: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.include) }} | |
| env: | |
| DBT_INVOCATION_ENV: github-actions | |
| DBT_TEST_USER_1: dbt_test_user_1 | |
| DBT_TEST_USER_2: dbt_test_user_2 | |
| DBT_TEST_USER_3: dbt_test_user_3 | |
| steps: | |
| - name: "Check out the repository@${{ needs.job-prep.outputs.dbt-core-ref }}" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: dbt-labs/dbt-core | |
| ref: ${{ needs.job-prep.outputs.dbt-core-ref }} | |
| - name: "Set up Python ${{ matrix.python-version }}" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: "Set up postgres (linux)" | |
| if: runner.os == 'Linux' | |
| uses: ./.github/actions/setup-postgres-linux | |
| - name: "Set up postgres (macos)" | |
| if: runner.os == 'macOS' | |
| uses: ./.github/actions/setup-postgres-macos | |
| - name: "Set up postgres (windows)" | |
| if: runner.os == 'Windows' | |
| uses: ./.github/actions/setup-postgres-windows | |
| - name: "Install python tools" | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip --version | |
| # tox takes care of installing the correct version of dbt-core dependencies but we need to | |
| # install them first s that we can override the dbt-common branch | |
| - name: "Manually install dbt-core dependencies" | |
| run: | | |
| python -m pip install -r dev-requirements.txt -r editable-requirements.txt | |
| # Since the dbt-common dependency is pinned in dev-requirements.txt we need to force update it | |
| # Since tox installs dependencies but doesn't force update, it won't get overridden in the next | |
| # step since the requirements will already be met | |
| - name: "Force update the version of dbt-common@${{ needs.job-prep.outputs.dbt-common-ref }}" | |
| run: | | |
| python -m pip uninstall dbt-common -y | |
| python -m pip install pip install git+https://github.com/dbt-labs/dbt-common.git@${{ needs.job-prep.outputs.dbt-common-ref }} --force-reinstall | |
| - name: "Run Functional tests" | |
| run: tox -- --ddtrace | |
| env: | |
| TOXENV: integration | |
| DBT_INVOCATION_ENV: github-actions | |
| DBT_TEST_USER_1: dbt_test_user_1 | |
| DBT_TEST_USER_2: dbt_test_user_2 | |
| DBT_TEST_USER_3: dbt_test_user_3 | |
| DD_CIVISIBILITY_AGENTLESS_ENABLED: true | |
| DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} | |
| DD_SITE: datadoghq.com | |
| DD_ENV: ci | |
| DD_SERVICE: dbt-core | |
| PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }} | |
| - name: "Check installed versions" | |
| run: pip freeze | |
| integration-report: | |
| if: ${{ always() }} | |
| name: Integration Test Suite | |
| runs-on: ubuntu-latest | |
| needs: [dbt-core-integration-tests] | |
| steps: | |
| - name: "Integration Tests Failed" | |
| if: ${{ contains(needs.dbt-core-integration-tests.result, 'failure') || contains(needs.dbt-core-integration-tests.result, 'cancelled') }} | |
| # when this is true the next step won't execute | |
| run: | | |
| echo "::notice title='Integration test suite failed'" | |
| exit 1 | |
| - name: "Integration Tests Passed" | |
| if: ${{ github.event_name != 'pull_request' }} | |
| run: | | |
| echo "::notice title='Integration test suite passed'" | |
| - name: "Integration Tests Skipped on Pull Request" | |
| if: ${{ github.event_name == 'pull_request' && !contains(github.event.label.name, 'Skip Core Testing')}} | |
| run: | | |
| echo "::notice title='Integration test suite skipped on Pull Requests - they will run on merge'" | |
| - name: "Integration Tests Skipped by Label" | |
| if: ${{ contains(github.event.label.name, 'Skip Core Testing')}} | |
| run: | | |
| echo "::notice title='dbt-core test suite skipped due to Skip Core Testing label'" |