if [[ pull_request =~ ^(workflow_dispatch|workflow_call)$ ]]; then "Test dbt-core@ with dbt-common@" else "Test dbt-core@main with dbt-common branch" fi #37
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # **what?** | |
| # Runs all tests in dbt-core with this branch of dbt-common to ensure nothing is broken | |
| # **why?** | |
| # Ensure dbt-common changes do not break dbt-core | |
| # **when?** | |
| # This will run when trying to merge a PR into main. | |
| # It can also be manually triggered. | |
| name: "Test Branches of dbt-core + dbt-common" | |
| run-name: >- | |
| if [[ ${{ github.event_name }} =~ ^(workflow_dispatch|workflow_call)$ ]]; then | |
| "Test dbt-core@${{ inputs.dbt-core-ref }} with dbt-common@${{ inputs.dbt-common-ref }}" | |
| else | |
| "Test dbt-core@main with dbt-common branch" | |
| fi | |
| on: | |
| merge_group: | |
| types: [checks_requested] | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| dbt-core-ref: | |
| description: "The branch of dbt-core to test against" | |
| default: "main" | |
| dbt-common-ref: | |
| description: "The branch of dbt-common to test against" | |
| default: "main" | |
| workflow_call: | |
| inputs: | |
| dbt-core-ref: | |
| description: "The branch of dbt-core to test against" | |
| type: string | |
| required: true | |
| default: "main" | |
| dbt-common-ref: | |
| description: "The branch of dbt-common to test against" | |
| type: string | |
| required: true | |
| default: "main" | |
| permissions: read-all | |
| # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| # top-level adjustments can be made here | |
| env: | |
| # number of parallel processes to spawn for python integration testing | |
| PYTHON_INTEGRATION_TEST_WORKERS: 5 | |
| jobs: | |
| job-prep: | |
| # This allow us to run the workflow on pull_requests as well so we can always run unit tests | |
| # and only run integration tests on merge for time purposes | |
| name: Setup Repo Refs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| dbt-core-ref: ${{ steps.ref.outputs.dbt-core-ref }} | |
| dbt-common-ref: ${{ steps.ref.outputs.dbt-common-ref }} | |
| steps: | |
| - name: "Display job inputs" | |
| id: job-inputs | |
| run: | | |
| echo "inputs.dbt-core-ref=${{ inputs.dbt-core-ref }}" | |
| echo "inputs.dbt-common-ref=${{ inputs.dbt-common-ref }}" | |
| - name: "Set up job outputs" | |
| id: ref | |
| run: | | |
| if [[ -z "${{ inputs.dbt-core-ref }}" ]]; then | |
| dbt-core-ref=${{ inputs.dbt-core-ref }} | |
| else | |
| dbt-core-ref=main | |
| fi | |
| if [[ -z "${{ inputs.dbt-common-ref }}" ]]; then | |
| dbt-common-ref=${{ inputs.dbt-common-ref }} | |
| else | |
| if [[ "${{ github.event_name }}" == "merge_group" ]]; then | |
| dbt-common-ref=${{ github.event.pull_request.merge_commit_sha }} | |
| else | |
| dbt-common-ref=${{ github.event.pull_request.base.sha }} | |
| fi | |
| fi | |
| echo "dbt-core-ref=$dbt-core-ref" >> $GITHUB_OUTPUT | |
| echo "dbt-common-ref=$dbt-common-ref" >> $GITHUB_OUTPUT | |
| - name: "Display Refs" | |
| run: | | |
| echo "dbt-core-ref=${{ steps.ref.outputs.dbt-core-ref }}" | |
| echo "dbt-common-ref=${{ steps.ref.outputs.dbt-common-ref }}" | |
| dbt-core-unit-test: | |
| name: "Run unit tests for dbt-core" | |
| needs: [job-prep] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: "Check out dbt-core@${{ needs.job-prep.outputs.dbt-core-ref }}" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: dbt-labs/dbt-core | |
| ref: ${{ needs.job-prep.outputs.dbt-core-ref }} | |
| - name: "Set up Python 3.11" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: "Install python tools" | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip --version | |
| # tox takes care of installing the correct version of dbt-core dependencies but we need to | |
| # install them first s that we can override the dbt-common branch | |
| - name: "Manually install dbt-core dependencies" | |
| run: | | |
| python -m pip install -r dev-requirements.txt -r editable-requirements.txt | |
| # Since the dbt-common dependency is pinned in dev-requirements.txt we need to force update it | |
| # Since tox installs dependencies but doesn't force update, it won't get overridden in the next | |
| # ssstep since the requirements will already be met | |
| - name: "Force update the version of dbt-common@${{ needs.job-prep.outputs.dbt-common-ref }}" | |
| run: | | |
| pip install pip install git+https://github.com/dbt-labs/dbt-common.git@${{ needs.job-prep.outputs.dbt-common-ref }} --force-reinstall | |
| - name: Run unit tests | |
| run: tox | |
| env: | |
| TOXENV: unit | |
| - name: "Check installed versions" | |
| run: pip freeze | |
| # dbt-core-integration-metadata: | |
| # name: integration test metadata generation | |
| # runs-on: ubuntu-latest | |
| # if: ${{ github.event_name != 'pull_request' }} | |
| # outputs: | |
| # split-groups: ${{ steps.generate-split-groups.outputs.split-groups }} | |
| # include: ${{ steps.generate-include.outputs.include }} | |
| # steps: | |
| # - name: generate split-groups | |
| # id: generate-split-groups | |
| # run: | | |
| # MATRIX_JSON="[" | |
| # for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do | |
| # MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}") | |
| # done | |
| # MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}" | |
| # MATRIX_JSON+="]" | |
| # echo "split-groups=${MATRIX_JSON}" | |
| # echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT | |
| # - name: generate include | |
| # id: generate-include | |
| # run: | | |
| # INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-latest"' ) | |
| # INCLUDE_GROUPS="[" | |
| # for include in ${INCLUDE[@]}; do | |
| # for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do | |
| # INCLUDE_GROUPS+=$(sed 's/$/, /' <<< "{\"split-group\":\"${group}\",${include}}") | |
| # done | |
| # done | |
| # INCLUDE_GROUPS=$(echo $INCLUDE_GROUPS | sed 's/,*$//g') | |
| # INCLUDE_GROUPS+="]" | |
| # echo "include=${INCLUDE_GROUPS}" | |
| # echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT | |
| # dbt-core-integration-tests: | |
| # name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }} | |
| # if: ${{ github.event_name != 'pull_request' }} | |
| # runs-on: ${{ matrix.os }} | |
| # timeout-minutes: 30 | |
| # needs: [job-prep, dbt-core-integration-metadata] | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| # os: [ubuntu-20.04] | |
| # split-group: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.split-groups) }} | |
| # include: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.include) }} | |
| # env: | |
| # DBT_INVOCATION_ENV: github-actions | |
| # DBT_TEST_USER_1: dbt_test_user_1 | |
| # DBT_TEST_USER_2: dbt_test_user_2 | |
| # DBT_TEST_USER_3: dbt_test_user_3 | |
| # steps: | |
| # - name: "Check out the repository@${{ needs.job-prep.outputs.dbt-core-ref }}" | |
| # uses: actions/checkout@v4 | |
| # with: | |
| # repository: dbt-labs/dbt-core | |
| # ref: ${{ needs.job-prep.outputs.dbt-core-ref }} | |
| # - name: Set up Python ${{ matrix.python-version }} | |
| # uses: actions/setup-python@v5 | |
| # with: | |
| # python-version: ${{ matrix.python-version }} | |
| # - name: Set up postgres (linux) | |
| # if: runner.os == 'Linux' | |
| # uses: ./.github/actions/setup-postgres-linux | |
| # - name: Set up postgres (macos) | |
| # if: runner.os == 'macOS' | |
| # uses: ./.github/actions/setup-postgres-macos | |
| # - name: Set up postgres (windows) | |
| # if: runner.os == 'Windows' | |
| # uses: ./.github/actions/setup-postgres-windows | |
| # - name: "Install python tools & dependencies" | |
| # run: | | |
| # python -m pip install --user --upgrade pip | |
| # python -m pip --version | |
| # python -m pip install -r dev-requirements.txt -r editable-requirements.txt | |
| # - name: "Check installed versions" | |
| # run: pip freeze | |
| # - name: "Update the version of dbt-common@${{ needs.job-prep.outputs.dbt-common-ref }}s" | |
| # run: | | |
| # pip install pip install git+https://github.com/dbt-labs/dbt-common.git@${{ needs.job-prep.outputs.dbt-common-ref }} --force-reinstall | |
| # - name: "Check installed versions" | |
| # run: pip freeze | |
| # - name: Run tests | |
| # run: | | |
| # python -m pytest {posargs} tests/functional -k "not tests/functional/graph_selection" | |
| # python -m pytest {posargs} tests/functional/graph_selection | |
| # python -m pytest {posargs} tests/adapter | |
| # env: | |
| # PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }} | |
| # integration-report: | |
| # if: ${{ always() }} | |
| # name: Integration Test Suite | |
| # runs-on: ubuntu-latest | |
| # needs: [dbt-core-integration-tests] | |
| # steps: | |
| # - name: "Integration Tests Failed" | |
| # if: ${{ contains(needs.dbt-core-integration-tests.result, 'failure') || contains(needs.dbt-core-integration-tests.result, 'cancelled') }} | |
| # # when this is true the next step won't execute | |
| # run: | | |
| # echo "::notice title='Integration test suite failed'" | |
| # exit 1 | |
| # - name: "Integration Tests Passed" | |
| # if: ${{ github.event_name != 'pull_request' }} | |
| # run: | | |
| # echo "::notice title='Integration test suite passed'" | |
| # - name: "Integration Tests Skipped" | |
| # if: ${{ github.event_name == 'pull_request' }} | |
| # run: | | |
| # echo "::notice title='Integration test suite skipped on Pull Requests'" |