Test dbt-core@main with dbt-common@main #12
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # **what?** | |
| # Runs all tests in dbt-core with this branch of dbt-common to ensure nothing is broken | |
| # **why?** | |
| # Ensure dbt-common changes do not break dbt-core | |
| # **when?** | |
| # This will run when trying to merge a PR into main. | |
| # It can also be manually triggered. | |
| name: "Test Branches of dbt-core + dbt-common" | |
| run-name: "Test dbt-core@${{ inputs.dbt-core-ref }} with dbt-common@${{ inputs.dbt-common-ref }}" | |
| on: | |
| merge_group: | |
| types: [checks_requested] | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| dbt-core-ref: | |
| description: "The branch of dbt-core to test against" | |
| default: "main" | |
| dbt-common-ref: | |
| description: "The branch of dbt-common to test against" | |
| default: "main" | |
| workflow_call: | |
| inputs: | |
| dbt-core-ref: | |
| description: "The branch of dbt-core to test against" | |
| type: string | |
| required: true | |
| default: "main" | |
| dbt-common-ref: | |
| description: "The branch of dbt-common to test against" | |
| type: string | |
| required: true | |
| default: "main" | |
| permissions: read-all | |
| defaults: | |
| run: | |
| shell: bash | |
| # top-level adjustments can be made here | |
| env: | |
| # number of parallel processes to spawn for python integration testing | |
| PYTHON_INTEGRATION_TEST_WORKERS: 5 | |
| jobs: | |
| job-prep: | |
| # This allow us to run the workflow on pull_requests as well so we can always run unit tests | |
| # and only run integration tests on merge for time purposes | |
| name: Setup Repo Refs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| dbt-core-ref: ${{ steps.ref.outputs.dbt-core-ref }} | |
| dbt-common-ref: ${{ steps.ref.outputs.dbt-common-ref }} | |
| steps: | |
| - name: Set up job inputs | |
| id: job-inputs | |
| run: | | |
| echo "dbt-core-ref=${{ github.event.inputs.dbt-core-ref }}" | |
| echo "dbt-common-ref=${{ github.event.inputs.dbt-common-ref }}" | |
| echo "dbt-core-ref=${{ github.event.inputs.dbt-core-ref }}" >> $GITHUB_OUTPUT | |
| echo "dbt-common-ref=${{ github.event.inputs.dbt-common-ref }}" >> $GITHUB_OUTPUT | |
| - name: Set up job outputs | |
| id: ref | |
| run: | | |
| if [[ -z "${{ github.event.inputs.dbt-core-ref }}" ]]; then | |
| echo "dbt-core-ref=${{ github.event.inputs.dbt-core-ref }}" | |
| else | |
| echo "dbt-core-ref=main" | |
| fi | |
| if [[ -z "${{ github.event.inputs.dbt-common-ref }}" ]]; then | |
| echo "dbt-common-ref=${{ github.event.inputs.dbt-common-ref }}" | |
| else | |
| echo "dbt-common-ref=main" | |
| fi | |
| echo "dbt-core-ref=${{ github.event.inputs.dbt-core-ref }}" >> $GITHUB_OUTPUT | |
| echo "dbt-common-ref=${{ github.event.inputs.dbt-common-ref }}" >> $GITHUB_OUTPUT | |
| # TODO: maybe split these into 2 workflows. Then we can call unit tests always and just do int tests on merge? or just use an trigger checks | |
| dbt-core-unit-test: | |
| name: "Run unit tests for dbt-core" | |
| needs: [job-prep] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: "Check out dbt-core" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: dbt-labs/dbt-core | |
| ref: ${{ needs.job-prep.outputs.dbt-core-ref }} | |
| - name: "Set up Python 3.11" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: "Install python tools & dependencies" | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip --version | |
| python -m pip install -r dev-requirements.txt -r editable-requirements.txt | |
| - name: "Check installed versions" | |
| run: pip freeze | |
| - name: "Update the version of dbt-common" | |
| run: | | |
| pip install pip install git+https://github.com/dbt-labs/dbt-common.git@${{ inputs.dbt-common-ref }} | |
| - name: "Check installed versions" | |
| run: pip freeze | |
| - name: Run unit tests | |
| run: python -m pytest tests/unit | |
| dbt-core-integration-metadata: | |
| name: integration test metadata generation | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event.name != 'pull_request' }} | |
| outputs: | |
| split-groups: ${{ steps.generate-split-groups.outputs.split-groups }} | |
| include: ${{ steps.generate-include.outputs.include }} | |
| steps: | |
| - name: generate split-groups | |
| id: generate-split-groups | |
| run: | | |
| MATRIX_JSON="[" | |
| for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do | |
| MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}") | |
| done | |
| MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}" | |
| MATRIX_JSON+="]" | |
| echo "split-groups=${MATRIX_JSON}" | |
| echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT | |
| - name: generate include | |
| id: generate-include | |
| run: | | |
| INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-latest"' ) | |
| INCLUDE_GROUPS="[" | |
| for include in ${INCLUDE[@]}; do | |
| for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do | |
| INCLUDE_GROUPS+=$(sed 's/$/, /' <<< "{\"split-group\":\"${group}\",${include}}") | |
| done | |
| done | |
| INCLUDE_GROUPS=$(echo $INCLUDE_GROUPS | sed 's/,*$//g') | |
| INCLUDE_GROUPS+="]" | |
| echo "include=${INCLUDE_GROUPS}" | |
| echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT | |
| dbt-core-integration-tests: | |
| name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }} | |
| if: ${{ github.event.name != 'pull_request' }} | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| needs: [job-prep, dbt-core-integration-metadata] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| os: [ubuntu-20.04] | |
| split-group: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.split-groups) }} | |
| include: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.include) }} | |
| env: | |
| DBT_INVOCATION_ENV: github-actions | |
| DBT_TEST_USER_1: dbt_test_user_1 | |
| DBT_TEST_USER_2: dbt_test_user_2 | |
| DBT_TEST_USER_3: dbt_test_user_3 | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: dbt-labs/dbt-core | |
| ref: ${{ needs.job-prep.outputs.dbt-core-ref }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up postgres (linux) | |
| if: runner.os == 'Linux' | |
| uses: ./.github/actions/setup-postgres-linux | |
| - name: Set up postgres (macos) | |
| if: runner.os == 'macOS' | |
| uses: ./.github/actions/setup-postgres-macos | |
| - name: Set up postgres (windows) | |
| if: runner.os == 'Windows' | |
| uses: ./.github/actions/setup-postgres-windows | |
| - name: "Install python tools & dependencies" | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip --version | |
| python -m pip install -r dev-requirements.txt -r editable-requirements.txt | |
| - name: "Check installed versions" | |
| run: pip freeze | |
| - name: "Update the version of dbt-common" | |
| run: | | |
| pip install pip install git+https://github.com/dbt-labs/dbt-common.git@${{ needs.job-prep.outputs.dbt-common-ref }} | |
| - name: "Check installed versions" | |
| run: pip freeze | |
| - name: Run tests | |
| run: | | |
| python -m pytest {posargs} tests/functional -k "not tests/functional/graph_selection" | |
| python -m pytest {posargs} tests/functional/graph_selection | |
| python -m pytest {posargs} tests/adapter | |
| env: | |
| PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }} | |
| integration-report: | |
| if: ${{ always() }} | |
| name: Integration Test Suite | |
| runs-on: ubuntu-latest | |
| needs: [dbt-core-integration-tests] | |
| steps: | |
| - name: "Integration Tests Failed" | |
| if: ${{ contains(needs.dbt-core-integration-tests.result, 'failure') || contains(needs.dbt-core-integration-tests.result, 'cancelled') }} | |
| # when this is true the next step won't execute | |
| run: | | |
| echo "::notice title='Integration test suite failed'" | |
| exit 1 | |
| - name: "Integration Tests Passed" | |
| if: ${{ github.event.name != 'pull_request' }} | |
| run: | | |
| echo "::notice title='Integration test suite passed'" | |
| - name: "Integration Tests Skipped" | |
| if: ${{ github.event.name == 'pull_request' }} | |
| run: | | |
| echo "::notice title='Integration test suite skipped on Pull Requests'" |