Test dbt-core@paw/integration-optimizations with dbt-common@paw/integration-optimizations #8
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # **what?** | |
| # Runs all tests in dbt-core with this branch of dbt-common to ensure nothing is broken | |
| # **why?** | |
| # Ensure dbt-common changes do not break dbt-core | |
| # **when?** | |
| # This will run when trying to merge a PR into main. | |
| # It can also be manually triggered. | |
| name: "Test Branches of dbt-core + dbt-common" | |
| run-name: "Test dbt-core@${{ inputs.dbt-core-ref }} with dbt-common@${{ inputs.dbt-common-ref }}" | |
| on: | |
| merge_group: | |
| types: [checks_requested] | |
| workflow_dispatch: | |
| inputs: | |
| dbt-core-ref: | |
| description: "The branch of dbt-core to test against" | |
| default: "main" | |
| dbt-common-ref: | |
| description: "The branch of dbt-common to test against" | |
| default: "main" | |
| workflow_call: | |
| inputs: | |
| dbt-core-ref: | |
| description: "The branch of dbt-core to test against" | |
| type: string | |
| required: true | |
| default: "main" | |
| dbt-common-ref: | |
| description: "The branch of dbt-common to test against" | |
| type: string | |
| required: true | |
| default: "main" | |
| permissions: read-all | |
| defaults: | |
| run: | |
| shell: bash | |
| # top-level adjustments can be made here | |
| env: | |
| # number of parallel processes to spawn for python integration testing | |
| PYTHON_INTEGRATION_TEST_WORKERS: 5 | |
| jobs: | |
| # TODO: maybe split these into 2 workflows. Then we can call unit tests always and just do int tests on merge? or just use an trigger checks | |
| dbt-core-unit-test: | |
| name: "Run unit tests for dbt-core" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: "Check out dbt-core" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: dbt-labs/dbt-core | |
| ref: ${{ github.event.inputs.dbt-core-ref }} | |
| - name: "Set up Python 3.11" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install python dependencies | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip --version | |
| python -m pip install tox | |
| tox --version | |
| - name: "Update the version of dbt-common" | |
| run: | | |
| pip install pip install git+https://github.com/dbt-labs/dbt-common.git@${{ inputs.dbt-common-ref }} | |
| - name: "Check installed versions" | |
| run: pip freeze | |
| - name: Run tox - unit tests | |
| env: | |
| TOXENV: "unit" | |
| run: tox | |
| dbt-core-integration-metadata: | |
| name: integration test metadata generation | |
| runs-on: ubuntu-latest | |
| outputs: | |
| split-groups: ${{ steps.generate-split-groups.outputs.split-groups }} | |
| include: ${{ steps.generate-include.outputs.include }} | |
| steps: | |
| - name: generate split-groups | |
| id: generate-split-groups | |
| run: | | |
| MATRIX_JSON="[" | |
| for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do | |
| MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}") | |
| done | |
| MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}" | |
| MATRIX_JSON+="]" | |
| echo "split-groups=${MATRIX_JSON}" | |
| echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT | |
| - name: generate include | |
| id: generate-include | |
| run: | | |
| INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-latest"' ) | |
| INCLUDE_GROUPS="[" | |
| for include in ${INCLUDE[@]}; do | |
| for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do | |
| INCLUDE_GROUPS+=$(sed 's/$/, /' <<< "{\"split-group\":\"${group}\",${include}}") | |
| done | |
| done | |
| INCLUDE_GROUPS=$(echo $INCLUDE_GROUPS | sed 's/,*$//g') | |
| INCLUDE_GROUPS+="]" | |
| echo "include=${INCLUDE_GROUPS}" | |
| echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT | |
| dbt-core-integration-tests: | |
| name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| needs: [dbt-core-integration-metadata] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| os: [ubuntu-20.04] | |
| split-group: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.split-groups) }} | |
| include: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.include) }} | |
| env: | |
| TOXENV: integration | |
| DBT_INVOCATION_ENV: github-actions | |
| DBT_TEST_USER_1: dbt_test_user_1 | |
| DBT_TEST_USER_2: dbt_test_user_2 | |
| DBT_TEST_USER_3: dbt_test_user_3 | |
| # DD_CIVISIBILITY_AGENTLESS_ENABLED: true | |
| # DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} TODO: set this up | |
| # DD_SITE: datadoghq.com | |
| # DD_ENV: ci | |
| DD_SERVICE: ${{ github.event.repository.name }} | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: dbt-labs/dbt-core | |
| ref: ${{ github.event.inputs.dbt-core-ref }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up postgres (linux) | |
| if: runner.os == 'Linux' | |
| uses: ./.github/actions/setup-postgres-linux | |
| - name: Set up postgres (macos) | |
| if: runner.os == 'macOS' | |
| uses: ./.github/actions/setup-postgres-macos | |
| - name: Set up postgres (windows) | |
| if: runner.os == 'Windows' | |
| uses: ./.github/actions/setup-postgres-windows | |
| - name: Install python tools | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip --version | |
| python -m pip install tox | |
| tox --version | |
| - name: "Update the version of dbt-common" | |
| run: | | |
| pip install pip install git+https://github.com/dbt-labs/dbt-common.git@${{ inputs.dbt-common-ref }} | |
| - name: "Check installed versions" | |
| run: pip freeze | |
| - name: Run tests | |
| run: tox -- --ddtrace | |
| env: | |
| PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }} | |
| integration-report: | |
| if: ${{ always() }} | |
| name: Integration Test Suite | |
| runs-on: ubuntu-latest | |
| needs: [dbt-core-integration-tests] | |
| steps: | |
| - name: "Integration Tests Failed" | |
| if: ${{ contains(needs.dbt-core-integration-tests.result, 'failure') || contains(needs.dbt-core-integration-tests.result, 'cancelled') }} | |
| # when this is true the next step won't execute | |
| run: | | |
| echo "::notice title='Integration test suite failed'" | |
| exit 1 | |
| - name: "Integration Tests Passed" | |
| run: | | |
| echo "::notice title='Integration test suite passed'" |