Skip to content

Commit

Permalink
Integration workflow update to support all-in-one adapter testing (#1082
Browse files Browse the repository at this point in the history
)
  • Loading branch information
gshank authored Aug 1, 2024
1 parent 46348bc commit 2580ac5
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 27 deletions.
17 changes: 0 additions & 17 deletions .github/scripts/update_dbt_core_branch.sh

This file was deleted.

21 changes: 21 additions & 0 deletions .github/scripts/update_dev_dependency_branches.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/bin/bash -e
set -e


dbt_adapters_branch=$1
dbt_core_branch=$2
dbt_common_branch=$3
target_req_file="dev-requirements.txt"
core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g"
adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g"
common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g"
if [[ "$OSTYPE" == darwin* ]]; then
# mac ships with a different version of sed that requires a delimiter arg
sed -i "" "$adapters_req_sed_pattern" $target_req_file
sed -i "" "$core_req_sed_pattern" $target_req_file
sed -i "" "$common_req_sed_pattern" $target_req_file
else
sed -i "$adapters_req_sed_pattern" $target_req_file
sed -i "$core_req_sed_pattern" $target_req_file
sed -i "$common_req_sed_pattern" $target_req_file
fi
53 changes: 43 additions & 10 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@

name: Adapter Integration Tests

run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') && inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}"

on:
push:
branches:
Expand All @@ -24,10 +26,31 @@ on:

workflow_dispatch:
inputs:
dbt-core-branch:
description: "branch of dbt-core to use in dev-requirements.txt"
name:
description: "Name to associate with run (example: 'dbt-adapters-242')"
required: false
type: string
default: "Adapter Integration Tests"
adapter_branch:
description: "The branch of this adapter repository to use"
type: string
required: false
default: "main"
dbt_adapters_branch:
description: "The branch of dbt-adapters to use"
type: string
required: false
default: "main"
dbt_core_branch:
description: "The branch of dbt-core to use"
type: string
required: false
default: "main"
dbt_common_branch:
description: "The branch of dbt-common to use"
type: string
required: false
default: "main"

# explicitly turn off permissions for `GITHUB_TOKEN`
permissions: read-all
Expand Down Expand Up @@ -74,12 +97,19 @@ jobs:
DBT_TEST_USER_3: "[email protected]"

steps:
- name: Check out the repository
if: github.event_name != 'pull_request_target'
- name: Check out the repository (push)
if: github.event_name == 'push'
uses: actions/checkout@v4
with:
persist-credentials: false

- name: Check out the repository (workflow_dispatch)
if: github.event_name == 'workflow_dispatch'
uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ inputs.adapter_branch }}

# explicitly checkout the branch for the PR,
# this is necessary for the `pull_request` event
- name: Check out the repository (PR)
Expand All @@ -95,17 +125,20 @@ jobs:
with:
python-version: "3.11"

- name: Update Adapters and Core branches (update dev_requirements.txt)
if: github.event_name == 'workflow_dispatch'
run: |
./.github/scripts/update_dev_dependency_branches.sh \
${{ inputs.dbt_adapters_branch }} \
${{ inputs.dbt_core_branch }} \
${{ inputs.dbt_common_branch }}
cat dev-requirements.txt
- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip --version
python -m pip install -r dagger/requirements.txt
- name: Update dev_requirements.txt
if: inputs.dbt-core-branch != ''
run: |
pip install bumpversion
./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}
- name: Run tests for ${{ matrix.test }}
run: python dagger/run_dbt_spark_tests.py --profile ${{ matrix.test }}

0 comments on commit 2580ac5

Please sign in to comment.