From a1cf81517c6c279c0e0cf30e8d81e9e215ecbb9d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 20 Apr 2024 11:07:35 -0400 Subject: [PATCH 01/15] draft commit --- .bumpversion.cfg | 35 -- .../actions/update-dev-branches/action.yml | 38 ++ .../update-dev-branches/update_dev_branch.sh | 15 + .github/dependabot.yml | 9 +- .github/scripts/integration-test-matrix.js | 87 ---- .github/scripts/update_dbt_core_branch.sh | 20 - .github/scripts/update_dependencies.sh | 18 - .github/scripts/update_release_branch.sh | 11 - .github/workflows/backport.yml | 38 +- .github/workflows/bot-changelog.yml | 59 ++- .github/workflows/build-artifacts.yml | 61 +++ .github/workflows/changelog-check.yml | 46 ++ .github/workflows/changelog-existence.yml | 42 -- .github/workflows/clean-repo.yml | 27 ++ .github/workflows/code-quality.yml | 86 ++++ .github/workflows/cut-release-branch.yml | 42 -- .github/workflows/docs-issues.yml | 56 +-- .github/workflows/integration-tests.yml | 145 ++++++ .github/workflows/integration.yml | 244 ---------- .github/workflows/main.yml | 213 --------- .github/workflows/nightly-release.yml | 116 ----- .github/workflows/publish-pypi-internal.yml | 36 ++ .github/workflows/release-branch-tests.yml | 31 -- .github/workflows/release-internal.yml | 49 -- .github/workflows/release.yml | 418 +++++++++--------- .github/workflows/repository-cleanup.yml | 30 -- .github/workflows/scheduled-test-releases.yml | 72 +++ .github/workflows/scheduled-tests.yml | 72 +++ .github/workflows/stale-issues.yml | 13 + .github/workflows/stale.yml | 12 - .github/workflows/triage-labels.yml | 28 +- .github/workflows/unit-tests.yml | 102 +++++ .github/workflows/version-bump.yml | 28 -- MANIFEST.in | 1 - Makefile | 67 --- dev-requirements.txt | 26 -- pyproject.toml | 137 ++++++ pytest.ini | 9 - setup.py | 87 ---- test.env.example | 11 +- tox.ini | 34 -- 41 files changed, 1152 insertions(+), 1519 deletions(-) delete mode 100644 .bumpversion.cfg create mode 100644 .github/actions/update-dev-branches/action.yml create mode 100755 .github/actions/update-dev-branches/update_dev_branch.sh delete mode 100644 .github/scripts/integration-test-matrix.js delete mode 100755 .github/scripts/update_dbt_core_branch.sh delete mode 100644 .github/scripts/update_dependencies.sh delete mode 100644 .github/scripts/update_release_branch.sh create mode 100644 .github/workflows/build-artifacts.yml create mode 100644 .github/workflows/changelog-check.yml delete mode 100644 .github/workflows/changelog-existence.yml create mode 100644 .github/workflows/clean-repo.yml create mode 100644 .github/workflows/code-quality.yml delete mode 100644 .github/workflows/cut-release-branch.yml create mode 100644 .github/workflows/integration-tests.yml delete mode 100644 .github/workflows/integration.yml delete mode 100644 .github/workflows/main.yml delete mode 100644 .github/workflows/nightly-release.yml create mode 100644 .github/workflows/publish-pypi-internal.yml delete mode 100644 .github/workflows/release-branch-tests.yml delete mode 100644 .github/workflows/release-internal.yml delete mode 100644 .github/workflows/repository-cleanup.yml create mode 100644 .github/workflows/scheduled-test-releases.yml create mode 100644 .github/workflows/scheduled-tests.yml create mode 100644 .github/workflows/stale-issues.yml delete mode 100644 .github/workflows/stale.yml create mode 100644 .github/workflows/unit-tests.yml delete mode 100644 .github/workflows/version-bump.yml delete mode 100644 MANIFEST.in delete mode 100644 Makefile delete mode 100644 dev-requirements.txt create mode 100644 pyproject.toml delete mode 100644 pytest.ini delete mode 100644 setup.py delete mode 100644 tox.ini diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 52cc0acca..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[bumpversion] -current_version = 1.8.0b3 -parse = (?P[\d]+) # major version number - \.(?P[\d]+) # minor version number - \.(?P[\d]+) # patch version number - (?P # optional pre-release - ex: a1, b2, rc25 - (?Pa|b|rc) # pre-release type - (?P[\d]+) # pre-release version number - )? - ( # optional nightly release indicator - \.(?Pdev[0-9]+) # ex: .dev02142023 - )? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0` -serialize = - {major}.{minor}.{patch}{prekind}{num}.{nightly} - {major}.{minor}.{patch}.{nightly} - {major}.{minor}.{patch}{prekind}{num} - {major}.{minor}.{patch} -commit = False -tag = False - -[bumpversion:part:prekind] -first_value = a -optional_value = final -values = - a - b - rc - final - -[bumpversion:part:num] -first_value = 1 - -[bumpversion:part:nightly] - -[bumpversion:file:dbt/adapters/redshift/__version__.py] diff --git a/.github/actions/update-dev-branches/action.yml b/.github/actions/update-dev-branches/action.yml new file mode 100644 index 000000000..a177e69d2 --- /dev/null +++ b/.github/actions/update-dev-branches/action.yml @@ -0,0 +1,38 @@ +name: "Update dev branches" +description: "Update dev branches for testing cross-repo changes" + +inputs: + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + default: "" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + default: "" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + default: "" + +runs: + using: composite + steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo dbt-adapters : ${{ inputs.dbt-adapters-branch }} + echo dbt-common : ${{ inputs.dbt-common-branch }} + echo dbt-core : ${{ inputs.dbt-core-branch }} + + - name: "Update `dbt-adapters` branch" + if: ${{ inputs.dbt-adapters-branch }} + shell: bash + run: ./.github/actions/update-dev-branches/update_dev_branch.sh "dbt-adapters" ${{ inputs.dbt-adapters-branch }} + + - name: "Update `dbt-common` branch" + if: ${{ inputs.dbt-common-branch }} + shell: bash + run: ./.github/actions/update-dev-branches/update_dev_branch.sh "dbt-common" ${{ inputs.dbt-common-branch }} + + - name: "Update `dbt-core` branch" + if: ${{ inputs.dbt-core-branch }} + shell: bash + run: ./.github/actions/update-dev-branches/update_dev_branch.sh "dbt-core" ${{ inputs.dbt-core-branch }} diff --git a/.github/actions/update-dev-branches/update_dev_branch.sh b/.github/actions/update-dev-branches/update_dev_branch.sh new file mode 100755 index 000000000..4d51f9211 --- /dev/null +++ b/.github/actions/update-dev-branches/update_dev_branch.sh @@ -0,0 +1,15 @@ +#!/bin/bash -e +set -e + +package=$1 +branch=$2 + +file="pyproject.toml" +sed_pattern="s|${package}.git@main*|${package}.git@${branch}|g" + +# mac ships with a different version of sed that requires a delimiter arg +if [[ "$OSTYPE" == darwin* ]]; then + sed -i "" "$sed_pattern" $file +else + sed -i "$sed_pattern" $file +fi diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4673f47cf..6a59ed01d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,12 +1,13 @@ version: 2 + updates: - - package-ecosystem: "pip" +- package-ecosystem: "pip" directory: "/" schedule: - interval: "daily" + interval: "daily" rebase-strategy: "disabled" - - package-ecosystem: "github-actions" +- package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "weekly" rebase-strategy: "disabled" diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js deleted file mode 100644 index 7db445d9e..000000000 --- a/.github/scripts/integration-test-matrix.js +++ /dev/null @@ -1,87 +0,0 @@ -module.exports = ({ context }) => { - const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11"]; - const supportedAdapters = ["redshift"]; - - // if PR, generate matrix based on files changed and PR labels - if (context.eventName.includes("pull_request")) { - // `changes` is a list of adapter names that have related - // file changes in the PR - // ex: ['postgres', 'snowflake'] - const labels = context.payload.pull_request.labels.map(({ name }) => name); - console.log("labels", labels); - const testAllLabel = labels.includes("test all"); - const include = []; - - for (const adapter of supportedAdapters) { - for (const pythonVersion of supportedPythonVersions) { - if ( - pythonVersion === defaultPythonVersion || - labels.includes(`test python${pythonVersion}`) || - testAllLabel - ) { - // always run tests on ubuntu by default - include.push({ - os: "ubuntu-latest", - adapter, - "python-version": pythonVersion, - }); - - if (labels.includes("test windows") || testAllLabel) { - include.push({ - os: "windows-latest", - adapter, - "python-version": pythonVersion, - }); - } - - if (labels.includes("test macos") || testAllLabel) { - include.push({ - os: "macos-latest", - adapter, - "python-version": pythonVersion, - }); - } - } - } - } - - console.log("matrix", { include }); - - return { - include, - }; - } - // if not PR, generate matrix of python version, adapter, and operating - // system to run integration tests on - - const include = []; - // run for all adapters and python versions on ubuntu - for (const adapter of supportedAdapters) { - for (const pythonVersion of supportedPythonVersions) { - include.push({ - os: 'ubuntu-latest', - adapter: adapter, - "python-version": pythonVersion, - }); - } - } - - // additionally include runs for all adapters, on macos and windows, - // but only for the default python version - for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-latest"]) { - include.push({ - os: operatingSystem, - adapter: adapter, - "python-version": defaultPythonVersion, - }); - } - } - - console.log("matrix", { include }); - - return { - include, - }; -}; diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh deleted file mode 100755 index d28a40c35..000000000 --- a/.github/scripts/update_dbt_core_branch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$postgres_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$postgres_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi -core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) -bumpversion --allow-dirty --new-version "$core_version" major diff --git a/.github/scripts/update_dependencies.sh b/.github/scripts/update_dependencies.sh deleted file mode 100644 index 6000b5006..000000000 --- a/.github/scripts/update_dependencies.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$postgres_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$postgres_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi diff --git a/.github/scripts/update_release_branch.sh b/.github/scripts/update_release_branch.sh deleted file mode 100644 index 75b9ccef6..000000000 --- a/.github/scripts/update_release_branch.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -e -set -e - -release_branch=$1 -target_req_file=".github/workflows/nightly-release.yml" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "s|[0-9].[0-9].latest|$release_branch|" $target_req_file -else - sed -i "s|[0-9].[0-9].latest|$release_branch|" $target_req_file -fi diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 46f240347..d9d500cf3 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -7,34 +7,32 @@ # Example: backport 1.0.latest # # You MUST "Squash and merge" the original PR or this won't work. - +# # **why?** # Changes sometimes need to be backported to release branches. # This automates the backporting process - +# # **when?** # Once a PR is "Squash and merge"'d, by adding a backport label, this is triggered +name: "Backport" -name: Backport on: - pull_request: - types: - - labeled + pull_request: + types: + - labeled permissions: - contents: write - pull-requests: write + contents: write + pull-requests: write jobs: - backport: - name: Backport - runs-on: ubuntu-latest - # Only react to merged PRs for security reasons. - # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target. - if: > - github.event.pull_request.merged - && contains(github.event.label.name, 'backport') - steps: - - uses: tibdex/backport@v2 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} + backport: + name: "Backport" + runs-on: ubuntu-latest + # Only react to merged PRs for security reasons + # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target + if: github.event.pull_request.merged && contains(github.event.label.name, 'backport') + steps: + - uses: tibdex/backport@v2 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/bot-changelog.yml b/.github/workflows/bot-changelog.yml index fcd344802..54832ed68 100644 --- a/.github/workflows/bot-changelog.yml +++ b/.github/workflows/bot-changelog.yml @@ -20,41 +20,36 @@ # # **how to add another bot?** # Add the label and changie kind to the include matrix. That's it! -# - -name: Bot Changelog +name: "Bot changelog" on: - pull_request: - # catch when the PR is opened with the label or when the label is added - types: [opened, labeled] + pull_request: + # catch when the PR is opened with the label or when the label is added + types: + - opened + - labeled permissions: - contents: write - pull-requests: read + contents: write + pull-requests: read jobs: - generate_changelog: - strategy: - matrix: - include: - - label: "dependencies" - changie_kind: "Dependencies" - - label: "snyk" - changie_kind: "Security" - runs-on: ubuntu-latest - - steps: - - - name: Create and commit changelog on bot PR - if: "contains(github.event.pull_request.labels.*.name, ${{ matrix.label }})" - id: bot_changelog - uses: emmyoop/changie_bot@v1 - with: - GITHUB_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} - commit_author_name: "Github Build Bot" - commit_author_email: "" - commit_message: "Add automated changelog yaml from template for bot PR" - changie_kind: ${{ matrix.changie_kind }} - label: ${{ matrix.label }} - custom_changelog_string: "custom:\n Author: ${{ github.event.pull_request.user.login }}\n PR: ${{ github.event.pull_request.number }}" + generate_changelog: + strategy: + matrix: + include: + - label: "dependencies" + changie_kind: "Dependencies" + runs-on: ubuntu-latest + steps: + - name: "Create and commit a changie entry for an automated PR" + if: ${{ contains(github.event.pull_request.labels.*.name, matrix.label) }} + uses: emmyoop/changie_bot@v1 + with: + GITHUB_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_author_name: "Github Build Bot" + commit_author_email: "" + commit_message: "[automated] changie entry" + changie_kind: ${{ matrix.changie_kind }} + label: ${{ matrix.label }} + custom_changelog_string: "custom:\n Author: ${{ github.event.pull_request.user.login }}\n PR: ${{ github.event.pull_request.number }}" diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 000000000..2907f7f1e --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,61 @@ +# **what?** +# Verifies python build on all code commited to the repository. This workflow +# should not require any secrets since it runs for PRs from forked repos. By +# default, secrets are not passed to workflows running from a forked repos. + +# **why?** +# Ensure code for dbt meets a certain quality standard. + +# **when?** +# This will run for all PRs, when code is pushed to main, and when manually triggered. +name: "Build release" + +on: + workflow_call: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + archive-name: + description: "The name to use for the upload archive, leave blank for no upload" + type: string + default: "" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + archive-name: + description: "The name to use for the upload archive, leave blank for no upload" + type: string + default: "" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.archive-name }}" + cancel-in-progress: true + +jobs: + build: + name: "Build a release" + runs-on: ubuntu-latest + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + - name: "Build ${{ github.event.repository.name }}" + uses: dbt-labs/dbt-adapters/.github/actions/build-artifacts@update-workflows + with: + archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/changelog-check.yml b/.github/workflows/changelog-check.yml new file mode 100644 index 000000000..dd4b11d76 --- /dev/null +++ b/.github/workflows/changelog-check.yml @@ -0,0 +1,46 @@ +# **what?** +# Checks that a file has been committed under the /.changes directory +# as a new CHANGELOG entry. Cannot check for a specific filename as +# it is dynamically generated by change type and timestamp. +# This workflow should not require any secrets since it runs for PRs +# from forked repos. +# By default, secrets are not passed to workflows running from +# a forked repo. +# +# **why?** +# Ensure code changes are reflected in the CHANGELOG. +# +# **when?** +# This will run for all PRs going into main and *.latest. It will run when: +# - the PR is opened or reopened +# - labels are updated on the PR +# - new code is pushed to the branch +# The action will get skipped if the 'Skip Changelog' label is present. +name: Check Changelog Entry + +on: + pull_request: + types: + - opened + - reopened + - labeled + - unlabeled + - synchronize + workflow_dispatch: + +defaults: + run: + shell: bash + +permissions: + contents: read + pull-requests: write + +jobs: + changelog: + uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main + with: + changelog_comment: "Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see the [dbt-redshift contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md)." + skip_label: "Skip Changelog" + # this is only acceptable because we own the action we're calling + secrets: inherit diff --git a/.github/workflows/changelog-existence.yml b/.github/workflows/changelog-existence.yml deleted file mode 100644 index 51c0a094a..000000000 --- a/.github/workflows/changelog-existence.yml +++ /dev/null @@ -1,42 +0,0 @@ -# **what?** -# Checks that a file has been committed under the /.changes directory -# as a new CHANGELOG entry. Cannot check for a specific filename as -# it is dynamically generated by change type and timestamp. -# This workflow should not require any secrets since it runs for PRs -# from forked repos. -# By default, secrets are not passed to workflows running from -# a forked repo. - -# **why?** -# Ensure code change gets reflected in the CHANGELOG. - -# **when?** -# This will run for all PRs going into main and *.latest. It will -# run when they are opened, reopened, when any label is added or removed -# and when new code is pushed to the branch. The action will then get -# skipped if the 'Skip Changelog' label is present is any of the labels. - -name: Check Changelog Entry - -on: - pull_request: - types: [opened, reopened, labeled, unlabeled, synchronize] - workflow_dispatch: - -defaults: - run: - shell: bash - -permissions: - contents: read - pull-requests: write - - -jobs: - changelog: - uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main - with: - changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see the [dbt-redshift contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md - ).' - skip_label: 'Skip Changelog' - secrets: inherit # this is only acceptable because we own the action we're calling diff --git a/.github/workflows/clean-repo.yml b/.github/workflows/clean-repo.yml new file mode 100644 index 000000000..2ce2837b3 --- /dev/null +++ b/.github/workflows/clean-repo.yml @@ -0,0 +1,27 @@ +# **what?** +# Cleanup branches left over from automation and testing. +# Also cleanup draft releases from release testing. +# +# **why?** +# The automations are leaving behind branches and releases that clutter the repository. +# Sometimes we need them to debug processes so we don't want them immediately deleted. +# Running on Saturday to avoid running at the same time as an actual release +# to prevent breaking a release mid-release. +# +# **when?** +# - every Saturday at noon UTC +# - manually +name: "Clean repo" + +on: + schedule: + - cron: '0 12 * * SAT' # noon UTC on Saturday - details in `why` above + workflow_dispatch: + +permissions: + contents: write + +jobs: + cleanup-repo: + uses: dbt-labs/actions/.github/workflows/repository-cleanup.yml@main + secrets: inherit diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 000000000..800de1a6f --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,86 @@ +# **what?** +# Run code quality checks, e.g. `black`, `flake8`, `mypy`, via `pre-commit` +# +# **why?** +# Ensure code quality meets dbt Labs standards +# +# **when?** +# - runs on all PRs into protected branches +# - runs indirectly during releases and release tests +# - can be manually run +name: "Code quality" + +on: + pull_request: + push: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch/tag to run code quality on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run code quality on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + cancel-in-progress: true + +jobs: + code-quality: + name: "Code quality" + runs-on: ubuntu-latest + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + persist-credentials: false + + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + - name: "Update development branches" + if: ${{ contains(github.event_name, 'workflow_') }} + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + + - name: "Run code quality" + shell: bash + run: hatch run code-quality diff --git a/.github/workflows/cut-release-branch.yml b/.github/workflows/cut-release-branch.yml deleted file mode 100644 index 1c487a8a8..000000000 --- a/.github/workflows/cut-release-branch.yml +++ /dev/null @@ -1,42 +0,0 @@ -# **what?** -# Calls a centralize3d workflow that will: -# 1. Cut a new branch (generally `*.latest`) -# 2. Also cleans up all files in `.changes/unreleased` and `.changes/previous version on -# `main` and bumps `main` to the input version. - -# **why?** -# Generally reduces the workload of engineers and reduces error. Allow automation. - -# **when?** -# This will run when called manually. - -name: Cut new release branch - -on: - workflow_dispatch: - inputs: - version_to_bump_main: - description: 'The alpha version main should bump to (ex. 1.6.0a1)' - required: true - new_branch_name: - description: 'The full name of the new branch (ex. 1.5.latest)' - required: true - -defaults: - run: - shell: bash - -permissions: - contents: write - -jobs: - cut_branch: - name: "Cut branch and clean up main for dbt-redshift" - uses: dbt-labs/actions/.github/workflows/cut-release-branch.yml@main - with: - version_to_bump_main: ${{ inputs.version_to_bump_main }} - new_branch_name: ${{ inputs.new_branch_name }} - PR_title: "Cleanup main after cutting new ${{ inputs.new_branch_name }} branch" - PR_body: "This PR will fail CI until the dbt-core PR has been merged due to release version conflicts." - secrets: - FISHTOWN_BOT_PAT: ${{ secrets.FISHTOWN_BOT_PAT }} diff --git a/.github/workflows/docs-issues.yml b/.github/workflows/docs-issues.yml index 00a098df8..ef98be5db 100644 --- a/.github/workflows/docs-issues.yml +++ b/.github/workflows/docs-issues.yml @@ -1,43 +1,43 @@ # **what?** # Open an issue in docs.getdbt.com when a PR is labeled `user docs` - +# # **why?** # To reduce barriers for keeping docs up to date - +# # **when?** # When a PR is labeled `user docs` and is merged. Runs on pull_request_target to run off the workflow already merged, # not the workflow that existed on the PR branch. This allows old PRs to get comments. - - -name: Open issues in docs.getdbt.com repo when a PR is labeled -run-name: "Open an issue in docs.getdbt.com for PR #${{ github.event.pull_request.number }}" +name: "Open issues in docs.getdbt.com repo when a PR is labeled" on: - pull_request_target: - types: [labeled, closed] + pull_request_target: + types: + - labeled + - closed defaults: - run: - shell: bash + run: + shell: bash permissions: - issues: write # opens new issues - pull-requests: write # comments on PRs - + issues: write # opens new issues + pull-requests: write # comments on PRs jobs: - open_issues: - # we only want to run this when the PR has been merged or the label in the labeled event is `user docs`. Otherwise it runs the - # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having - # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we - # decide if it should run or not. - if: | - (github.event.pull_request.merged == true) && - ((github.event.action == 'closed' && contains( github.event.pull_request.labels.*.name, 'user docs')) || - (github.event.action == 'labeled' && github.event.label.name == 'user docs')) - uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main - with: - issue_repository: "dbt-labs/docs.getdbt.com" - issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} PR #${{ github.event.pull_request.number }}" - issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." - secrets: inherit + open_issues: + # We only want to run this when the PR has been merged or the label in the labeled event is `user docs`. + # Otherwise, it runs the risk of duplication of issues being created due to merge and label both + # triggering this workflow to run and neither having generated the comment before the other runs. + # This lives here instead of the shared workflow because this is where we decide if it should run or not. + if: | + github.event.pull_request.merged == true && + ( + (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'user docs')) || + (github.event.action == 'labeled' && github.event.label.name == 'user docs') + ) + uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main + with: + issue_repository: "dbt-labs/docs.getdbt.com" + issue_title: "Docs Changes Needed from ${{ github.repository }} PR #${{ github.event.pull_request.number }}" + issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." + secrets: inherit diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..2873de21b --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,145 @@ +name: "Integration tests" + +on: + pull_request_target: + push: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + cancel-in-progress: true + +jobs: + require-approval-on-forks: + # if it's a pull request from a fork that has not been approved yet, don't run tests + if: >- + github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name != github.repository && + !contains(github.event.pull_request.labels.*.name, 'ok to test') + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: "Author requires permission to run tests" + uses: unsplash/comment-on-pr@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + msg: | + "Thanks for your contribution! "\ + "Since this is a fork, integration tests need to be approved prior to running. "\ + "@dbt-labs/core will review this PR and approve running integration tests." + check_for_duplicate_msg: true + + integration-tests: + name: "Integration tests" + needs: require-approval-on-forks + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ${{ fromJSON(vars.DBT_TEST_PYTHON_ALL_VERSIONS) }} + include: + - os: macos-latest + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + - os: windows-latest + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + env: + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_PORT: ${{ vars.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + DBT_TEST_USER_1: ${{ vars.REDSHIFT_TEST_USER_1 }} + DBT_TEST_USER_2: ${{ vars.REDSHIFT_TEST_USER_2 }} + DBT_TEST_USER_3: ${{ vars.REDSHIFT_TEST_USER_3 }} + DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }} + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: ${{ vars.DD_SITE }} + DD_ENV: ${{ vars.DD_ENV }} + DD_SERVICE: ${{ github.event.repository.name }} + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + persist-credentials: false + + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ matrix.python-version }} + + - name: "Update development branches" + if: ${{ contains(github.event_name, 'workflow_') }} + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + dbt-core-branch: ${{ inputs.dbt-core-branch }} + + - name: "Run integration tests" + shell: bash + run: hatch run integration-tests:all + + aggregate-results: + name: "Successful integration tests" + needs: integration-tests + if: ${{ !failure() && !cancelled() }} + runs-on: ubuntu-latest + steps: + - name: "Integration tests completed successfully" + shell: bash + run: | + title="Integration tests" + message="Integration tests completed successfully!" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index 9d1fe0807..000000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,244 +0,0 @@ -# **what?** -# This workflow runs all integration tests for supported OS -# and python versions and core adapters. If triggered by PR, -# the workflow will only run tests for adapters related -# to code changes. Use the `test all` and `test ${adapter}` -# label to run all or additional tests. Use `ok to test` -# label to mark PRs from forked repositories that are safe -# to run integration tests for. Requires secrets to run -# against different warehouses. - -# **why?** -# This checks the functionality of dbt from a user's perspective -# and attempts to catch functional regressions. - -# **when?** -# This workflow will run on every push to a protected branch -# and when manually triggered. It will also run for all PRs, including -# PRs from forks. The workflow will be skipped until there is a label -# to mark the PR as safe to run. - -name: Adapter Integration Tests - -on: - # pushes to release branches - push: - branches: - - "main" - - "develop" - - "*.latest" - - "releases/*" - # all PRs, important to note that `pull_request_target` workflows - # will run in the context of the target branch of a PR - pull_request_target: - # manual trigger - workflow_dispatch: - inputs: - dbt-core-branch: - description: "branch of dbt-core to use in dev-requirements.txt" - required: false - type: string - -# explicitly turn off permissions for `GITHUB_TOKEN` -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -# sets default shell to bash, for all operating systems -defaults: - run: - shell: bash - -jobs: - # generate test metadata about what files changed and the testing matrix to use - test-metadata: - # run if not a PR from a forked repository or has a label to mark as safe to test - if: >- - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest - - outputs: - matrix: ${{ steps.generate-matrix.outputs.result }} - - steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Check if relevant files changed - if: github.event_name == 'pull_request_target' - # https://github.com/marketplace/actions/paths-changes-filter - # For each filter, it sets output variable named by the filter to the text: - # 'true' - if any of changed files matches any of filter rules - # 'false' - if none of changed files matches any of filter rules - # also, returns: - # `changes` - JSON array with names of all filters matching any of the changed files - uses: dorny/paths-filter@v2 - id: get-changes - with: - token: ${{ secrets.GITHUB_TOKEN }} - filters: | - redshift: - - 'dbt/**' - - 'tests/**' - - 'dev-requirements.txt' - - - name: Generate integration test matrix - id: generate-matrix - uses: actions/github-script@v6 - env: - CHANGES: ${{ steps.get-changes.outputs.changes }} - with: - script: | - const script = require('./.github/scripts/integration-test-matrix.js') - const matrix = script({ context }) - console.log(matrix) - return matrix - test: - name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} - - # run if not a PR from a forked repository or has a label to mark as safe to test - # also checks that the matrix generated is not empty - if: >- - needs.test-metadata.outputs.matrix && - fromJSON( needs.test-metadata.outputs.matrix ).include[0] && - ( - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - ) - runs-on: ${{ matrix.os }} - - needs: test-metadata - - strategy: - fail-fast: false - max-parallel: 3 - matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} - - env: - TOXENV: integration-${{ matrix.adapter }} - PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" - DBT_INVOCATION_ENV: github-actions - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: datadoghq.com - DD_ENV: ci - DD_SERVICE: ${{ github.event.repository.name }} - - steps: - - name: Check out the repository - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - - # explicity checkout the branch for the PR, - # this is necessary for the `pull_request_target` event - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' - run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} - - - name: Run tox (redshift) - if: matrix.adapter == 'redshift' - env: - REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} - REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} - REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} - REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} - REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} - DBT_TEST_USER_1: dbt_test_user_1 - DBT_TEST_USER_2: dbt_test_user_2 - DBT_TEST_USER_3: dbt_test_user_3 - run: tox -- --ddtrace - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: logs - path: ./logs - - - name: Get current date - if: always() - id: date - run: | - echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv - path: integration_results.csv - - require-label-comment: - runs-on: ubuntu-latest - - needs: test - - permissions: - pull-requests: write - - steps: - - name: Needs permission PR comment - if: >- - needs.test.result == 'skipped' && - github.event_name == 'pull_request_target' && - github.event.pull_request.head.repo.full_name != github.repository - uses: unsplash/comment-on-pr@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - msg: | - "You do not have permissions to run integration tests, @dbt-labs/core "\ - "needs to label this PR with `ok to test` in order to run integration tests!" - check_for_duplicate_msg: true - - post-failure: - runs-on: ubuntu-latest - needs: test - if: ${{ failure() }} - - steps: - - name: Posting scheduled run failures - uses: ravsamhq/notify-slack-action@v2 - if: ${{ github.event_name == 'schedule' }} - with: - notification_title: 'Redshift nightly integration test failed' - status: ${{ job.status }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 389f55a88..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,213 +0,0 @@ -# **what?** -# Runs code quality checks, unit tests, and verifies python build on -# all code commited to the repository. This workflow should not -# require any secrets since it runs for PRs from forked repos. -# By default, secrets are not passed to workflows running from -# a forked repo. - -# **why?** -# Ensure code for dbt meets a certain quality standard. - -# **when?** -# This will run for all PRs, when code is pushed to a release -# branch, and when manually triggered. - -name: Tests and Code Checks - -on: - push: - branches: - - "main" - - "develop" - - "*.latest" - - "releases/*" - pull_request: - workflow_dispatch: - -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - code-quality: - name: code-quality - - runs-on: ubuntu-latest - - steps: - - name: Check out the repository - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install -r dev-requirements.txt - python -m pip --version - pre-commit --version - - - name: pre-commit hooks - run: pre-commit run --all-files --show-diff-on-failure - - unit: - name: unit test / python ${{ matrix.python-version }} - - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] - - env: - TOXENV: "unit" - PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv" - - steps: - - name: Check out the repository - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Run tox - run: tox - - - name: Get current date - if: always() - id: date - #no colons allowed for artifacts - run: | - echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv - path: unit_results.csv - - build: - name: build packages - - runs-on: ubuntu-latest - - outputs: - is_alpha: ${{ steps.check-is-alpha.outputs.is_alpha }} - - steps: - - name: Check out the repository - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade setuptools wheel twine check-wheel-contents - python -m pip --version - - - name: Build distributions - run: ./scripts/build-dist.sh - - - name: Show distributions - run: ls -lh dist/ - - - name: Check distribution descriptions - run: | - twine check dist/* - - - name: Check wheel contents - run: | - check-wheel-contents dist/*.whl --ignore W007,W008 - - - name: Check if this is an alpha version - id: check-is-alpha - run: | - export is_alpha=0 - if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi - echo "is_alpha=$is_alpha" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v3 - with: - name: dist - path: dist/ - - test-build: - name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} - - if: needs.build.outputs.is_alpha == 0 - - needs: build - - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] - - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade wheel - python -m pip --version - - - uses: actions/download-artifact@v3 - with: - name: dist - path: dist/ - - - name: Show distributions - run: ls -lh dist/ - - - name: Install wheel distributions - run: | - find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check wheel distributions - run: | - python -c "import dbt.adapters.redshift" - - - name: Install source distributions - run: | - find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check source distributions - run: | - python -c "import dbt.adapters.redshift" diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml deleted file mode 100644 index 46db5b749..000000000 --- a/.github/workflows/nightly-release.yml +++ /dev/null @@ -1,116 +0,0 @@ -# **what?** -# Nightly releases to GitHub and PyPI. This workflow produces the following outcome: -# - generate and validate data for night release (commit SHA, version number, release branch); -# - pass data to release workflow; -# - night release will be pushed to GitHub as a draft release; -# - night build will be pushed to test PyPI; -# -# **why?** -# Ensure an automated and tested release process for nightly builds -# -# **when?** -# This workflow runs on schedule or can be run manually on demand. - -name: Nightly Test Release to GitHub and PyPI - -on: - workflow_dispatch: # for manual triggering - schedule: - - cron: 0 9 * * * - -permissions: - contents: write # this is the permission that allows creating a new release - -defaults: - run: - shell: bash - -env: - RELEASE_BRANCH: "1.5.latest" # must test against most recent .latest branch to have parity for dependency with core - -jobs: - aggregate-release-data: - runs-on: ubuntu-latest - - outputs: - commit_sha: ${{ steps.resolve-commit-sha.outputs.release_commit }} - version_number: ${{ steps.nightly-release-version.outputs.number }} - release_branch: ${{ steps.release-branch.outputs.name }} - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ env.RELEASE_BRANCH }}" - uses: actions/checkout@v3 - with: - ref: ${{ env.RELEASE_BRANCH }} - - - name: "Resolve Commit To Release" - id: resolve-commit-sha - run: | - commit_sha=$(git rev-parse HEAD) - echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT - - - name: "Get Current Version Number" - id: version-number-sources - run: | - current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '` - echo "current_version=$current_version" >> $GITHUB_OUTPUT - - - name: "Audit Version And Parse Into Parts" - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ steps.version-number-sources.outputs.current_version }} - - - name: "Get Current Date" - id: current-date - run: echo "date=$(date +'%m%d%Y')" >> $GITHUB_OUTPUT - - # Bump to the next patch because when this is a previously released patch, the changelog - # markdown will already exist and cause a failure in another step - - name: "Bump Patch Number" - id: bump_patch - run: | - echo "patch=$((${{ steps.semver.outputs.patch }}+1))" >> $GITHUB_OUTPUT - - - name: "Generate Nightly Release Version Number" - id: nightly-release-version - run: | - number="${{ steps.semver.outputs.major }}.${{ steps.semver.outputs.minor }}.${{ steps.bump_patch.outputs.patch }}.dev${{ steps.current-date.outputs.date }}" - echo "number=$number" >> $GITHUB_OUTPUT - - - name: "Audit Nightly Release Version And Parse Into Parts" - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ steps.nightly-release-version.outputs.number }} - - - name: "Set Release Branch" - id: release-branch - run: | - echo "name=${{ env.RELEASE_BRANCH }}" >> $GITHUB_OUTPUT - - log-outputs-aggregate-release-data: - runs-on: ubuntu-latest - needs: [aggregate-release-data] - - steps: - - name: "[DEBUG] Log Outputs" - run: | - echo commit_sha : ${{ needs.aggregate-release-data.outputs.commit_sha }} - echo version_number: ${{ needs.aggregate-release-data.outputs.version_number }} - echo release_branch: ${{ needs.aggregate-release-data.outputs.release_branch }} - - release-github-pypi: - needs: [aggregate-release-data] - - uses: ./.github/workflows/release.yml - with: - sha: ${{ needs.aggregate-release-data.outputs.commit_sha }} - target_branch: ${{ needs.aggregate-release-data.outputs.release_branch }} - version_number: ${{ needs.aggregate-release-data.outputs.version_number }} - build_script_path: "scripts/build-dist.sh" - env_setup_script_path: "scripts/env-setup.sh" - s3_bucket_name: "core-team-artifacts" - package_test_command: "dbt -h" - test_run: true - nightly_release: true - secrets: inherit diff --git a/.github/workflows/publish-pypi-internal.yml b/.github/workflows/publish-pypi-internal.yml new file mode 100644 index 000000000..8bcad76eb --- /dev/null +++ b/.github/workflows/publish-pypi-internal.yml @@ -0,0 +1,36 @@ +# What? +# Tag and release an arbitrary ref. Uploads to an internal archive. +# +# How? +# After checking out and testing the provided ref, the image is built and uploaded. +# +# When? +# - manually +name: "Publish to internal PyPI" + +on: + workflow_dispatch: + inputs: + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + ref: + description: "The ref (sha or branch name) to use" + type: string + required: true + test-command: + description: "Package test command" + type: string + default: "python -c \"import dbt.adapters.redshift\"" + +jobs: + build-and-publish: + name: "Build and release to internal PyPI" + uses: dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main + with: + version_number: "${{ inputs.version }}" + package_test_command: "${{ inputs.test-command }}" + dbms_name: "redshift" + ref: ${{ inputs.ref }} + secrets: inherit diff --git a/.github/workflows/release-branch-tests.yml b/.github/workflows/release-branch-tests.yml deleted file mode 100644 index 004c6fb29..000000000 --- a/.github/workflows/release-branch-tests.yml +++ /dev/null @@ -1,31 +0,0 @@ -# **what?** -# The purpose of this workflow is to trigger CI to run for each release -# branch on a regular cadence. If the CI workflow fails for a branch, it -# will post to dev-core-alerts to raise awareness. The - -# **why?** -# Ensures release branches are always shippable and not broken. -# Also, can catch any dependencies shifting beneath us that might -# introduce breaking changes (could also impact Cloud). - -# **when?** -# Once each morning. Manual trigger can also test on demand - -name: Release branch scheduled testing - -on: - # run this once per night to ensure no regressions from latest dbt-core changes - schedule: - - cron: '0 5 * * *' # 5 UTC - - workflow_dispatch: # for manual triggering - -# no special access is needed -permissions: read-all - -jobs: - run_tests: - uses: dbt-labs/actions/.github/workflows/release-branch-tests.yml@main - with: - workflows_to_run: '["main.yml", "integration.yml"]' - secrets: inherit diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml deleted file mode 100644 index 0ce0f9adf..000000000 --- a/.github/workflows/release-internal.yml +++ /dev/null @@ -1,49 +0,0 @@ -# What? -# -# Tag and release an arbitrary ref. Uploads to an internal archive for further processing. -# -# How? -# -# After checking out and testing the provided ref, the image is built and uploaded. -# -# When? -# -# Manual trigger. - -name: "Release internal patch" - -on: - workflow_dispatch: - inputs: - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - ref: - description: "The ref (sha or branch name) to use" - type: string - default: "main" - required: true - package_test_command: - description: "Package test command" - type: string - default: "python -c \"import dbt.adapters.redshift\"" - required: true - -defaults: - run: - shell: "bash" - -jobs: - invoke-reusable-workflow: - name: "Build and Release Internally" - - uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main" - - with: - version_number: "${{ inputs.version_number }}" - package_test_command: "${{ inputs.package_test_command }}" - dbms_name: "redshift" - ref: "${{ inputs.ref }}" - - secrets: "inherit" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f68eb3320..bdee63147 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,216 +13,220 @@ # # **when?** # This workflow can be run manually on demand or can be called by other workflows -name: Release to GitHub and PyPI +name: "Release pipeline" +run-name: "Release `${{ inputs.version }}` to `${{ inputs.deploy-environment }}`" on: - workflow_dispatch: - inputs: - sha: - description: "The last commit sha in the release" - type: string - required: true - target_branch: - description: "The branch to release from" - type: string - required: true - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - build_script_path: - description: "Build script path" - type: string - default: "scripts/build-dist.sh" - required: true - env_setup_script_path: - description: "Environment setup script path" - type: string - default: "scripts/env-setup.sh" - required: false - s3_bucket_name: - description: "AWS S3 bucket name" - type: string - default: "core-team-artifacts" - required: true - package_test_command: - description: "Package test command" - type: string - default: "python -c \"import dbt.adapters.redshift\"" - required: true - test_run: - description: "Test run (Publish release as draft)" - type: boolean - default: true - required: false - nightly_release: - description: "Nightly release to dev environment" - type: boolean - default: false - required: false - workflow_call: - inputs: - sha: - description: "The last commit sha in the release" - type: string - required: true - target_branch: - description: "The branch to release from" - type: string - required: true - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - build_script_path: - description: "Build script path" - type: string - default: "scripts/build-dist.sh" - required: true - env_setup_script_path: - description: "Environment setup script path" - type: string - default: "scripts/env-setup.sh" - required: false - s3_bucket_name: - description: "AWS S3 bucket name" - type: string - default: "core-team-artifacts" - required: true - package_test_command: - description: "Package test command" - type: string - default: "python -c \"import dbt.adapters.redshift\"" - required: true - test_run: - description: "Test run (Publish release as draft)" - type: boolean - default: true - required: false - nightly_release: - description: "Nightly release to dev environment" - type: boolean - default: false - required: false - + workflow_call: + inputs: + sha: + description: "The commit sha to release" + type: string + required: true + branch: + description: "The branch to release from" + type: string + default: "main" + version: + description: "The release version (e.g. 1.0.0b1)" + type: string + required: true + deploy-environment: + description: "Where to publish" + type: string + default: "prod" + publish-slack-override: + description: "Use to publish a Slack notification for non-prod deploy environments" + type: boolean + default: false + workflow_dispatch: + inputs: + sha: + description: "The commit sha to release" + type: string + required: true + branch: + description: "The branch to release from" + type: string + default: "main" + version: + description: "The release version (e.g. 1.0.0b1)" + type: string + required: true + deploy-environment: + description: "Where to publish" + type: environment + publish-slack-override: + description: "Use to publish a Slack notification for non-prod deploy environments" + type: boolean + default: false + +# this is the permission that allows creating a new release to both GitHub and PyPI permissions: - contents: write # this is the permission that allows creating a new release + contents: write + id-token: write -defaults: - run: - shell: bash +# deploying the same version of a package to the same environment should override any previous deployment with those attributes +concurrency: + group: "${{ github.workflow }}-${{ inputs.version }}-${{ inputs.deploy-environment }}" + cancel-in-progress: true jobs: - log-inputs: - name: Log Inputs - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The branch to release from: ${{ inputs.target_branch }} - echo The release version number: ${{ inputs.version_number }} - echo Build script path: ${{ inputs.build_script_path }} - echo Environment setup script path: ${{ inputs.env_setup_script_path }} - echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }} - echo Package test command: ${{ inputs.package_test_command }} - echo Test run: ${{ inputs.test_run }} - echo Nightly release: ${{ inputs.nightly_release }} - - bump-version-generate-changelog: - name: Bump package version, Generate changelog - - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main - - with: - sha: ${{ inputs.sha }} - version_number: ${{ inputs.version_number }} - target_branch: ${{ inputs.target_branch }} - env_setup_script_path: ${{ inputs.env_setup_script_path }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - - secrets: inherit - - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} - - needs: [bump-version-generate-changelog] - - runs-on: ubuntu-latest - - steps: - - name: Print variables - run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - - build-test-package: - name: Build, Test, Package - if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog] - - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - build_script_path: ${{ inputs.build_script_path }} - s3_bucket_name: ${{ inputs.s3_bucket_name }} - package_test_command: ${{ inputs.package_test_command }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - github-release: - name: GitHub Release - if: ${{ !failure() && !cancelled() }} - - needs: [bump-version-generate-changelog, build-test-package] - - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main - - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - test_run: ${{ inputs.test_run }} - - pypi-release: - name: PyPI Release - - needs: [github-release] - - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - - with: - version_number: ${{ inputs.version_number }} - test_run: ${{ inputs.test_run }} - - secrets: - PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - - slack-notification: - name: Slack Notification - if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} - - needs: - [ - bump-version-generate-changelog, - build-test-package, - github-release, - pypi-release, - ] - - uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main - with: - status: "failure" - - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} + release-inputs: + name: "Release inputs" + runs-on: ubuntu-latest + outputs: + archive-name: ${{ steps.computed-inputs.outputs.archive-name }} + steps: + - name: "Set: archive name" + id: archive + shell: bash + run: | + archive_name=${{ inputs.package }}-${{ inputs.version_number }}-${{ inputs.deploy-environment }} + echo "name=$archive_name" >> $GITHUB_OUTPUT + + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo branch : ${{ inputs.branch }} + echo package : ${{ inputs.package }} + echo version : ${{ inputs.version }} + echo deploy-environment : ${{ inputs.deploy-environment }} + echo archive-name : ${{ steps.archive.outputs.name }} + echo dbt-adapters-branch : ${{ inputs.dbt-adapters-branch }} + echo dbt-common-branch : ${{ inputs.dbt-common-branch }} + echo dbt-core-branch : ${{ inputs.dbt-core-branch }} + + release-branch: + name: "Create a release branch" + needs: release-inputs + uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-create.yml@update-workflows + with: + branch: ${{ inputs.branch }} + version: ${{ inputs.version }} + secrets: inherit + + version-bump: + name: "Bump the version" + runs-on: ubuntu-latest + needs: release-branch + steps: + - name: "Check out ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.branch }} + + - name: "Bump version to `${{ inputs.version }}`" + shell: bash + run: hatch version ${{ inputs.version }} + + - name: "Commit and push changes" + shell: bash + run: commit + + changelog: + name: "Generate a new changelog" + needs: release-branch + uses: dbt-labs/dbt-adapters/.github/workflows/build-changelog.yml@update-workflows + with: + branch: ${{ needs.release-branch.outputs.branch }} + version: ${{ inputs.version }} + secrets: inherit + + code-quality: + name: "Run code quality" + needs: + - release-branch + - version-bump + - changelog + uses: ./.github/workflows/code-quality.yml + with: + branch: ${{ needs.release-branch.outputs.branch }} + + unit-tests: + name: "Run unit tests" + needs: + - release-branch + - version-bump + - changelog + uses: ./.github/workflows/unit-tests.yml + with: + branch: ${{ needs.release-branch.outputs.branch }} + + integration-tests: + name: "Run unit tests" + needs: + - release-branch + - version-bump + - changelog + uses: ./.github/workflows/integration-tests.yml + with: + branch: ${{ needs.release-branch.outputs.branch }} + + build-artifacts: + name: "Build artifacts" + if: ${{ !failure() && !cancelled() }} + needs: + - release-inputs + - release-branch + - code-quality + - unit-tests + - integration-tests + uses: ./.github/workflows/build-artifacts.yml + with: + branch: ${{ needs.release-branch.outputs.branch }} + archive-name: ${{ needs.release-inputs.outputs.archive-name }} + + merge-changes: + name: "Merge the version bump and changelog updates" + needs: + - build-artifacts + - release-branch + if: >- + !failure() && !cancelled() && + inputs.deploy-environment == 'prod' && + needs.release-branch.outputs.name + uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-merge.yml@update-workflows + with: + branch: ${{ inputs.branch }} + release-branch: ${{ needs.release-branch.outputs.branch }} + secrets: inherit + + publish-github: + name: "Publish to GitHub" + if: ${{ !failure() && !cancelled() }} + needs: + - release-inputs + - changelog + - merge-changes + uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows + with: + archive-name: ${{ needs.release-inputs.outputs.archive-name }} + version: ${{ inputs.version }} + deploy-environment: ${{ inputs.deploy-environment }} + sha: ${{ needs.merge-changes.outputs.sha }} + changelog-path: ${{ needs.changelog.outputs.changelog-path }} + + publish-pypi: + name: "Publish to PyPI" + if: ${{ !failure() && !cancelled() }} + needs: + - release-inputs + - merge-changes + uses: dbt-labs/dbt-adapters/.github/workflows/publish-pypi.yml@update-workflows + with: + archive-name: ${{ needs.release-inputs.outputs.archive-name }} + version: ${{ inputs.version }} + deploy-environment: ${{ inputs.deploy-environment }} + + slack-notification: + name: "Slack notification" + if: ${{ failure() && (inputs.deploy-environment == 'prod' || inputs.publish-slack-override) }} + needs: + - github-release + - pypi-release + uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main + with: + status: "failure" + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.github/workflows/repository-cleanup.yml b/.github/workflows/repository-cleanup.yml deleted file mode 100644 index c1d780281..000000000 --- a/.github/workflows/repository-cleanup.yml +++ /dev/null @@ -1,30 +0,0 @@ -# **what?** -# Cleanup branches left over from automation and testing. Also cleanup -# draft releases from release testing. - -# **why?** -# The automations are leaving behind branches and releases that clutter -# the repository. Sometimes we need them to debug processes so we don't -# want them immediately deleted. Running on Saturday to avoid running -# at the same time as an actual release to prevent breaking a release -# mid-release. - -# **when?** -# Mainly on a schedule of 12:00 Saturday. -# Manual trigger can also run on demand - -name: Repository Cleanup - -on: - schedule: - - cron: '0 12 * * SAT' # At 12:00 on Saturday - details in `why` above - - workflow_dispatch: # for manual triggering - -permissions: - contents: write - -jobs: - cleanup-repo: - uses: dbt-labs/actions/.github/workflows/repository-cleanup.yml@main - secrets: inherit diff --git a/.github/workflows/scheduled-test-releases.yml b/.github/workflows/scheduled-test-releases.yml new file mode 100644 index 000000000..1537d7b42 --- /dev/null +++ b/.github/workflows/scheduled-test-releases.yml @@ -0,0 +1,72 @@ +# **what?** +# Nightly releases to GitHub and PyPI. This workflow produces the following outcome: +# - generate and validate data for night release (commit SHA, version number, release branch); +# - pass data to release workflow; +# - night release will be pushed to GitHub as a draft release; +# - night build will be pushed to test PyPI; +# +# **why?** +# Ensure an automated and tested release process for nightly builds +# +# **when?** +# - scheduled with the repository variable DBT_SCHEDULED_TEST_RELEASES_SCHEDULE +# - manually +name: "Scheduled test releases" + +on: + workflow_dispatch: # for manual triggering + schedule: + - cron: ${{ vars.DBT_SCHEDULED_TEST_RELEASES_SCHEDULE }} + +permissions: + contents: write # this is the permission that allows creating a new release + +defaults: + run: + shell: bash + +jobs: + release-inputs: + name: "Get test release inputs" + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.commit.outputs.sha }} + version: ${{ steps.release-version.outputs.version }} + + steps: + - name: "Checkout ${{ github.repository }}" + uses: actions/checkout@v3 + + - name: "Set: commit" + id: commit + run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: "Set: version" + id: current-version + run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT + + - name: "Audit Version And Parse Into Parts" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ steps.current-version.outputs.version }} + + - name: "Set: release version" + id: release-version + run: | + version="${{ steps.semver.outputs.major }}" + version+=".${{ steps.semver.outputs.minor }}" + version+=".$(${{ steps.semver.outputs.patch }}+1)" + version+="dev$(date +'%m%d%Y')" + echo "version=$version" >> $GITHUB_OUTPUT + + release: + name: "Release test release" + needs: release-inputs + uses: ./.github/workflows/release.yml + with: + sha: ${{ needs.release-inputs.outputs.sha }} + version: ${{ needs.release-inputs.outputs.version }} + branch: "main" + deploy-environment: "test" + publish-slack-override: true diff --git a/.github/workflows/scheduled-tests.yml b/.github/workflows/scheduled-tests.yml new file mode 100644 index 000000000..cd5099492 --- /dev/null +++ b/.github/workflows/scheduled-tests.yml @@ -0,0 +1,72 @@ +# **what?** +# The purpose of this workflow is to trigger CI to run for each release +# branch on a regular cadence. If the CI workflow fails for a branch, it +# will post to dev-core-alerts to raise awareness. +# +# **why?** +# Ensures release branches are always shippable and not broken. +# Also, can catch any dependencies shifting beneath us that might +# introduce breaking changes (could also impact Cloud). + +# **when?** +# - scheduled with the repository variable DBT_SCHEDULED_TESTS_SCHEDULE +# - manually +name: "Scheduled tests" + +on: + schedule: + - cron: ${{ vars.DBT_SCHEDULED_TESTS_SCHEDULE }} + workflow_dispatch: + +permissions: read-all + +jobs: + code-quality: + name: "Code quality" + uses: dbt-labs/dbt-redshift/.github/workflows/code-quality.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + integration-tests: + name: "Integration tests" + uses: dbt-labs/dbt-redshift/.github/workflows/integration-tests.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + unit-tests: + name: "Unit tests" + uses: dbt-labs/dbt-redshift/.github/workflows/unit-tests.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + verify-build-artifacts: + name: "Unit tests" + uses: dbt-labs/dbt-redshift/.github/workflows/build-artifacts.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + post-failure: + runs-on: ubuntu-latest + needs: test + if: ${{ failure() }} + steps: + - name: Posting scheduled run failures + uses: ravsamhq/notify-slack-action@v2 + if: ${{ github.event_name == 'schedule' }} + with: + notification_title: 'Redshift nightly integration test failed' + status: ${{ job.status }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml new file mode 100644 index 000000000..e2ecd89dd --- /dev/null +++ b/.github/workflows/stale-issues.yml @@ -0,0 +1,13 @@ +name: "Close stale issues and PRs" + +on: + schedule: + - cron: "30 1 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index d902340a9..000000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: "Close stale issues and PRs" -on: - schedule: - - cron: "30 1 * * *" - -permissions: - issues: write - pull-requests: write - -jobs: - stale: - uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/triage-labels.yml b/.github/workflows/triage-labels.yml index 91f529e3e..8cf60b141 100644 --- a/.github/workflows/triage-labels.yml +++ b/.github/workflows/triage-labels.yml @@ -3,29 +3,25 @@ # those cases we remove the `triage` label and add the `awaiting_response` label. Once we # recieve a response in the form of a comment, we want the `awaiting_response` label removed # in favor of the `triage` label so we are aware that the issue needs action. - +# # **why?** # To help with out team triage issue tracking - +# # **when?** # This will run when a comment is added to an issue and that issue has to `awaiting_response` label. - -name: Update Triage Label +name: "Update triage Label" on: issue_comment -defaults: - run: - shell: bash - permissions: - issues: write + issues: write jobs: - triage_label: - if: contains(github.event.issue.labels.*.name, 'awaiting_response') - uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main - with: - add_label: "triage" - remove_label: "awaiting_response" - secrets: inherit # this is only acceptable because we own the action we're calling + triage-label: + if: contains(github.event.issue.labels.*.name, 'awaiting_response') + uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main + with: + add_label: "triage" + remove_label: "awaiting_response" + # this is only acceptable because we own the action we're calling + secrets: inherit diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 000000000..6da8914b6 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,102 @@ +name: "Unit tests" + +on: + pull_request_target: + push: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + cancel-in-progress: true + +jobs: + unit-tests: + name: "Unit tests" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJSON(vars.DBT_TEST_PYTHON_ALL_VERSIONS) }} + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + persist-credentials: false + + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + - name: "Update development branches" + if: ${{ contains(github.event_name, 'workflow_') }} + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + dbt-core-branch: ${{ inputs.dbt-core-branch }} + + - name: "Run unit tests" + shell: bash + run: hatch run unit-tests:all + + aggregate-results: + name: "Successful unit tests" + needs: unit-tests + if: ${{ !failure() && !cancelled() }} + runs-on: ubuntu-latest + steps: + - name: "Unit tests completed successfully" + shell: bash + run: | + title="Unit tests" + message="Unit tests completed successfully!" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml deleted file mode 100644 index bde34d683..000000000 --- a/.github/workflows/version-bump.yml +++ /dev/null @@ -1,28 +0,0 @@ -# **what?** -# This workflow will take the new version number to bump to. With that -# it will run versionbump to update the version number everywhere in the -# code base and then run changie to create the corresponding changelog. -# A PR will be created with the changes that can be reviewed before committing. - -# **why?** -# This is to aid in releasing dbt and making sure we have updated -# the version in all places and generated the changelog. - -# **when?** -# This is triggered manually - -name: Version Bump - -on: - workflow_dispatch: - inputs: - version_number: - description: 'The version number to bump to (ex. 1.2.0, 1.3.0b1)' - required: true - -jobs: - version_bump_and_changie: - uses: dbt-labs/actions/.github/workflows/version-bump.yml@main - with: - version_number: ${{ inputs.version_number }} - secrets: inherit # ok since what we are calling is internally maintained diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index cfbc714ed..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -recursive-include dbt/include *.sql *.yml *.md diff --git a/Makefile b/Makefile deleted file mode 100644 index efd23b806..000000000 --- a/Makefile +++ /dev/null @@ -1,67 +0,0 @@ -.DEFAULT_GOAL:=help - -.PHONY: dev -dev: ## Installs adapter in develop mode along with development dependencies - @\ - pip install -e . -r dev-requirements.txt && pre-commit install - -.PHONY: dev-uninstall -dev-uninstall: ## Uninstalls all packages while maintaining the virtual environment - ## Useful when updating versions, or if you accidentally installed into the system interpreter - pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y - pip uninstall -y dbt-redshift - -.PHONY: mypy -mypy: ## Runs mypy against staged changes for static type checking. - @\ - pre-commit run --hook-stage manual mypy-check | grep -v "INFO" - -.PHONY: flake8 -flake8: ## Runs flake8 against staged changes to enforce style guide. - @\ - pre-commit run --hook-stage manual flake8-check | grep -v "INFO" - -.PHONY: black -black: ## Runs black against staged changes to enforce style guide. - @\ - pre-commit run --hook-stage manual black-check -v | grep -v "INFO" - -.PHONY: lint -lint: ## Runs flake8 and mypy code checks against staged changes. - @\ - pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run mypy-check --hook-stage manual | grep -v "INFO" - -.PHONY: linecheck -linecheck: ## Checks for all Python lines 100 characters or more - @\ - find dbt -type f -name "*.py" -exec grep -I -r -n '.\{100\}' {} \; - -.PHONY: unit -unit: ## Runs unit tests with py38. - @\ - tox -e py38 - -.PHONY: test -test: ## Runs unit tests with py38 and code checks against staged changes. - @\ - tox -p -e py38; \ - pre-commit run black-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run mypy-check --hook-stage manual | grep -v "INFO" - -.PHONY: integration -integration: ## Runs redshift integration tests with py38. - @\ - tox -e py38-redshift -- - -.PHONY: clean - @echo "cleaning repo" - @git clean -f -X - -.PHONY: help -help: ## Show this help message. - @echo 'usage: make [target]' - @echo - @echo 'targets:' - @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index d02863ae0..000000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,26 +0,0 @@ -# install latest changes in dbt-core + dbt-postgres -git+https://github.com/dbt-labs/dbt-adapters.git -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git -git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core -git+https://github.com/dbt-labs/dbt-postgres.git - -# dev -ipdb~=0.13.13 -pre-commit==3.7.0;python_version >="3.9" -pre-commit==3.5.0;python_version <"3.9" - -# test -ddtrace==2.3.0 -freezegun~=1.3 -pytest~=7.4 -pytest-csv~=3.0 -pytest-dotenv~=0.5.2 -pytest-logbook~=1.2 -pytest-xdist~=3.5 -tox~=4.11 - -# build -bumpversion~=0.6.0 -twine~=4.0 -wheel~=0.42 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..7a6fcca54 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,137 @@ +[project] +dynamic = ["version"] +name = "dbt-redshift" +description = "The Redshift adapter plugin for dbt" +readme = "README.md" +keywords = ["dbt", "redshift", "dbt-redshift", "adapter", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs"] +requires-python = ">=3.8.0" +authors = [ + { name = "dbt Labs", email = "info@dbtlabs.com" }, +] +maintainers = [ + { name = "dbt Labs", email = "info@dbtlabs.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +dependencies = [ + "dbt-adapters>=0.1.0a1,<2.0", + "dbt-common>=0.1.0a1,<2.0", + # `dbt-redshift` depends deeply on `redshift-connector` + # This package has experienced breaking changes in previous patch releases + # Pin to the patch version and bump version cautiously + "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", + "dbt-postgres>=1.8.0a1,<1.9", + "sqlparse>=0.5.0,<0.6.0", + "agate>=1.0,<2.0", + # Add `dbt-core` to ensure backwards compatibility of installation + # This is not a functional dependency + "dbt-core>=1.8.0b3", +] + +[project.urls] +Homepage = "https://github.com/dbt-labs/dbt-redshift" +Documentation = "https://docs.getdbt.com" +Repository = "https://github.com/dbt-labs/dbt-redshift.git" +Issues = "https://github.com/dbt-labs/dbt-redshift/issues" +Changelog = "https://github.com/dbt-labs/dbt-redshift/blob/main/CHANGELOG.md" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.sdist] +include = ["dbt"] + +[tool.hatch.build.targets.wheel] +packages = ["dbt"] + +[tool.hatch.version] +path = "dbt/adapters/redshift/__version__.py" + +[tool.hatch.envs.default] +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-postgres @ git+https://github.com/dbt-labs/dbt-postgres.git", + "ipdb==0.13.13", + 'pre-commit==3.7.0;python_version>="3.9"', + 'pre-commit==3.5.0;python_version=="3.8"', +] +[tool.hatch.envs.default.scripts] +dev = "pre-commit install" +code-quality = "pre-commit run {args:--all-files}" +clean = "git clean -f -X" + +[tool.hatch.envs.unit-tests] +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "dbt-postgres @ git+https://github.com/dbt-labs/dbt-postgres.git", + "ddtrace==2.3.0", + "freezegun", + "pytest", + "pytest-csv", + "pytest-dotenv", + "pytest-logbook", + "pytest-mock", + "pytest-xdist", +] +[tool.hatch.envs.unit-tests.scripts] +all = "python -m pytest {args:tests/unit}" + +[tool.hatch.envs.integration-tests] +template = "unit-tests" +extra-dependencies = [ + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", +] +[tool.hatch.envs.integration-tests.scripts] +all = [ + 'python -m pytest --dist=loadscope tests/functional -k "not tests/functional/adapter/utils and not tests/functional/adapter/incremental"', + 'python -m pytest --dist=loadscope tests/functional/adapter/incremental', + 'python -m pytest --dist=loadscope tests/functional/adapter/utils', +] +[tool.hatch.envs.build] +detached = true +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] +[tool.hatch.envs.build.scripts] +check-all = [ + "- check-wheel", + "- check-sdist", +] +check-wheel = [ + "twine check dist/*", + "find ./dist/dbt*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-redshift", +] +check-sdist = [ + "check-wheel-contents dist/*.whl --ignore W007,W008", + "find ./dist/dbt*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-redshift", +] + +[tool.pytest] +color = true +verbose = true +env_files = ["test.env"] +testpaths = [ + "tests/functional", + "tests/unit", +] +filterwarnings = [ + "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", + "ignore:unclosed file .*:ResourceWarning", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b3d74bc14..000000000 --- a/pytest.ini +++ /dev/null @@ -1,9 +0,0 @@ -[pytest] -filterwarnings = - ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning - ignore:unclosed file .*:ResourceWarning -env_files = - test.env -testpaths = - tests/unit - tests/functional diff --git a/setup.py b/setup.py deleted file mode 100644 index dbb3913b9..000000000 --- a/setup.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -import re -import sys - -if sys.version_info < (3, 8): - print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.8 or higher.") - sys.exit(1) - - -try: - from setuptools import find_namespace_packages -except ImportError: - print("Error: dbt requires setuptools v40.1.0 or higher.") - print('Please upgrade setuptools with "pip install --upgrade setuptools" and try again') - sys.exit(1) - - -from pathlib import Path -from setuptools import setup - - -# pull the long description from the README -README = Path(__file__).parent / "README.md" - - -# used for this adapter's version and in determining the compatible dbt-core version -VERSION = Path(__file__).parent / "dbt/adapters/redshift/__version__.py" - - -def _plugin_version() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - return attributes["version"] - - -def _plugin_version_trim() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - pattern = r"\+build\d+$" - return re.sub(pattern, "", attributes["version"]) - - -setup( - name="dbt-redshift", - version=_plugin_version(), - description="The Redshift adapter plugin for dbt", - long_description=README.read_text(), - long_description_content_type="text/markdown", - author="dbt Labs", - author_email="info@dbtlabs.com", - url="https://github.com/dbt-labs/dbt-redshift", - packages=find_namespace_packages(include=["dbt", "dbt.*"]), - include_package_data=True, - install_requires=[ - "dbt-common>=0.1.0a1,<2.0", - "dbt-adapters>=0.1.0a1,<2.0", - f"dbt-postgres~={_plugin_version_trim()}", - # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases - # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. - "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", - # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency - "dbt-core>=1.8.0b3", - # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core - "sqlparse>=0.5.0,<0.6.0", - "agate", - ], - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: Apache Software License", - "Operating System :: Microsoft :: Windows", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - ], - python_requires=">=3.8", -) diff --git a/test.env.example b/test.env.example index 4de05edab..44b34f5b1 100644 --- a/test.env.example +++ b/test.env.example @@ -1,18 +1,9 @@ -# Note: Make sure you have a Redshift account that is set up so these fields are easy to complete. - -### Test Environment field definitions -# These will all be gathered from account information or created by you. -# Endpoint for Redshift connection REDSHIFT_TEST_HOST= -# Username on your account REDSHIFT_TEST_USER= -# Password for Redshift account REDSHIFT_TEST_PASS= -# Local port to connect on REDSHIFT_TEST_PORT= -# Name of Redshift database in your account to test against REDSHIFT_TEST_DBNAME= -# Users for testing + DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index c490fed9a..000000000 --- a/tox.ini +++ /dev/null @@ -1,34 +0,0 @@ -[tox] -skipsdist = True -envlist = py38,py39,py310,py311 - -[testenv:{unit,py38,py39,py310,py311,py}] -description = unit testing -skip_install = true -passenv = - DBT_* - PYTEST_ADDOPTS -commands = {envpython} -m pytest {posargs} tests/unit -deps = - -rdev-requirements.txt - -e. - -[testenv:{integration,py38,py39,py310,py311,py}-{redshift}] -description = adapter plugin integration testing -skip_install = true -passenv = - DBT_* - REDSHIFT_TEST_* - PYTEST_ADDOPTS - DD_CIVISIBILITY_AGENTLESS_ENABLED - DD_API_KEY - DD_SITE - DD_ENV - DD_SERVICE -commands = - {envpython} -m pytest --dist=loadscope {posargs} tests/functional -k "not tests/functional/adapter/utils and not tests/functional/adapter/incremental" - {envpython} -m pytest --dist=loadscope {posargs} tests/functional/adapter/utils - {envpython} -m pytest --dist=loadscope {posargs} tests/functional/adapter/incremental -deps = - -rdev-requirements.txt - -e. From 2ab9ff513a6621afd068b7f61b2fde068f18d013 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 20 Apr 2024 11:20:04 -0400 Subject: [PATCH 02/15] merge internal pypi release into full release pipeline --- .github/workflows/publish-pypi-internal.yml | 36 --------------------- .github/workflows/release.yml | 14 ++++++++ 2 files changed, 14 insertions(+), 36 deletions(-) delete mode 100644 .github/workflows/publish-pypi-internal.yml diff --git a/.github/workflows/publish-pypi-internal.yml b/.github/workflows/publish-pypi-internal.yml deleted file mode 100644 index 8bcad76eb..000000000 --- a/.github/workflows/publish-pypi-internal.yml +++ /dev/null @@ -1,36 +0,0 @@ -# What? -# Tag and release an arbitrary ref. Uploads to an internal archive. -# -# How? -# After checking out and testing the provided ref, the image is built and uploaded. -# -# When? -# - manually -name: "Publish to internal PyPI" - -on: - workflow_dispatch: - inputs: - version: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - ref: - description: "The ref (sha or branch name) to use" - type: string - required: true - test-command: - description: "Package test command" - type: string - default: "python -c \"import dbt.adapters.redshift\"" - -jobs: - build-and-publish: - name: "Build and release to internal PyPI" - uses: dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main - with: - version_number: "${{ inputs.version }}" - package_test_command: "${{ inputs.test-command }}" - dbms_name: "redshift" - ref: ${{ inputs.ref }} - secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bdee63147..6766c4c7a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -219,6 +219,20 @@ jobs: version: ${{ inputs.version }} deploy-environment: ${{ inputs.deploy-environment }} + publish-pypi-internal: + name: "Publish to internal PyPI" + if: ${{ !failure() && !cancelled() }} + needs: + - release-inputs + - merge-changes + uses: dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main + with: + version_number: "${{ inputs.version }}" + package_test_command: "python -c \"import dbt.adapters.redshift\"" + dbms_name: "redshift" + ref: ${{ needs.merge-changes.outputs.sha }} + secrets: inherit + slack-notification: name: "Slack notification" if: ${{ failure() && (inputs.deploy-environment == 'prod' || inputs.publish-slack-override) }} From c2da70c77cc8b89b9b5cb21abd766f43e65bc8a8 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 20 Apr 2024 11:07:35 -0400 Subject: [PATCH 03/15] draft commit --- .bumpversion.cfg | 35 -- .../actions/update-dev-branches/action.yml | 38 ++ .../update-dev-branches/update_dev_branch.sh | 15 + .github/scripts/integration-test-matrix.js | 87 ---- .github/scripts/update_dbt_core_branch.sh | 20 - .github/workflows/backport.yml | 3 +- .github/workflows/bot-changelog.yml | 1 + .github/workflows/build-artifacts.yml | 61 +++ .github/workflows/changelog-check.yml | 46 ++ .github/workflows/clean-repo.yml | 27 ++ .github/workflows/code-quality.yml | 86 ++++ .github/workflows/docs-issues.yml | 0 .github/workflows/integration-tests.yml | 145 ++++++ .github/workflows/integration.yml | 244 ---------- .github/workflows/main.yml | 213 --------- .github/workflows/nightly-release.yml | 116 ----- .github/workflows/publish-pypi-internal.yml | 36 ++ .github/workflows/release-branch-tests.yml | 31 -- .github/workflows/release-internal.yml | 49 -- .github/workflows/release.yml | 418 +++++++++--------- .github/workflows/scheduled-test-releases.yml | 72 +++ .github/workflows/scheduled-tests.yml | 72 +++ .github/workflows/stale-issues.yml | 13 + .github/workflows/triage-labels.yml | 0 .github/workflows/unit-tests.yml | 102 +++++ .github/workflows/version-bump.yml | 28 -- MANIFEST.in | 1 - Makefile | 67 --- dev-requirements.txt | 26 -- pyproject.toml | 137 ++++++ pytest.ini | 9 - setup.py | 87 ---- test.env.example | 11 +- tox.ini | 34 -- 34 files changed, 1065 insertions(+), 1265 deletions(-) delete mode 100644 .bumpversion.cfg create mode 100644 .github/actions/update-dev-branches/action.yml create mode 100755 .github/actions/update-dev-branches/update_dev_branch.sh delete mode 100644 .github/scripts/integration-test-matrix.js delete mode 100755 .github/scripts/update_dbt_core_branch.sh create mode 100644 .github/workflows/build-artifacts.yml create mode 100644 .github/workflows/changelog-check.yml create mode 100644 .github/workflows/clean-repo.yml create mode 100644 .github/workflows/code-quality.yml create mode 100644 .github/workflows/docs-issues.yml create mode 100644 .github/workflows/integration-tests.yml delete mode 100644 .github/workflows/integration.yml delete mode 100644 .github/workflows/main.yml delete mode 100644 .github/workflows/nightly-release.yml create mode 100644 .github/workflows/publish-pypi-internal.yml delete mode 100644 .github/workflows/release-branch-tests.yml delete mode 100644 .github/workflows/release-internal.yml create mode 100644 .github/workflows/scheduled-test-releases.yml create mode 100644 .github/workflows/scheduled-tests.yml create mode 100644 .github/workflows/stale-issues.yml create mode 100644 .github/workflows/triage-labels.yml create mode 100644 .github/workflows/unit-tests.yml delete mode 100644 .github/workflows/version-bump.yml delete mode 100644 MANIFEST.in delete mode 100644 Makefile delete mode 100644 dev-requirements.txt create mode 100644 pyproject.toml delete mode 100644 pytest.ini delete mode 100644 setup.py delete mode 100644 tox.ini diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 52cc0acca..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[bumpversion] -current_version = 1.8.0b3 -parse = (?P[\d]+) # major version number - \.(?P[\d]+) # minor version number - \.(?P[\d]+) # patch version number - (?P # optional pre-release - ex: a1, b2, rc25 - (?Pa|b|rc) # pre-release type - (?P[\d]+) # pre-release version number - )? - ( # optional nightly release indicator - \.(?Pdev[0-9]+) # ex: .dev02142023 - )? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0` -serialize = - {major}.{minor}.{patch}{prekind}{num}.{nightly} - {major}.{minor}.{patch}.{nightly} - {major}.{minor}.{patch}{prekind}{num} - {major}.{minor}.{patch} -commit = False -tag = False - -[bumpversion:part:prekind] -first_value = a -optional_value = final -values = - a - b - rc - final - -[bumpversion:part:num] -first_value = 1 - -[bumpversion:part:nightly] - -[bumpversion:file:dbt/adapters/redshift/__version__.py] diff --git a/.github/actions/update-dev-branches/action.yml b/.github/actions/update-dev-branches/action.yml new file mode 100644 index 000000000..a177e69d2 --- /dev/null +++ b/.github/actions/update-dev-branches/action.yml @@ -0,0 +1,38 @@ +name: "Update dev branches" +description: "Update dev branches for testing cross-repo changes" + +inputs: + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + default: "" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + default: "" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + default: "" + +runs: + using: composite + steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo dbt-adapters : ${{ inputs.dbt-adapters-branch }} + echo dbt-common : ${{ inputs.dbt-common-branch }} + echo dbt-core : ${{ inputs.dbt-core-branch }} + + - name: "Update `dbt-adapters` branch" + if: ${{ inputs.dbt-adapters-branch }} + shell: bash + run: ./.github/actions/update-dev-branches/update_dev_branch.sh "dbt-adapters" ${{ inputs.dbt-adapters-branch }} + + - name: "Update `dbt-common` branch" + if: ${{ inputs.dbt-common-branch }} + shell: bash + run: ./.github/actions/update-dev-branches/update_dev_branch.sh "dbt-common" ${{ inputs.dbt-common-branch }} + + - name: "Update `dbt-core` branch" + if: ${{ inputs.dbt-core-branch }} + shell: bash + run: ./.github/actions/update-dev-branches/update_dev_branch.sh "dbt-core" ${{ inputs.dbt-core-branch }} diff --git a/.github/actions/update-dev-branches/update_dev_branch.sh b/.github/actions/update-dev-branches/update_dev_branch.sh new file mode 100755 index 000000000..4d51f9211 --- /dev/null +++ b/.github/actions/update-dev-branches/update_dev_branch.sh @@ -0,0 +1,15 @@ +#!/bin/bash -e +set -e + +package=$1 +branch=$2 + +file="pyproject.toml" +sed_pattern="s|${package}.git@main*|${package}.git@${branch}|g" + +# mac ships with a different version of sed that requires a delimiter arg +if [[ "$OSTYPE" == darwin* ]]; then + sed -i "" "$sed_pattern" $file +else + sed -i "$sed_pattern" $file +fi diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js deleted file mode 100644 index 7db445d9e..000000000 --- a/.github/scripts/integration-test-matrix.js +++ /dev/null @@ -1,87 +0,0 @@ -module.exports = ({ context }) => { - const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11"]; - const supportedAdapters = ["redshift"]; - - // if PR, generate matrix based on files changed and PR labels - if (context.eventName.includes("pull_request")) { - // `changes` is a list of adapter names that have related - // file changes in the PR - // ex: ['postgres', 'snowflake'] - const labels = context.payload.pull_request.labels.map(({ name }) => name); - console.log("labels", labels); - const testAllLabel = labels.includes("test all"); - const include = []; - - for (const adapter of supportedAdapters) { - for (const pythonVersion of supportedPythonVersions) { - if ( - pythonVersion === defaultPythonVersion || - labels.includes(`test python${pythonVersion}`) || - testAllLabel - ) { - // always run tests on ubuntu by default - include.push({ - os: "ubuntu-latest", - adapter, - "python-version": pythonVersion, - }); - - if (labels.includes("test windows") || testAllLabel) { - include.push({ - os: "windows-latest", - adapter, - "python-version": pythonVersion, - }); - } - - if (labels.includes("test macos") || testAllLabel) { - include.push({ - os: "macos-latest", - adapter, - "python-version": pythonVersion, - }); - } - } - } - } - - console.log("matrix", { include }); - - return { - include, - }; - } - // if not PR, generate matrix of python version, adapter, and operating - // system to run integration tests on - - const include = []; - // run for all adapters and python versions on ubuntu - for (const adapter of supportedAdapters) { - for (const pythonVersion of supportedPythonVersions) { - include.push({ - os: 'ubuntu-latest', - adapter: adapter, - "python-version": pythonVersion, - }); - } - } - - // additionally include runs for all adapters, on macos and windows, - // but only for the default python version - for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-latest"]) { - include.push({ - os: operatingSystem, - adapter: adapter, - "python-version": defaultPythonVersion, - }); - } - } - - console.log("matrix", { include }); - - return { - include, - }; -}; diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh deleted file mode 100755 index d28a40c35..000000000 --- a/.github/scripts/update_dbt_core_branch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$postgres_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$postgres_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi -core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) -bumpversion --allow-dirty --new-version "$core_version" major diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index d91fd5e6f..454ab12c9 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -19,7 +19,8 @@ on: - labeled permissions: - pull-requests: write # to create the PR + + pull-requests: write# to create the PR contents: write # to create the comment if the process fails jobs: diff --git a/.github/workflows/bot-changelog.yml b/.github/workflows/bot-changelog.yml index d7d80b2e8..92d857636 100644 --- a/.github/workflows/bot-changelog.yml +++ b/.github/workflows/bot-changelog.yml @@ -21,6 +21,7 @@ name: "Bot changelog" on: pull_request: + types: - opened - labeled diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 000000000..2907f7f1e --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,61 @@ +# **what?** +# Verifies python build on all code commited to the repository. This workflow +# should not require any secrets since it runs for PRs from forked repos. By +# default, secrets are not passed to workflows running from a forked repos. + +# **why?** +# Ensure code for dbt meets a certain quality standard. + +# **when?** +# This will run for all PRs, when code is pushed to main, and when manually triggered. +name: "Build release" + +on: + workflow_call: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + archive-name: + description: "The name to use for the upload archive, leave blank for no upload" + type: string + default: "" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + archive-name: + description: "The name to use for the upload archive, leave blank for no upload" + type: string + default: "" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.archive-name }}" + cancel-in-progress: true + +jobs: + build: + name: "Build a release" + runs-on: ubuntu-latest + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + - name: "Build ${{ github.event.repository.name }}" + uses: dbt-labs/dbt-adapters/.github/actions/build-artifacts@update-workflows + with: + archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/changelog-check.yml b/.github/workflows/changelog-check.yml new file mode 100644 index 000000000..dd4b11d76 --- /dev/null +++ b/.github/workflows/changelog-check.yml @@ -0,0 +1,46 @@ +# **what?** +# Checks that a file has been committed under the /.changes directory +# as a new CHANGELOG entry. Cannot check for a specific filename as +# it is dynamically generated by change type and timestamp. +# This workflow should not require any secrets since it runs for PRs +# from forked repos. +# By default, secrets are not passed to workflows running from +# a forked repo. +# +# **why?** +# Ensure code changes are reflected in the CHANGELOG. +# +# **when?** +# This will run for all PRs going into main and *.latest. It will run when: +# - the PR is opened or reopened +# - labels are updated on the PR +# - new code is pushed to the branch +# The action will get skipped if the 'Skip Changelog' label is present. +name: Check Changelog Entry + +on: + pull_request: + types: + - opened + - reopened + - labeled + - unlabeled + - synchronize + workflow_dispatch: + +defaults: + run: + shell: bash + +permissions: + contents: read + pull-requests: write + +jobs: + changelog: + uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main + with: + changelog_comment: "Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see the [dbt-redshift contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md)." + skip_label: "Skip Changelog" + # this is only acceptable because we own the action we're calling + secrets: inherit diff --git a/.github/workflows/clean-repo.yml b/.github/workflows/clean-repo.yml new file mode 100644 index 000000000..2ce2837b3 --- /dev/null +++ b/.github/workflows/clean-repo.yml @@ -0,0 +1,27 @@ +# **what?** +# Cleanup branches left over from automation and testing. +# Also cleanup draft releases from release testing. +# +# **why?** +# The automations are leaving behind branches and releases that clutter the repository. +# Sometimes we need them to debug processes so we don't want them immediately deleted. +# Running on Saturday to avoid running at the same time as an actual release +# to prevent breaking a release mid-release. +# +# **when?** +# - every Saturday at noon UTC +# - manually +name: "Clean repo" + +on: + schedule: + - cron: '0 12 * * SAT' # noon UTC on Saturday - details in `why` above + workflow_dispatch: + +permissions: + contents: write + +jobs: + cleanup-repo: + uses: dbt-labs/actions/.github/workflows/repository-cleanup.yml@main + secrets: inherit diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 000000000..800de1a6f --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,86 @@ +# **what?** +# Run code quality checks, e.g. `black`, `flake8`, `mypy`, via `pre-commit` +# +# **why?** +# Ensure code quality meets dbt Labs standards +# +# **when?** +# - runs on all PRs into protected branches +# - runs indirectly during releases and release tests +# - can be manually run +name: "Code quality" + +on: + pull_request: + push: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch/tag to run code quality on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run code quality on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + cancel-in-progress: true + +jobs: + code-quality: + name: "Code quality" + runs-on: ubuntu-latest + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + persist-credentials: false + + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + - name: "Update development branches" + if: ${{ contains(github.event_name, 'workflow_') }} + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + + - name: "Run code quality" + shell: bash + run: hatch run code-quality diff --git a/.github/workflows/docs-issues.yml b/.github/workflows/docs-issues.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..2873de21b --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,145 @@ +name: "Integration tests" + +on: + pull_request_target: + push: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + cancel-in-progress: true + +jobs: + require-approval-on-forks: + # if it's a pull request from a fork that has not been approved yet, don't run tests + if: >- + github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name != github.repository && + !contains(github.event.pull_request.labels.*.name, 'ok to test') + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: "Author requires permission to run tests" + uses: unsplash/comment-on-pr@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + msg: | + "Thanks for your contribution! "\ + "Since this is a fork, integration tests need to be approved prior to running. "\ + "@dbt-labs/core will review this PR and approve running integration tests." + check_for_duplicate_msg: true + + integration-tests: + name: "Integration tests" + needs: require-approval-on-forks + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ${{ fromJSON(vars.DBT_TEST_PYTHON_ALL_VERSIONS) }} + include: + - os: macos-latest + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + - os: windows-latest + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + env: + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_PORT: ${{ vars.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + DBT_TEST_USER_1: ${{ vars.REDSHIFT_TEST_USER_1 }} + DBT_TEST_USER_2: ${{ vars.REDSHIFT_TEST_USER_2 }} + DBT_TEST_USER_3: ${{ vars.REDSHIFT_TEST_USER_3 }} + DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }} + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: ${{ vars.DD_SITE }} + DD_ENV: ${{ vars.DD_ENV }} + DD_SERVICE: ${{ github.event.repository.name }} + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + persist-credentials: false + + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ matrix.python-version }} + + - name: "Update development branches" + if: ${{ contains(github.event_name, 'workflow_') }} + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + dbt-core-branch: ${{ inputs.dbt-core-branch }} + + - name: "Run integration tests" + shell: bash + run: hatch run integration-tests:all + + aggregate-results: + name: "Successful integration tests" + needs: integration-tests + if: ${{ !failure() && !cancelled() }} + runs-on: ubuntu-latest + steps: + - name: "Integration tests completed successfully" + shell: bash + run: | + title="Integration tests" + message="Integration tests completed successfully!" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index 9d1fe0807..000000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,244 +0,0 @@ -# **what?** -# This workflow runs all integration tests for supported OS -# and python versions and core adapters. If triggered by PR, -# the workflow will only run tests for adapters related -# to code changes. Use the `test all` and `test ${adapter}` -# label to run all or additional tests. Use `ok to test` -# label to mark PRs from forked repositories that are safe -# to run integration tests for. Requires secrets to run -# against different warehouses. - -# **why?** -# This checks the functionality of dbt from a user's perspective -# and attempts to catch functional regressions. - -# **when?** -# This workflow will run on every push to a protected branch -# and when manually triggered. It will also run for all PRs, including -# PRs from forks. The workflow will be skipped until there is a label -# to mark the PR as safe to run. - -name: Adapter Integration Tests - -on: - # pushes to release branches - push: - branches: - - "main" - - "develop" - - "*.latest" - - "releases/*" - # all PRs, important to note that `pull_request_target` workflows - # will run in the context of the target branch of a PR - pull_request_target: - # manual trigger - workflow_dispatch: - inputs: - dbt-core-branch: - description: "branch of dbt-core to use in dev-requirements.txt" - required: false - type: string - -# explicitly turn off permissions for `GITHUB_TOKEN` -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -# sets default shell to bash, for all operating systems -defaults: - run: - shell: bash - -jobs: - # generate test metadata about what files changed and the testing matrix to use - test-metadata: - # run if not a PR from a forked repository or has a label to mark as safe to test - if: >- - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest - - outputs: - matrix: ${{ steps.generate-matrix.outputs.result }} - - steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Check if relevant files changed - if: github.event_name == 'pull_request_target' - # https://github.com/marketplace/actions/paths-changes-filter - # For each filter, it sets output variable named by the filter to the text: - # 'true' - if any of changed files matches any of filter rules - # 'false' - if none of changed files matches any of filter rules - # also, returns: - # `changes` - JSON array with names of all filters matching any of the changed files - uses: dorny/paths-filter@v2 - id: get-changes - with: - token: ${{ secrets.GITHUB_TOKEN }} - filters: | - redshift: - - 'dbt/**' - - 'tests/**' - - 'dev-requirements.txt' - - - name: Generate integration test matrix - id: generate-matrix - uses: actions/github-script@v6 - env: - CHANGES: ${{ steps.get-changes.outputs.changes }} - with: - script: | - const script = require('./.github/scripts/integration-test-matrix.js') - const matrix = script({ context }) - console.log(matrix) - return matrix - test: - name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} - - # run if not a PR from a forked repository or has a label to mark as safe to test - # also checks that the matrix generated is not empty - if: >- - needs.test-metadata.outputs.matrix && - fromJSON( needs.test-metadata.outputs.matrix ).include[0] && - ( - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - ) - runs-on: ${{ matrix.os }} - - needs: test-metadata - - strategy: - fail-fast: false - max-parallel: 3 - matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} - - env: - TOXENV: integration-${{ matrix.adapter }} - PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" - DBT_INVOCATION_ENV: github-actions - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: datadoghq.com - DD_ENV: ci - DD_SERVICE: ${{ github.event.repository.name }} - - steps: - - name: Check out the repository - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - - # explicity checkout the branch for the PR, - # this is necessary for the `pull_request_target` event - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' - run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} - - - name: Run tox (redshift) - if: matrix.adapter == 'redshift' - env: - REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} - REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} - REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} - REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} - REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} - DBT_TEST_USER_1: dbt_test_user_1 - DBT_TEST_USER_2: dbt_test_user_2 - DBT_TEST_USER_3: dbt_test_user_3 - run: tox -- --ddtrace - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: logs - path: ./logs - - - name: Get current date - if: always() - id: date - run: | - echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv - path: integration_results.csv - - require-label-comment: - runs-on: ubuntu-latest - - needs: test - - permissions: - pull-requests: write - - steps: - - name: Needs permission PR comment - if: >- - needs.test.result == 'skipped' && - github.event_name == 'pull_request_target' && - github.event.pull_request.head.repo.full_name != github.repository - uses: unsplash/comment-on-pr@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - msg: | - "You do not have permissions to run integration tests, @dbt-labs/core "\ - "needs to label this PR with `ok to test` in order to run integration tests!" - check_for_duplicate_msg: true - - post-failure: - runs-on: ubuntu-latest - needs: test - if: ${{ failure() }} - - steps: - - name: Posting scheduled run failures - uses: ravsamhq/notify-slack-action@v2 - if: ${{ github.event_name == 'schedule' }} - with: - notification_title: 'Redshift nightly integration test failed' - status: ${{ job.status }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 389f55a88..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,213 +0,0 @@ -# **what?** -# Runs code quality checks, unit tests, and verifies python build on -# all code commited to the repository. This workflow should not -# require any secrets since it runs for PRs from forked repos. -# By default, secrets are not passed to workflows running from -# a forked repo. - -# **why?** -# Ensure code for dbt meets a certain quality standard. - -# **when?** -# This will run for all PRs, when code is pushed to a release -# branch, and when manually triggered. - -name: Tests and Code Checks - -on: - push: - branches: - - "main" - - "develop" - - "*.latest" - - "releases/*" - pull_request: - workflow_dispatch: - -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - code-quality: - name: code-quality - - runs-on: ubuntu-latest - - steps: - - name: Check out the repository - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install -r dev-requirements.txt - python -m pip --version - pre-commit --version - - - name: pre-commit hooks - run: pre-commit run --all-files --show-diff-on-failure - - unit: - name: unit test / python ${{ matrix.python-version }} - - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] - - env: - TOXENV: "unit" - PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv" - - steps: - - name: Check out the repository - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Run tox - run: tox - - - name: Get current date - if: always() - id: date - #no colons allowed for artifacts - run: | - echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv - path: unit_results.csv - - build: - name: build packages - - runs-on: ubuntu-latest - - outputs: - is_alpha: ${{ steps.check-is-alpha.outputs.is_alpha }} - - steps: - - name: Check out the repository - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade setuptools wheel twine check-wheel-contents - python -m pip --version - - - name: Build distributions - run: ./scripts/build-dist.sh - - - name: Show distributions - run: ls -lh dist/ - - - name: Check distribution descriptions - run: | - twine check dist/* - - - name: Check wheel contents - run: | - check-wheel-contents dist/*.whl --ignore W007,W008 - - - name: Check if this is an alpha version - id: check-is-alpha - run: | - export is_alpha=0 - if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi - echo "is_alpha=$is_alpha" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v3 - with: - name: dist - path: dist/ - - test-build: - name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} - - if: needs.build.outputs.is_alpha == 0 - - needs: build - - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] - - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade wheel - python -m pip --version - - - uses: actions/download-artifact@v3 - with: - name: dist - path: dist/ - - - name: Show distributions - run: ls -lh dist/ - - - name: Install wheel distributions - run: | - find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check wheel distributions - run: | - python -c "import dbt.adapters.redshift" - - - name: Install source distributions - run: | - find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check source distributions - run: | - python -c "import dbt.adapters.redshift" diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml deleted file mode 100644 index 46db5b749..000000000 --- a/.github/workflows/nightly-release.yml +++ /dev/null @@ -1,116 +0,0 @@ -# **what?** -# Nightly releases to GitHub and PyPI. This workflow produces the following outcome: -# - generate and validate data for night release (commit SHA, version number, release branch); -# - pass data to release workflow; -# - night release will be pushed to GitHub as a draft release; -# - night build will be pushed to test PyPI; -# -# **why?** -# Ensure an automated and tested release process for nightly builds -# -# **when?** -# This workflow runs on schedule or can be run manually on demand. - -name: Nightly Test Release to GitHub and PyPI - -on: - workflow_dispatch: # for manual triggering - schedule: - - cron: 0 9 * * * - -permissions: - contents: write # this is the permission that allows creating a new release - -defaults: - run: - shell: bash - -env: - RELEASE_BRANCH: "1.5.latest" # must test against most recent .latest branch to have parity for dependency with core - -jobs: - aggregate-release-data: - runs-on: ubuntu-latest - - outputs: - commit_sha: ${{ steps.resolve-commit-sha.outputs.release_commit }} - version_number: ${{ steps.nightly-release-version.outputs.number }} - release_branch: ${{ steps.release-branch.outputs.name }} - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ env.RELEASE_BRANCH }}" - uses: actions/checkout@v3 - with: - ref: ${{ env.RELEASE_BRANCH }} - - - name: "Resolve Commit To Release" - id: resolve-commit-sha - run: | - commit_sha=$(git rev-parse HEAD) - echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT - - - name: "Get Current Version Number" - id: version-number-sources - run: | - current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '` - echo "current_version=$current_version" >> $GITHUB_OUTPUT - - - name: "Audit Version And Parse Into Parts" - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ steps.version-number-sources.outputs.current_version }} - - - name: "Get Current Date" - id: current-date - run: echo "date=$(date +'%m%d%Y')" >> $GITHUB_OUTPUT - - # Bump to the next patch because when this is a previously released patch, the changelog - # markdown will already exist and cause a failure in another step - - name: "Bump Patch Number" - id: bump_patch - run: | - echo "patch=$((${{ steps.semver.outputs.patch }}+1))" >> $GITHUB_OUTPUT - - - name: "Generate Nightly Release Version Number" - id: nightly-release-version - run: | - number="${{ steps.semver.outputs.major }}.${{ steps.semver.outputs.minor }}.${{ steps.bump_patch.outputs.patch }}.dev${{ steps.current-date.outputs.date }}" - echo "number=$number" >> $GITHUB_OUTPUT - - - name: "Audit Nightly Release Version And Parse Into Parts" - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ steps.nightly-release-version.outputs.number }} - - - name: "Set Release Branch" - id: release-branch - run: | - echo "name=${{ env.RELEASE_BRANCH }}" >> $GITHUB_OUTPUT - - log-outputs-aggregate-release-data: - runs-on: ubuntu-latest - needs: [aggregate-release-data] - - steps: - - name: "[DEBUG] Log Outputs" - run: | - echo commit_sha : ${{ needs.aggregate-release-data.outputs.commit_sha }} - echo version_number: ${{ needs.aggregate-release-data.outputs.version_number }} - echo release_branch: ${{ needs.aggregate-release-data.outputs.release_branch }} - - release-github-pypi: - needs: [aggregate-release-data] - - uses: ./.github/workflows/release.yml - with: - sha: ${{ needs.aggregate-release-data.outputs.commit_sha }} - target_branch: ${{ needs.aggregate-release-data.outputs.release_branch }} - version_number: ${{ needs.aggregate-release-data.outputs.version_number }} - build_script_path: "scripts/build-dist.sh" - env_setup_script_path: "scripts/env-setup.sh" - s3_bucket_name: "core-team-artifacts" - package_test_command: "dbt -h" - test_run: true - nightly_release: true - secrets: inherit diff --git a/.github/workflows/publish-pypi-internal.yml b/.github/workflows/publish-pypi-internal.yml new file mode 100644 index 000000000..8bcad76eb --- /dev/null +++ b/.github/workflows/publish-pypi-internal.yml @@ -0,0 +1,36 @@ +# What? +# Tag and release an arbitrary ref. Uploads to an internal archive. +# +# How? +# After checking out and testing the provided ref, the image is built and uploaded. +# +# When? +# - manually +name: "Publish to internal PyPI" + +on: + workflow_dispatch: + inputs: + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + ref: + description: "The ref (sha or branch name) to use" + type: string + required: true + test-command: + description: "Package test command" + type: string + default: "python -c \"import dbt.adapters.redshift\"" + +jobs: + build-and-publish: + name: "Build and release to internal PyPI" + uses: dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main + with: + version_number: "${{ inputs.version }}" + package_test_command: "${{ inputs.test-command }}" + dbms_name: "redshift" + ref: ${{ inputs.ref }} + secrets: inherit diff --git a/.github/workflows/release-branch-tests.yml b/.github/workflows/release-branch-tests.yml deleted file mode 100644 index 004c6fb29..000000000 --- a/.github/workflows/release-branch-tests.yml +++ /dev/null @@ -1,31 +0,0 @@ -# **what?** -# The purpose of this workflow is to trigger CI to run for each release -# branch on a regular cadence. If the CI workflow fails for a branch, it -# will post to dev-core-alerts to raise awareness. The - -# **why?** -# Ensures release branches are always shippable and not broken. -# Also, can catch any dependencies shifting beneath us that might -# introduce breaking changes (could also impact Cloud). - -# **when?** -# Once each morning. Manual trigger can also test on demand - -name: Release branch scheduled testing - -on: - # run this once per night to ensure no regressions from latest dbt-core changes - schedule: - - cron: '0 5 * * *' # 5 UTC - - workflow_dispatch: # for manual triggering - -# no special access is needed -permissions: read-all - -jobs: - run_tests: - uses: dbt-labs/actions/.github/workflows/release-branch-tests.yml@main - with: - workflows_to_run: '["main.yml", "integration.yml"]' - secrets: inherit diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml deleted file mode 100644 index 0ce0f9adf..000000000 --- a/.github/workflows/release-internal.yml +++ /dev/null @@ -1,49 +0,0 @@ -# What? -# -# Tag and release an arbitrary ref. Uploads to an internal archive for further processing. -# -# How? -# -# After checking out and testing the provided ref, the image is built and uploaded. -# -# When? -# -# Manual trigger. - -name: "Release internal patch" - -on: - workflow_dispatch: - inputs: - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - ref: - description: "The ref (sha or branch name) to use" - type: string - default: "main" - required: true - package_test_command: - description: "Package test command" - type: string - default: "python -c \"import dbt.adapters.redshift\"" - required: true - -defaults: - run: - shell: "bash" - -jobs: - invoke-reusable-workflow: - name: "Build and Release Internally" - - uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main" - - with: - version_number: "${{ inputs.version_number }}" - package_test_command: "${{ inputs.package_test_command }}" - dbms_name: "redshift" - ref: "${{ inputs.ref }}" - - secrets: "inherit" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f68eb3320..bdee63147 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,216 +13,220 @@ # # **when?** # This workflow can be run manually on demand or can be called by other workflows -name: Release to GitHub and PyPI +name: "Release pipeline" +run-name: "Release `${{ inputs.version }}` to `${{ inputs.deploy-environment }}`" on: - workflow_dispatch: - inputs: - sha: - description: "The last commit sha in the release" - type: string - required: true - target_branch: - description: "The branch to release from" - type: string - required: true - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - build_script_path: - description: "Build script path" - type: string - default: "scripts/build-dist.sh" - required: true - env_setup_script_path: - description: "Environment setup script path" - type: string - default: "scripts/env-setup.sh" - required: false - s3_bucket_name: - description: "AWS S3 bucket name" - type: string - default: "core-team-artifacts" - required: true - package_test_command: - description: "Package test command" - type: string - default: "python -c \"import dbt.adapters.redshift\"" - required: true - test_run: - description: "Test run (Publish release as draft)" - type: boolean - default: true - required: false - nightly_release: - description: "Nightly release to dev environment" - type: boolean - default: false - required: false - workflow_call: - inputs: - sha: - description: "The last commit sha in the release" - type: string - required: true - target_branch: - description: "The branch to release from" - type: string - required: true - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - build_script_path: - description: "Build script path" - type: string - default: "scripts/build-dist.sh" - required: true - env_setup_script_path: - description: "Environment setup script path" - type: string - default: "scripts/env-setup.sh" - required: false - s3_bucket_name: - description: "AWS S3 bucket name" - type: string - default: "core-team-artifacts" - required: true - package_test_command: - description: "Package test command" - type: string - default: "python -c \"import dbt.adapters.redshift\"" - required: true - test_run: - description: "Test run (Publish release as draft)" - type: boolean - default: true - required: false - nightly_release: - description: "Nightly release to dev environment" - type: boolean - default: false - required: false - + workflow_call: + inputs: + sha: + description: "The commit sha to release" + type: string + required: true + branch: + description: "The branch to release from" + type: string + default: "main" + version: + description: "The release version (e.g. 1.0.0b1)" + type: string + required: true + deploy-environment: + description: "Where to publish" + type: string + default: "prod" + publish-slack-override: + description: "Use to publish a Slack notification for non-prod deploy environments" + type: boolean + default: false + workflow_dispatch: + inputs: + sha: + description: "The commit sha to release" + type: string + required: true + branch: + description: "The branch to release from" + type: string + default: "main" + version: + description: "The release version (e.g. 1.0.0b1)" + type: string + required: true + deploy-environment: + description: "Where to publish" + type: environment + publish-slack-override: + description: "Use to publish a Slack notification for non-prod deploy environments" + type: boolean + default: false + +# this is the permission that allows creating a new release to both GitHub and PyPI permissions: - contents: write # this is the permission that allows creating a new release + contents: write + id-token: write -defaults: - run: - shell: bash +# deploying the same version of a package to the same environment should override any previous deployment with those attributes +concurrency: + group: "${{ github.workflow }}-${{ inputs.version }}-${{ inputs.deploy-environment }}" + cancel-in-progress: true jobs: - log-inputs: - name: Log Inputs - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The branch to release from: ${{ inputs.target_branch }} - echo The release version number: ${{ inputs.version_number }} - echo Build script path: ${{ inputs.build_script_path }} - echo Environment setup script path: ${{ inputs.env_setup_script_path }} - echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }} - echo Package test command: ${{ inputs.package_test_command }} - echo Test run: ${{ inputs.test_run }} - echo Nightly release: ${{ inputs.nightly_release }} - - bump-version-generate-changelog: - name: Bump package version, Generate changelog - - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main - - with: - sha: ${{ inputs.sha }} - version_number: ${{ inputs.version_number }} - target_branch: ${{ inputs.target_branch }} - env_setup_script_path: ${{ inputs.env_setup_script_path }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - - secrets: inherit - - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} - - needs: [bump-version-generate-changelog] - - runs-on: ubuntu-latest - - steps: - - name: Print variables - run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - - build-test-package: - name: Build, Test, Package - if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog] - - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - build_script_path: ${{ inputs.build_script_path }} - s3_bucket_name: ${{ inputs.s3_bucket_name }} - package_test_command: ${{ inputs.package_test_command }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - github-release: - name: GitHub Release - if: ${{ !failure() && !cancelled() }} - - needs: [bump-version-generate-changelog, build-test-package] - - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main - - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - test_run: ${{ inputs.test_run }} - - pypi-release: - name: PyPI Release - - needs: [github-release] - - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - - with: - version_number: ${{ inputs.version_number }} - test_run: ${{ inputs.test_run }} - - secrets: - PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - - slack-notification: - name: Slack Notification - if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} - - needs: - [ - bump-version-generate-changelog, - build-test-package, - github-release, - pypi-release, - ] - - uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main - with: - status: "failure" - - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} + release-inputs: + name: "Release inputs" + runs-on: ubuntu-latest + outputs: + archive-name: ${{ steps.computed-inputs.outputs.archive-name }} + steps: + - name: "Set: archive name" + id: archive + shell: bash + run: | + archive_name=${{ inputs.package }}-${{ inputs.version_number }}-${{ inputs.deploy-environment }} + echo "name=$archive_name" >> $GITHUB_OUTPUT + + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo branch : ${{ inputs.branch }} + echo package : ${{ inputs.package }} + echo version : ${{ inputs.version }} + echo deploy-environment : ${{ inputs.deploy-environment }} + echo archive-name : ${{ steps.archive.outputs.name }} + echo dbt-adapters-branch : ${{ inputs.dbt-adapters-branch }} + echo dbt-common-branch : ${{ inputs.dbt-common-branch }} + echo dbt-core-branch : ${{ inputs.dbt-core-branch }} + + release-branch: + name: "Create a release branch" + needs: release-inputs + uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-create.yml@update-workflows + with: + branch: ${{ inputs.branch }} + version: ${{ inputs.version }} + secrets: inherit + + version-bump: + name: "Bump the version" + runs-on: ubuntu-latest + needs: release-branch + steps: + - name: "Check out ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.branch }} + + - name: "Bump version to `${{ inputs.version }}`" + shell: bash + run: hatch version ${{ inputs.version }} + + - name: "Commit and push changes" + shell: bash + run: commit + + changelog: + name: "Generate a new changelog" + needs: release-branch + uses: dbt-labs/dbt-adapters/.github/workflows/build-changelog.yml@update-workflows + with: + branch: ${{ needs.release-branch.outputs.branch }} + version: ${{ inputs.version }} + secrets: inherit + + code-quality: + name: "Run code quality" + needs: + - release-branch + - version-bump + - changelog + uses: ./.github/workflows/code-quality.yml + with: + branch: ${{ needs.release-branch.outputs.branch }} + + unit-tests: + name: "Run unit tests" + needs: + - release-branch + - version-bump + - changelog + uses: ./.github/workflows/unit-tests.yml + with: + branch: ${{ needs.release-branch.outputs.branch }} + + integration-tests: + name: "Run unit tests" + needs: + - release-branch + - version-bump + - changelog + uses: ./.github/workflows/integration-tests.yml + with: + branch: ${{ needs.release-branch.outputs.branch }} + + build-artifacts: + name: "Build artifacts" + if: ${{ !failure() && !cancelled() }} + needs: + - release-inputs + - release-branch + - code-quality + - unit-tests + - integration-tests + uses: ./.github/workflows/build-artifacts.yml + with: + branch: ${{ needs.release-branch.outputs.branch }} + archive-name: ${{ needs.release-inputs.outputs.archive-name }} + + merge-changes: + name: "Merge the version bump and changelog updates" + needs: + - build-artifacts + - release-branch + if: >- + !failure() && !cancelled() && + inputs.deploy-environment == 'prod' && + needs.release-branch.outputs.name + uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-merge.yml@update-workflows + with: + branch: ${{ inputs.branch }} + release-branch: ${{ needs.release-branch.outputs.branch }} + secrets: inherit + + publish-github: + name: "Publish to GitHub" + if: ${{ !failure() && !cancelled() }} + needs: + - release-inputs + - changelog + - merge-changes + uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows + with: + archive-name: ${{ needs.release-inputs.outputs.archive-name }} + version: ${{ inputs.version }} + deploy-environment: ${{ inputs.deploy-environment }} + sha: ${{ needs.merge-changes.outputs.sha }} + changelog-path: ${{ needs.changelog.outputs.changelog-path }} + + publish-pypi: + name: "Publish to PyPI" + if: ${{ !failure() && !cancelled() }} + needs: + - release-inputs + - merge-changes + uses: dbt-labs/dbt-adapters/.github/workflows/publish-pypi.yml@update-workflows + with: + archive-name: ${{ needs.release-inputs.outputs.archive-name }} + version: ${{ inputs.version }} + deploy-environment: ${{ inputs.deploy-environment }} + + slack-notification: + name: "Slack notification" + if: ${{ failure() && (inputs.deploy-environment == 'prod' || inputs.publish-slack-override) }} + needs: + - github-release + - pypi-release + uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main + with: + status: "failure" + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.github/workflows/scheduled-test-releases.yml b/.github/workflows/scheduled-test-releases.yml new file mode 100644 index 000000000..1537d7b42 --- /dev/null +++ b/.github/workflows/scheduled-test-releases.yml @@ -0,0 +1,72 @@ +# **what?** +# Nightly releases to GitHub and PyPI. This workflow produces the following outcome: +# - generate and validate data for night release (commit SHA, version number, release branch); +# - pass data to release workflow; +# - night release will be pushed to GitHub as a draft release; +# - night build will be pushed to test PyPI; +# +# **why?** +# Ensure an automated and tested release process for nightly builds +# +# **when?** +# - scheduled with the repository variable DBT_SCHEDULED_TEST_RELEASES_SCHEDULE +# - manually +name: "Scheduled test releases" + +on: + workflow_dispatch: # for manual triggering + schedule: + - cron: ${{ vars.DBT_SCHEDULED_TEST_RELEASES_SCHEDULE }} + +permissions: + contents: write # this is the permission that allows creating a new release + +defaults: + run: + shell: bash + +jobs: + release-inputs: + name: "Get test release inputs" + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.commit.outputs.sha }} + version: ${{ steps.release-version.outputs.version }} + + steps: + - name: "Checkout ${{ github.repository }}" + uses: actions/checkout@v3 + + - name: "Set: commit" + id: commit + run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: "Set: version" + id: current-version + run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT + + - name: "Audit Version And Parse Into Parts" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ steps.current-version.outputs.version }} + + - name: "Set: release version" + id: release-version + run: | + version="${{ steps.semver.outputs.major }}" + version+=".${{ steps.semver.outputs.minor }}" + version+=".$(${{ steps.semver.outputs.patch }}+1)" + version+="dev$(date +'%m%d%Y')" + echo "version=$version" >> $GITHUB_OUTPUT + + release: + name: "Release test release" + needs: release-inputs + uses: ./.github/workflows/release.yml + with: + sha: ${{ needs.release-inputs.outputs.sha }} + version: ${{ needs.release-inputs.outputs.version }} + branch: "main" + deploy-environment: "test" + publish-slack-override: true diff --git a/.github/workflows/scheduled-tests.yml b/.github/workflows/scheduled-tests.yml new file mode 100644 index 000000000..cd5099492 --- /dev/null +++ b/.github/workflows/scheduled-tests.yml @@ -0,0 +1,72 @@ +# **what?** +# The purpose of this workflow is to trigger CI to run for each release +# branch on a regular cadence. If the CI workflow fails for a branch, it +# will post to dev-core-alerts to raise awareness. +# +# **why?** +# Ensures release branches are always shippable and not broken. +# Also, can catch any dependencies shifting beneath us that might +# introduce breaking changes (could also impact Cloud). + +# **when?** +# - scheduled with the repository variable DBT_SCHEDULED_TESTS_SCHEDULE +# - manually +name: "Scheduled tests" + +on: + schedule: + - cron: ${{ vars.DBT_SCHEDULED_TESTS_SCHEDULE }} + workflow_dispatch: + +permissions: read-all + +jobs: + code-quality: + name: "Code quality" + uses: dbt-labs/dbt-redshift/.github/workflows/code-quality.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + integration-tests: + name: "Integration tests" + uses: dbt-labs/dbt-redshift/.github/workflows/integration-tests.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + unit-tests: + name: "Unit tests" + uses: dbt-labs/dbt-redshift/.github/workflows/unit-tests.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + verify-build-artifacts: + name: "Unit tests" + uses: dbt-labs/dbt-redshift/.github/workflows/build-artifacts.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + post-failure: + runs-on: ubuntu-latest + needs: test + if: ${{ failure() }} + steps: + - name: Posting scheduled run failures + uses: ravsamhq/notify-slack-action@v2 + if: ${{ github.event_name == 'schedule' }} + with: + notification_title: 'Redshift nightly integration test failed' + status: ${{ job.status }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml new file mode 100644 index 000000000..e2ecd89dd --- /dev/null +++ b/.github/workflows/stale-issues.yml @@ -0,0 +1,13 @@ +name: "Close stale issues and PRs" + +on: + schedule: + - cron: "30 1 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/triage-labels.yml b/.github/workflows/triage-labels.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 000000000..6da8914b6 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,102 @@ +name: "Unit tests" + +on: + pull_request_target: + push: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + cancel-in-progress: true + +jobs: + unit-tests: + name: "Unit tests" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJSON(vars.DBT_TEST_PYTHON_ALL_VERSIONS) }} + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + persist-credentials: false + + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + - name: "Update development branches" + if: ${{ contains(github.event_name, 'workflow_') }} + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + dbt-core-branch: ${{ inputs.dbt-core-branch }} + + - name: "Run unit tests" + shell: bash + run: hatch run unit-tests:all + + aggregate-results: + name: "Successful unit tests" + needs: unit-tests + if: ${{ !failure() && !cancelled() }} + runs-on: ubuntu-latest + steps: + - name: "Unit tests completed successfully" + shell: bash + run: | + title="Unit tests" + message="Unit tests completed successfully!" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml deleted file mode 100644 index bde34d683..000000000 --- a/.github/workflows/version-bump.yml +++ /dev/null @@ -1,28 +0,0 @@ -# **what?** -# This workflow will take the new version number to bump to. With that -# it will run versionbump to update the version number everywhere in the -# code base and then run changie to create the corresponding changelog. -# A PR will be created with the changes that can be reviewed before committing. - -# **why?** -# This is to aid in releasing dbt and making sure we have updated -# the version in all places and generated the changelog. - -# **when?** -# This is triggered manually - -name: Version Bump - -on: - workflow_dispatch: - inputs: - version_number: - description: 'The version number to bump to (ex. 1.2.0, 1.3.0b1)' - required: true - -jobs: - version_bump_and_changie: - uses: dbt-labs/actions/.github/workflows/version-bump.yml@main - with: - version_number: ${{ inputs.version_number }} - secrets: inherit # ok since what we are calling is internally maintained diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index cfbc714ed..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -recursive-include dbt/include *.sql *.yml *.md diff --git a/Makefile b/Makefile deleted file mode 100644 index efd23b806..000000000 --- a/Makefile +++ /dev/null @@ -1,67 +0,0 @@ -.DEFAULT_GOAL:=help - -.PHONY: dev -dev: ## Installs adapter in develop mode along with development dependencies - @\ - pip install -e . -r dev-requirements.txt && pre-commit install - -.PHONY: dev-uninstall -dev-uninstall: ## Uninstalls all packages while maintaining the virtual environment - ## Useful when updating versions, or if you accidentally installed into the system interpreter - pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y - pip uninstall -y dbt-redshift - -.PHONY: mypy -mypy: ## Runs mypy against staged changes for static type checking. - @\ - pre-commit run --hook-stage manual mypy-check | grep -v "INFO" - -.PHONY: flake8 -flake8: ## Runs flake8 against staged changes to enforce style guide. - @\ - pre-commit run --hook-stage manual flake8-check | grep -v "INFO" - -.PHONY: black -black: ## Runs black against staged changes to enforce style guide. - @\ - pre-commit run --hook-stage manual black-check -v | grep -v "INFO" - -.PHONY: lint -lint: ## Runs flake8 and mypy code checks against staged changes. - @\ - pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run mypy-check --hook-stage manual | grep -v "INFO" - -.PHONY: linecheck -linecheck: ## Checks for all Python lines 100 characters or more - @\ - find dbt -type f -name "*.py" -exec grep -I -r -n '.\{100\}' {} \; - -.PHONY: unit -unit: ## Runs unit tests with py38. - @\ - tox -e py38 - -.PHONY: test -test: ## Runs unit tests with py38 and code checks against staged changes. - @\ - tox -p -e py38; \ - pre-commit run black-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run mypy-check --hook-stage manual | grep -v "INFO" - -.PHONY: integration -integration: ## Runs redshift integration tests with py38. - @\ - tox -e py38-redshift -- - -.PHONY: clean - @echo "cleaning repo" - @git clean -f -X - -.PHONY: help -help: ## Show this help message. - @echo 'usage: make [target]' - @echo - @echo 'targets:' - @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index d02863ae0..000000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,26 +0,0 @@ -# install latest changes in dbt-core + dbt-postgres -git+https://github.com/dbt-labs/dbt-adapters.git -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git -git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core -git+https://github.com/dbt-labs/dbt-postgres.git - -# dev -ipdb~=0.13.13 -pre-commit==3.7.0;python_version >="3.9" -pre-commit==3.5.0;python_version <"3.9" - -# test -ddtrace==2.3.0 -freezegun~=1.3 -pytest~=7.4 -pytest-csv~=3.0 -pytest-dotenv~=0.5.2 -pytest-logbook~=1.2 -pytest-xdist~=3.5 -tox~=4.11 - -# build -bumpversion~=0.6.0 -twine~=4.0 -wheel~=0.42 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..7a6fcca54 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,137 @@ +[project] +dynamic = ["version"] +name = "dbt-redshift" +description = "The Redshift adapter plugin for dbt" +readme = "README.md" +keywords = ["dbt", "redshift", "dbt-redshift", "adapter", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs"] +requires-python = ">=3.8.0" +authors = [ + { name = "dbt Labs", email = "info@dbtlabs.com" }, +] +maintainers = [ + { name = "dbt Labs", email = "info@dbtlabs.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +dependencies = [ + "dbt-adapters>=0.1.0a1,<2.0", + "dbt-common>=0.1.0a1,<2.0", + # `dbt-redshift` depends deeply on `redshift-connector` + # This package has experienced breaking changes in previous patch releases + # Pin to the patch version and bump version cautiously + "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", + "dbt-postgres>=1.8.0a1,<1.9", + "sqlparse>=0.5.0,<0.6.0", + "agate>=1.0,<2.0", + # Add `dbt-core` to ensure backwards compatibility of installation + # This is not a functional dependency + "dbt-core>=1.8.0b3", +] + +[project.urls] +Homepage = "https://github.com/dbt-labs/dbt-redshift" +Documentation = "https://docs.getdbt.com" +Repository = "https://github.com/dbt-labs/dbt-redshift.git" +Issues = "https://github.com/dbt-labs/dbt-redshift/issues" +Changelog = "https://github.com/dbt-labs/dbt-redshift/blob/main/CHANGELOG.md" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.sdist] +include = ["dbt"] + +[tool.hatch.build.targets.wheel] +packages = ["dbt"] + +[tool.hatch.version] +path = "dbt/adapters/redshift/__version__.py" + +[tool.hatch.envs.default] +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-postgres @ git+https://github.com/dbt-labs/dbt-postgres.git", + "ipdb==0.13.13", + 'pre-commit==3.7.0;python_version>="3.9"', + 'pre-commit==3.5.0;python_version=="3.8"', +] +[tool.hatch.envs.default.scripts] +dev = "pre-commit install" +code-quality = "pre-commit run {args:--all-files}" +clean = "git clean -f -X" + +[tool.hatch.envs.unit-tests] +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "dbt-postgres @ git+https://github.com/dbt-labs/dbt-postgres.git", + "ddtrace==2.3.0", + "freezegun", + "pytest", + "pytest-csv", + "pytest-dotenv", + "pytest-logbook", + "pytest-mock", + "pytest-xdist", +] +[tool.hatch.envs.unit-tests.scripts] +all = "python -m pytest {args:tests/unit}" + +[tool.hatch.envs.integration-tests] +template = "unit-tests" +extra-dependencies = [ + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", +] +[tool.hatch.envs.integration-tests.scripts] +all = [ + 'python -m pytest --dist=loadscope tests/functional -k "not tests/functional/adapter/utils and not tests/functional/adapter/incremental"', + 'python -m pytest --dist=loadscope tests/functional/adapter/incremental', + 'python -m pytest --dist=loadscope tests/functional/adapter/utils', +] +[tool.hatch.envs.build] +detached = true +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] +[tool.hatch.envs.build.scripts] +check-all = [ + "- check-wheel", + "- check-sdist", +] +check-wheel = [ + "twine check dist/*", + "find ./dist/dbt*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-redshift", +] +check-sdist = [ + "check-wheel-contents dist/*.whl --ignore W007,W008", + "find ./dist/dbt*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-redshift", +] + +[tool.pytest] +color = true +verbose = true +env_files = ["test.env"] +testpaths = [ + "tests/functional", + "tests/unit", +] +filterwarnings = [ + "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", + "ignore:unclosed file .*:ResourceWarning", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b3d74bc14..000000000 --- a/pytest.ini +++ /dev/null @@ -1,9 +0,0 @@ -[pytest] -filterwarnings = - ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning - ignore:unclosed file .*:ResourceWarning -env_files = - test.env -testpaths = - tests/unit - tests/functional diff --git a/setup.py b/setup.py deleted file mode 100644 index dbb3913b9..000000000 --- a/setup.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -import re -import sys - -if sys.version_info < (3, 8): - print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.8 or higher.") - sys.exit(1) - - -try: - from setuptools import find_namespace_packages -except ImportError: - print("Error: dbt requires setuptools v40.1.0 or higher.") - print('Please upgrade setuptools with "pip install --upgrade setuptools" and try again') - sys.exit(1) - - -from pathlib import Path -from setuptools import setup - - -# pull the long description from the README -README = Path(__file__).parent / "README.md" - - -# used for this adapter's version and in determining the compatible dbt-core version -VERSION = Path(__file__).parent / "dbt/adapters/redshift/__version__.py" - - -def _plugin_version() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - return attributes["version"] - - -def _plugin_version_trim() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - pattern = r"\+build\d+$" - return re.sub(pattern, "", attributes["version"]) - - -setup( - name="dbt-redshift", - version=_plugin_version(), - description="The Redshift adapter plugin for dbt", - long_description=README.read_text(), - long_description_content_type="text/markdown", - author="dbt Labs", - author_email="info@dbtlabs.com", - url="https://github.com/dbt-labs/dbt-redshift", - packages=find_namespace_packages(include=["dbt", "dbt.*"]), - include_package_data=True, - install_requires=[ - "dbt-common>=0.1.0a1,<2.0", - "dbt-adapters>=0.1.0a1,<2.0", - f"dbt-postgres~={_plugin_version_trim()}", - # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases - # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. - "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", - # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency - "dbt-core>=1.8.0b3", - # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core - "sqlparse>=0.5.0,<0.6.0", - "agate", - ], - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: Apache Software License", - "Operating System :: Microsoft :: Windows", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - ], - python_requires=">=3.8", -) diff --git a/test.env.example b/test.env.example index 4de05edab..44b34f5b1 100644 --- a/test.env.example +++ b/test.env.example @@ -1,18 +1,9 @@ -# Note: Make sure you have a Redshift account that is set up so these fields are easy to complete. - -### Test Environment field definitions -# These will all be gathered from account information or created by you. -# Endpoint for Redshift connection REDSHIFT_TEST_HOST= -# Username on your account REDSHIFT_TEST_USER= -# Password for Redshift account REDSHIFT_TEST_PASS= -# Local port to connect on REDSHIFT_TEST_PORT= -# Name of Redshift database in your account to test against REDSHIFT_TEST_DBNAME= -# Users for testing + DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index c490fed9a..000000000 --- a/tox.ini +++ /dev/null @@ -1,34 +0,0 @@ -[tox] -skipsdist = True -envlist = py38,py39,py310,py311 - -[testenv:{unit,py38,py39,py310,py311,py}] -description = unit testing -skip_install = true -passenv = - DBT_* - PYTEST_ADDOPTS -commands = {envpython} -m pytest {posargs} tests/unit -deps = - -rdev-requirements.txt - -e. - -[testenv:{integration,py38,py39,py310,py311,py}-{redshift}] -description = adapter plugin integration testing -skip_install = true -passenv = - DBT_* - REDSHIFT_TEST_* - PYTEST_ADDOPTS - DD_CIVISIBILITY_AGENTLESS_ENABLED - DD_API_KEY - DD_SITE - DD_ENV - DD_SERVICE -commands = - {envpython} -m pytest --dist=loadscope {posargs} tests/functional -k "not tests/functional/adapter/utils and not tests/functional/adapter/incremental" - {envpython} -m pytest --dist=loadscope {posargs} tests/functional/adapter/utils - {envpython} -m pytest --dist=loadscope {posargs} tests/functional/adapter/incremental -deps = - -rdev-requirements.txt - -e. From c30dd660c58c183f74436405cb224375689ebd9b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 20 Apr 2024 11:20:04 -0400 Subject: [PATCH 04/15] merge internal pypi release into full release pipeline --- .github/workflows/publish-pypi-internal.yml | 36 --------------------- .github/workflows/release.yml | 14 ++++++++ 2 files changed, 14 insertions(+), 36 deletions(-) delete mode 100644 .github/workflows/publish-pypi-internal.yml diff --git a/.github/workflows/publish-pypi-internal.yml b/.github/workflows/publish-pypi-internal.yml deleted file mode 100644 index 8bcad76eb..000000000 --- a/.github/workflows/publish-pypi-internal.yml +++ /dev/null @@ -1,36 +0,0 @@ -# What? -# Tag and release an arbitrary ref. Uploads to an internal archive. -# -# How? -# After checking out and testing the provided ref, the image is built and uploaded. -# -# When? -# - manually -name: "Publish to internal PyPI" - -on: - workflow_dispatch: - inputs: - version: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - ref: - description: "The ref (sha or branch name) to use" - type: string - required: true - test-command: - description: "Package test command" - type: string - default: "python -c \"import dbt.adapters.redshift\"" - -jobs: - build-and-publish: - name: "Build and release to internal PyPI" - uses: dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main - with: - version_number: "${{ inputs.version }}" - package_test_command: "${{ inputs.test-command }}" - dbms_name: "redshift" - ref: ${{ inputs.ref }} - secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bdee63147..6766c4c7a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -219,6 +219,20 @@ jobs: version: ${{ inputs.version }} deploy-environment: ${{ inputs.deploy-environment }} + publish-pypi-internal: + name: "Publish to internal PyPI" + if: ${{ !failure() && !cancelled() }} + needs: + - release-inputs + - merge-changes + uses: dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main + with: + version_number: "${{ inputs.version }}" + package_test_command: "python -c \"import dbt.adapters.redshift\"" + dbms_name: "redshift" + ref: ${{ needs.merge-changes.outputs.sha }} + secrets: inherit + slack-notification: name: "Slack notification" if: ${{ failure() && (inputs.deploy-environment == 'prod' || inputs.publish-slack-override) }} From 2944b5e3b96da8735e916c937bb47a2445bc303e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 21 Apr 2024 16:53:17 -0400 Subject: [PATCH 05/15] commit changes to break into separate prs --- .github/workflows/backport.yml | 3 +- .github/workflows/bot-changelog.yml | 10 +- .github/workflows/build-artifacts.yml | 61 ---------- .github/workflows/clean-repo.yml | 27 ----- .github/workflows/code-quality.yml | 86 -------------- ...s.yml => dev-branch-integration-tests.yml} | 57 ++++----- ...k.yml => pull-request-changelog-check.yml} | 4 +- .github/workflows/pull-request-checks.yml | 44 +++++++ .github/workflows/release.yml | 110 +++++++++++------- ...heduled-tests.yml => scheduled-checks.yml} | 18 +-- ...-issues.yml => scheduled-stale-issues.yml} | 1 + .github/workflows/scheduled-test-releases.yml | 6 +- .github/workflows/unit-tests.yml | 102 ---------------- 13 files changed, 157 insertions(+), 372 deletions(-) delete mode 100644 .github/workflows/build-artifacts.yml delete mode 100644 .github/workflows/clean-repo.yml delete mode 100644 .github/workflows/code-quality.yml rename .github/workflows/{integration-tests.yml => dev-branch-integration-tests.yml} (75%) rename .github/workflows/{changelog-check.yml => pull-request-changelog-check.yml} (91%) create mode 100644 .github/workflows/pull-request-checks.yml rename .github/workflows/{scheduled-tests.yml => scheduled-checks.yml} (86%) rename .github/workflows/{stale-issues.yml => scheduled-stale-issues.yml} (90%) delete mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 454ab12c9..d91fd5e6f 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -19,8 +19,7 @@ on: - labeled permissions: - - pull-requests: write# to create the PR + pull-requests: write # to create the PR contents: write # to create the comment if the process fails jobs: diff --git a/.github/workflows/bot-changelog.yml b/.github/workflows/bot-changelog.yml index 92d857636..98b461c42 100644 --- a/.github/workflows/bot-changelog.yml +++ b/.github/workflows/bot-changelog.yml @@ -2,11 +2,11 @@ # - create a `changie` entry for automated PRs using this template: # # kind: (e.g. Dependencies) -# body: -# time: -# custom: -# Author: -# PR: +# body: +# time: +# custom: +# Author: +# PR: # # **why?** # - automate and standardize `changie` entry generation for automated PRs diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml deleted file mode 100644 index 2907f7f1e..000000000 --- a/.github/workflows/build-artifacts.yml +++ /dev/null @@ -1,61 +0,0 @@ -# **what?** -# Verifies python build on all code commited to the repository. This workflow -# should not require any secrets since it runs for PRs from forked repos. By -# default, secrets are not passed to workflows running from a forked repos. - -# **why?** -# Ensure code for dbt meets a certain quality standard. - -# **when?** -# This will run for all PRs, when code is pushed to main, and when manually triggered. -name: "Build release" - -on: - workflow_call: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - archive-name: - description: "The name to use for the upload archive, leave blank for no upload" - type: string - default: "" - workflow_dispatch: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - archive-name: - description: "The name to use for the upload archive, leave blank for no upload" - type: string - default: "" - -permissions: read-all - -concurrency: - group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.archive-name }}" - cancel-in-progress: true - -jobs: - build: - name: "Build a release" - runs-on: ubuntu-latest - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - steps: - - name: "Check out ${{ github.repository }}@${{ inputs.branch }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - - - name: "Setup environment" - uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows - with: - python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} - - - name: "Build ${{ github.event.repository.name }}" - uses: dbt-labs/dbt-adapters/.github/actions/build-artifacts@update-workflows - with: - archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/clean-repo.yml b/.github/workflows/clean-repo.yml deleted file mode 100644 index 2ce2837b3..000000000 --- a/.github/workflows/clean-repo.yml +++ /dev/null @@ -1,27 +0,0 @@ -# **what?** -# Cleanup branches left over from automation and testing. -# Also cleanup draft releases from release testing. -# -# **why?** -# The automations are leaving behind branches and releases that clutter the repository. -# Sometimes we need them to debug processes so we don't want them immediately deleted. -# Running on Saturday to avoid running at the same time as an actual release -# to prevent breaking a release mid-release. -# -# **when?** -# - every Saturday at noon UTC -# - manually -name: "Clean repo" - -on: - schedule: - - cron: '0 12 * * SAT' # noon UTC on Saturday - details in `why` above - workflow_dispatch: - -permissions: - contents: write - -jobs: - cleanup-repo: - uses: dbt-labs/actions/.github/workflows/repository-cleanup.yml@main - secrets: inherit diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml deleted file mode 100644 index 800de1a6f..000000000 --- a/.github/workflows/code-quality.yml +++ /dev/null @@ -1,86 +0,0 @@ -# **what?** -# Run code quality checks, e.g. `black`, `flake8`, `mypy`, via `pre-commit` -# -# **why?** -# Ensure code quality meets dbt Labs standards -# -# **when?** -# - runs on all PRs into protected branches -# - runs indirectly during releases and release tests -# - can be manually run -name: "Code quality" - -on: - pull_request: - push: - branches: - - "main" - - "*.latest" - workflow_call: - inputs: - branch: - description: "The branch/tag to run code quality on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - workflow_dispatch: - inputs: - branch: - description: "The branch/tag to run code quality on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - -permissions: read-all - -concurrency: - group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" - cancel-in-progress: true - -jobs: - code-quality: - name: "Code quality" - runs-on: ubuntu-latest - steps: - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - persist-credentials: false - - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: "Setup environment" - uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows - with: - python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} - - - name: "Update development branches" - if: ${{ contains(github.event_name, 'workflow_') }} - uses: ./.github/actions/update-dev-branches - with: - dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} - dbt-common-branch: ${{ inputs.dbt-common-branch }} - - - name: "Run code quality" - shell: bash - run: hatch run code-quality diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/dev-branch-integration-tests.yml similarity index 75% rename from .github/workflows/integration-tests.yml rename to .github/workflows/dev-branch-integration-tests.yml index 2873de21b..5e413b23c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/dev-branch-integration-tests.yml @@ -1,29 +1,6 @@ -name: "Integration tests" +name: "Dev branch integration tests" on: - pull_request_target: - push: - branches: - - "main" - - "*.latest" - workflow_call: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - dbt-core-branch: - description: "The branch/tag of `dbt-core` to use" - type: string - default: "main" workflow_dispatch: inputs: branch: @@ -46,7 +23,7 @@ on: permissions: read-all concurrency: - group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.sha }}" cancel-in-progress: true jobs: @@ -67,7 +44,7 @@ jobs: with: msg: | "Thanks for your contribution! "\ - "Since this is a fork, integration tests need to be approved prior to running. "\ + "This PR was created from a fork, integration tests need to be approved prior to running. "\ "@dbt-labs/core will review this PR and approve running integration tests." check_for_duplicate_msg: true @@ -115,7 +92,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: "Setup environment" - uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + uses: dbt-labs/actions/setup-environment@add-hatch-actions with: python-version: ${{ matrix.python-version }} @@ -132,14 +109,28 @@ jobs: run: hatch run integration-tests:all aggregate-results: - name: "Successful integration tests" + name: "Collect integration tests results" needs: integration-tests - if: ${{ !failure() && !cancelled() }} + if: always() && !cancelled() runs-on: ubuntu-latest steps: - - name: "Integration tests completed successfully" + - name: "[INFO] Integration tests - FAILED" + if: failure() shell: bash run: | - title="Integration tests" - message="Integration tests completed successfully!" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + echo "::notice title=[$WORKFLOW]: $TITLE::$MESSAGE" + exit 1 + env: + WORKFLOW: ${{ env.NOTIFICATION_PREFIX }} + TITLE: "Integration tests - FAILED" + MESSAGE: "Integration tests failed!" + + - name: "[INFO] Integration tests - PASSED" + if: ${{ !failure() }} # includes skipped + shell: bash + run: | + echo "::notice title=[$WORKFLOW]: $TITLE::$MESSAGE" + env: + WORKFLOW: ${{ env.NOTIFICATION_PREFIX }} + TITLE: "Integration tests - PASSED" + MESSAGE: "Integration tests completed successfully!" diff --git a/.github/workflows/changelog-check.yml b/.github/workflows/pull-request-changelog-check.yml similarity index 91% rename from .github/workflows/changelog-check.yml rename to .github/workflows/pull-request-changelog-check.yml index dd4b11d76..15c712c84 100644 --- a/.github/workflows/changelog-check.yml +++ b/.github/workflows/pull-request-changelog-check.yml @@ -16,7 +16,7 @@ # - labels are updated on the PR # - new code is pushed to the branch # The action will get skipped if the 'Skip Changelog' label is present. -name: Check Changelog Entry +name: "Changelog check" on: pull_request: @@ -26,7 +26,6 @@ on: - labeled - unlabeled - synchronize - workflow_dispatch: defaults: run: @@ -42,5 +41,4 @@ jobs: with: changelog_comment: "Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see the [dbt-redshift contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md)." skip_label: "Skip Changelog" - # this is only acceptable because we own the action we're calling secrets: inherit diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml new file mode 100644 index 000000000..a5b3358f1 --- /dev/null +++ b/.github/workflows/pull-request-checks.yml @@ -0,0 +1,44 @@ +name: "Pull request checks" + +on: + pull_request_target: + types: + - opened + - synchronize + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.head.ref }}" + cancel-in-progress: true + +jobs: + code-quality: + name: "Run code quality" + uses: dbt-labs/actions/.github/workflows/code-quality.yml + with: + ref: ${{ github.event.pull_request.head.ref }} + check-command: "hatch run code-quality" + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + unit-tests: + name: "Run unit tests" + uses: dbt-labs/actions/.github/workflows/unit-tests.yml + with: + ref: ${{ github.event.pull_request.head.ref }} + test-command: "hatch run unit-tests:all" + python-versions: ${{ vars.DBT_TEST_PYTHON_ALL_VERSIONS }} + + integration-tests: + name: "Run unit tests" + uses: dbt-labs/actions/.github/workflows/integration-tests.yml + with: + ref: ${{ github.event.pull_request.head.ref }} + test-command: "hatch run integration-tests:all" + python-versions-ubuntu: ${{ vars.DBT_TEST_PYTHON_ALL_VERSIONS }} + python-versions-macos: ${{ vars.DBT_TEST_PYTHON_VERSION }} + python-versions-windows: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + verify-build-artifacts: + name: "Verify build artifacts" + uses: dbt-labs/actions/.github/workflows/build-artifacts.yml@add-hatch-actions diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6766c4c7a..1b5dce7ad 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -100,34 +100,36 @@ jobs: release-branch: name: "Create a release branch" needs: release-inputs - uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-create.yml@update-workflows - with: - branch: ${{ inputs.branch }} - version: ${{ inputs.version }} - secrets: inherit - - version-bump: - name: "Bump the version" runs-on: ubuntu-latest - needs: release-branch + outputs: + branch: ${{ steps.branch.outputs.branch-name }} steps: - - name: "Check out ${{ github.event.repository.name }}@${{ inputs.branch }}" + - name: "Checkout `${{ inputs.branch }}`" uses: actions/checkout@v4 with: - ref: ${{ needs.release-branch.outputs.branch }} + ref: ${{ inputs.branch }} - - name: "Bump version to `${{ inputs.version }}`" - shell: bash - run: hatch version ${{ inputs.version }} + - name: "Create release branch" + id: branch + uses: dbt-labs/actions/create-temp-branch@add-hatch-actions + with: + branch-stub: "release/${{ inputs.version }}/" - - name: "Commit and push changes" - shell: bash - run: commit + bump-version: + name: "Bump version" + needs: release-branch + uses: dbt-labs/actions/.github/workflows/build-bump-version.yml@add-hatch-actions + with: + branch: ${{ needs.release-branch.outputs.branch }} + version: ${{ inputs.version }} + secrets: inherit changelog: name: "Generate a new changelog" - needs: release-branch - uses: dbt-labs/dbt-adapters/.github/workflows/build-changelog.yml@update-workflows + needs: + - release-branch + - bump-version + uses: dbt-labs/actions/.github/workflows/build-changelog.yml@add-hatch-actions with: branch: ${{ needs.release-branch.outputs.branch }} version: ${{ inputs.version }} @@ -139,9 +141,10 @@ jobs: - release-branch - version-bump - changelog - uses: ./.github/workflows/code-quality.yml + uses: dbt-labs/actions/.github/workflows/code-quality.yml@add-hatch-actions with: branch: ${{ needs.release-branch.outputs.branch }} + check-command: "hatch run code-quality" unit-tests: name: "Run unit tests" @@ -149,9 +152,9 @@ jobs: - release-branch - version-bump - changelog - uses: ./.github/workflows/unit-tests.yml + uses: dbt-labs/actions/.github/workflows/unit-tests.yml@add-hatch-actions with: - branch: ${{ needs.release-branch.outputs.branch }} + ref: ${{ needs.release-branch.outputs.branch }} integration-tests: name: "Run unit tests" @@ -162,6 +165,7 @@ jobs: uses: ./.github/workflows/integration-tests.yml with: branch: ${{ needs.release-branch.outputs.branch }} + test-command: "hatch run unit-tests:all" build-artifacts: name: "Build artifacts" @@ -172,7 +176,7 @@ jobs: - code-quality - unit-tests - integration-tests - uses: ./.github/workflows/build-artifacts.yml + uses: dbt-labs/actions/.github/workflows/build-artifacts.yml@add-hatch-actions with: branch: ${{ needs.release-branch.outputs.branch }} archive-name: ${{ needs.release-inputs.outputs.archive-name }} @@ -182,15 +186,31 @@ jobs: needs: - build-artifacts - release-branch - if: >- - !failure() && !cancelled() && - inputs.deploy-environment == 'prod' && - needs.release-branch.outputs.name - uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-merge.yml@update-workflows - with: - branch: ${{ inputs.branch }} - release-branch: ${{ needs.release-branch.outputs.branch }} - secrets: inherit + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.merge.outputs.sha }} || ${{ steps.merge-test.outputs.sha }} ) + steps: + - name: "Checkout `${{ needs.release-branch.outputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.branch }} + + - name: "Merge `${{ needs.release-branch.outputs.branch }}` into `${{ inputs.branch }}`" + if: ${{ inputs.deploy-environment == 'prod' }} + id: merge + uses: dbt-labs/actions/github-merge@add-hatch-actions + with: + source-branch: ${{ needs.release-branch.outputs.branch }} + target-branch: ${{ inputs.branch }} + message: "merge {source_ref} into {target_branch} for a release" + + - name: "Get HEAD SHA for test release" + id: merge-test + if: ${{ !(inputs.deploy-environment == 'prod') }} + shell: bash + run: | + git pull + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT publish-github: name: "Publish to GitHub" @@ -199,7 +219,7 @@ jobs: - release-inputs - changelog - merge-changes - uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows + uses: dbt-labs/actions/.github/workflows/publish-github.yml@add-hatch-actions with: archive-name: ${{ needs.release-inputs.outputs.archive-name }} version: ${{ inputs.version }} @@ -209,15 +229,20 @@ jobs: publish-pypi: name: "Publish to PyPI" - if: ${{ !failure() && !cancelled() }} needs: - release-inputs - merge-changes - uses: dbt-labs/dbt-adapters/.github/workflows/publish-pypi.yml@update-workflows - with: - archive-name: ${{ needs.release-inputs.outputs.archive-name }} - version: ${{ inputs.version }} - deploy-environment: ${{ inputs.deploy-environment }} + if: ${{ !failure() && !cancelled() }} + environment: + name: ${{ inputs.deploy-environment }} + url: ${{ vars.PYPI_PROJECT_URL }} + runs-on: ubuntu-latest + steps: + - name: "Publish to PyPI" + uses: .github/actions/publish-pypi + with: + repository-url: ${{ vars.PYPI_REPOSITORY_URL }} + archive-name: ${{ inputs.archive-name }} publish-pypi-internal: name: "Publish to internal PyPI" @@ -235,7 +260,12 @@ jobs: slack-notification: name: "Slack notification" - if: ${{ failure() && (inputs.deploy-environment == 'prod' || inputs.publish-slack-override) }} + if: >- + failure() && + ( + inputs.deploy-environment == 'prod' || + inputs.publish-slack-override + ) needs: - github-release - pypi-release diff --git a/.github/workflows/scheduled-tests.yml b/.github/workflows/scheduled-checks.yml similarity index 86% rename from .github/workflows/scheduled-tests.yml rename to .github/workflows/scheduled-checks.yml index cd5099492..e2af8d7e5 100644 --- a/.github/workflows/scheduled-tests.yml +++ b/.github/workflows/scheduled-checks.yml @@ -23,25 +23,25 @@ permissions: read-all jobs: code-quality: name: "Code quality" - uses: dbt-labs/dbt-redshift/.github/workflows/code-quality.yml + uses: ./.github/workflows/code-quality.yml strategy: matrix: branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} with: branch: ${{ matrix.branch }} - integration-tests: - name: "Integration tests" - uses: dbt-labs/dbt-redshift/.github/workflows/integration-tests.yml + unit-tests: + name: "Unit tests" + uses: ./.github/workflows/unit-tests.yml strategy: matrix: branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} with: branch: ${{ matrix.branch }} - unit-tests: - name: "Unit tests" - uses: dbt-labs/dbt-redshift/.github/workflows/unit-tests.yml + integration-tests: + name: "Integration tests" + uses: ./.github/workflows/integration-tests.yml strategy: matrix: branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} @@ -49,8 +49,8 @@ jobs: branch: ${{ matrix.branch }} verify-build-artifacts: - name: "Unit tests" - uses: dbt-labs/dbt-redshift/.github/workflows/build-artifacts.yml + name: "Verify build artifacts" + uses: dbt-labs/actions/.github/workflows/build-artifacts.yml@add-hatch-actions strategy: matrix: branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/scheduled-stale-issues.yml similarity index 90% rename from .github/workflows/stale-issues.yml rename to .github/workflows/scheduled-stale-issues.yml index e2ecd89dd..51d0a1f49 100644 --- a/.github/workflows/stale-issues.yml +++ b/.github/workflows/scheduled-stale-issues.yml @@ -3,6 +3,7 @@ name: "Close stale issues and PRs" on: schedule: - cron: "30 1 * * *" + workflow_dispatch: permissions: issues: write diff --git a/.github/workflows/scheduled-test-releases.yml b/.github/workflows/scheduled-test-releases.yml index 1537d7b42..296e2b193 100644 --- a/.github/workflows/scheduled-test-releases.yml +++ b/.github/workflows/scheduled-test-releases.yml @@ -14,9 +14,9 @@ name: "Scheduled test releases" on: - workflow_dispatch: # for manual triggering schedule: - cron: ${{ vars.DBT_SCHEDULED_TEST_RELEASES_SCHEDULE }} + workflow_dispatch: permissions: contents: write # this is the permission that allows creating a new release @@ -32,9 +32,8 @@ jobs: outputs: sha: ${{ steps.commit.outputs.sha }} version: ${{ steps.release-version.outputs.version }} - steps: - - name: "Checkout ${{ github.repository }}" + - name: "Checkout `${{ github.repository }}`" uses: actions/checkout@v3 - name: "Set: commit" @@ -67,6 +66,5 @@ jobs: with: sha: ${{ needs.release-inputs.outputs.sha }} version: ${{ needs.release-inputs.outputs.version }} - branch: "main" deploy-environment: "test" publish-slack-override: true diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 6da8914b6..000000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: "Unit tests" - -on: - pull_request_target: - push: - branches: - - "main" - - "*.latest" - workflow_call: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - dbt-core-branch: - description: "The branch/tag of `dbt-core` to use" - type: string - default: "main" - workflow_dispatch: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - dbt-core-branch: - description: "The branch/tag of `dbt-core` to use" - type: string - default: "main" - -permissions: read-all - -concurrency: - group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" - cancel-in-progress: true - -jobs: - unit-tests: - name: "Unit tests" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ${{ fromJSON(vars.DBT_TEST_PYTHON_ALL_VERSIONS) }} - steps: - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - persist-credentials: false - - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: "Setup environment" - uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows - with: - python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} - - - name: "Update development branches" - if: ${{ contains(github.event_name, 'workflow_') }} - uses: ./.github/actions/update-dev-branches - with: - dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} - dbt-common-branch: ${{ inputs.dbt-common-branch }} - dbt-core-branch: ${{ inputs.dbt-core-branch }} - - - name: "Run unit tests" - shell: bash - run: hatch run unit-tests:all - - aggregate-results: - name: "Successful unit tests" - needs: unit-tests - if: ${{ !failure() && !cancelled() }} - runs-on: ubuntu-latest - steps: - - name: "Unit tests completed successfully" - shell: bash - run: | - title="Unit tests" - message="Unit tests completed successfully!" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" From b4580e609bceffc7020b2b864f3cea426c301cb8 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 21 Apr 2024 21:11:01 -0400 Subject: [PATCH 06/15] clean up rebase artifacts --- .github/workflows/bot-changelog.yml | 11 +- .github/workflows/changelog-check.yml | 46 -------- .github/workflows/clean-repo.yml | 27 ----- .github/workflows/code-quality.yml | 86 -------------- .github/workflows/docs-issues.yml | 0 .github/workflows/integration-tests.yml | 145 ------------------------ .github/workflows/release.yml | 2 +- .github/workflows/stale-issues.yml | 13 --- .github/workflows/triage-labels.yml | 0 .github/workflows/unit-tests.yml | 102 ----------------- 10 files changed, 6 insertions(+), 426 deletions(-) delete mode 100644 .github/workflows/changelog-check.yml delete mode 100644 .github/workflows/clean-repo.yml delete mode 100644 .github/workflows/code-quality.yml delete mode 100644 .github/workflows/docs-issues.yml delete mode 100644 .github/workflows/integration-tests.yml delete mode 100644 .github/workflows/stale-issues.yml delete mode 100644 .github/workflows/triage-labels.yml delete mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/bot-changelog.yml b/.github/workflows/bot-changelog.yml index 98b461c42..d7d80b2e8 100644 --- a/.github/workflows/bot-changelog.yml +++ b/.github/workflows/bot-changelog.yml @@ -2,11 +2,11 @@ # - create a `changie` entry for automated PRs using this template: # # kind: (e.g. Dependencies) -# body: -# time: -# custom: -# Author: -# PR: +# body: +# time: +# custom: +# Author: +# PR: # # **why?** # - automate and standardize `changie` entry generation for automated PRs @@ -21,7 +21,6 @@ name: "Bot changelog" on: pull_request: - types: - opened - labeled diff --git a/.github/workflows/changelog-check.yml b/.github/workflows/changelog-check.yml deleted file mode 100644 index dd4b11d76..000000000 --- a/.github/workflows/changelog-check.yml +++ /dev/null @@ -1,46 +0,0 @@ -# **what?** -# Checks that a file has been committed under the /.changes directory -# as a new CHANGELOG entry. Cannot check for a specific filename as -# it is dynamically generated by change type and timestamp. -# This workflow should not require any secrets since it runs for PRs -# from forked repos. -# By default, secrets are not passed to workflows running from -# a forked repo. -# -# **why?** -# Ensure code changes are reflected in the CHANGELOG. -# -# **when?** -# This will run for all PRs going into main and *.latest. It will run when: -# - the PR is opened or reopened -# - labels are updated on the PR -# - new code is pushed to the branch -# The action will get skipped if the 'Skip Changelog' label is present. -name: Check Changelog Entry - -on: - pull_request: - types: - - opened - - reopened - - labeled - - unlabeled - - synchronize - workflow_dispatch: - -defaults: - run: - shell: bash - -permissions: - contents: read - pull-requests: write - -jobs: - changelog: - uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main - with: - changelog_comment: "Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see the [dbt-redshift contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md)." - skip_label: "Skip Changelog" - # this is only acceptable because we own the action we're calling - secrets: inherit diff --git a/.github/workflows/clean-repo.yml b/.github/workflows/clean-repo.yml deleted file mode 100644 index 2ce2837b3..000000000 --- a/.github/workflows/clean-repo.yml +++ /dev/null @@ -1,27 +0,0 @@ -# **what?** -# Cleanup branches left over from automation and testing. -# Also cleanup draft releases from release testing. -# -# **why?** -# The automations are leaving behind branches and releases that clutter the repository. -# Sometimes we need them to debug processes so we don't want them immediately deleted. -# Running on Saturday to avoid running at the same time as an actual release -# to prevent breaking a release mid-release. -# -# **when?** -# - every Saturday at noon UTC -# - manually -name: "Clean repo" - -on: - schedule: - - cron: '0 12 * * SAT' # noon UTC on Saturday - details in `why` above - workflow_dispatch: - -permissions: - contents: write - -jobs: - cleanup-repo: - uses: dbt-labs/actions/.github/workflows/repository-cleanup.yml@main - secrets: inherit diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml deleted file mode 100644 index 800de1a6f..000000000 --- a/.github/workflows/code-quality.yml +++ /dev/null @@ -1,86 +0,0 @@ -# **what?** -# Run code quality checks, e.g. `black`, `flake8`, `mypy`, via `pre-commit` -# -# **why?** -# Ensure code quality meets dbt Labs standards -# -# **when?** -# - runs on all PRs into protected branches -# - runs indirectly during releases and release tests -# - can be manually run -name: "Code quality" - -on: - pull_request: - push: - branches: - - "main" - - "*.latest" - workflow_call: - inputs: - branch: - description: "The branch/tag to run code quality on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - workflow_dispatch: - inputs: - branch: - description: "The branch/tag to run code quality on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - -permissions: read-all - -concurrency: - group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" - cancel-in-progress: true - -jobs: - code-quality: - name: "Code quality" - runs-on: ubuntu-latest - steps: - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - persist-credentials: false - - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: "Setup environment" - uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows - with: - python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} - - - name: "Update development branches" - if: ${{ contains(github.event_name, 'workflow_') }} - uses: ./.github/actions/update-dev-branches - with: - dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} - dbt-common-branch: ${{ inputs.dbt-common-branch }} - - - name: "Run code quality" - shell: bash - run: hatch run code-quality diff --git a/.github/workflows/docs-issues.yml b/.github/workflows/docs-issues.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml deleted file mode 100644 index 2873de21b..000000000 --- a/.github/workflows/integration-tests.yml +++ /dev/null @@ -1,145 +0,0 @@ -name: "Integration tests" - -on: - pull_request_target: - push: - branches: - - "main" - - "*.latest" - workflow_call: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - dbt-core-branch: - description: "The branch/tag of `dbt-core` to use" - type: string - default: "main" - workflow_dispatch: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - dbt-core-branch: - description: "The branch/tag of `dbt-core` to use" - type: string - default: "main" - -permissions: read-all - -concurrency: - group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" - cancel-in-progress: true - -jobs: - require-approval-on-forks: - # if it's a pull request from a fork that has not been approved yet, don't run tests - if: >- - github.event_name == 'pull_request_target' && - github.event.pull_request.head.repo.full_name != github.repository && - !contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest - permissions: - pull-requests: write - steps: - - name: "Author requires permission to run tests" - uses: unsplash/comment-on-pr@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - msg: | - "Thanks for your contribution! "\ - "Since this is a fork, integration tests need to be approved prior to running. "\ - "@dbt-labs/core will review this PR and approve running integration tests." - check_for_duplicate_msg: true - - integration-tests: - name: "Integration tests" - needs: require-approval-on-forks - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - python-version: ${{ fromJSON(vars.DBT_TEST_PYTHON_ALL_VERSIONS) }} - include: - - os: macos-latest - python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} - - os: windows-latest - python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} - env: - REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} - REDSHIFT_TEST_PORT: ${{ vars.REDSHIFT_TEST_PORT }} - REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }} - REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }} - REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} - DBT_TEST_USER_1: ${{ vars.REDSHIFT_TEST_USER_1 }} - DBT_TEST_USER_2: ${{ vars.REDSHIFT_TEST_USER_2 }} - DBT_TEST_USER_3: ${{ vars.REDSHIFT_TEST_USER_3 }} - DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }} - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: ${{ vars.DD_SITE }} - DD_ENV: ${{ vars.DD_ENV }} - DD_SERVICE: ${{ github.event.repository.name }} - steps: - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - persist-credentials: false - - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: "Setup environment" - uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows - with: - python-version: ${{ matrix.python-version }} - - - name: "Update development branches" - if: ${{ contains(github.event_name, 'workflow_') }} - uses: ./.github/actions/update-dev-branches - with: - dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} - dbt-common-branch: ${{ inputs.dbt-common-branch }} - dbt-core-branch: ${{ inputs.dbt-core-branch }} - - - name: "Run integration tests" - shell: bash - run: hatch run integration-tests:all - - aggregate-results: - name: "Successful integration tests" - needs: integration-tests - if: ${{ !failure() && !cancelled() }} - runs-on: ubuntu-latest - steps: - - name: "Integration tests completed successfully" - shell: bash - run: | - title="Integration tests" - message="Integration tests completed successfully!" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1b5dce7ad..272bb9c07 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -162,7 +162,7 @@ jobs: - release-branch - version-bump - changelog - uses: ./.github/workflows/integration-tests.yml + uses: dbt-labs/actions/.github/workflows/integration-tests.yml@add-hatch-actions with: branch: ${{ needs.release-branch.outputs.branch }} test-command: "hatch run unit-tests:all" diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml deleted file mode 100644 index e2ecd89dd..000000000 --- a/.github/workflows/stale-issues.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: "Close stale issues and PRs" - -on: - schedule: - - cron: "30 1 * * *" - -permissions: - issues: write - pull-requests: write - -jobs: - stale: - uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/triage-labels.yml b/.github/workflows/triage-labels.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 6da8914b6..000000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: "Unit tests" - -on: - pull_request_target: - push: - branches: - - "main" - - "*.latest" - workflow_call: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - dbt-core-branch: - description: "The branch/tag of `dbt-core` to use" - type: string - default: "main" - workflow_dispatch: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - dbt-core-branch: - description: "The branch/tag of `dbt-core` to use" - type: string - default: "main" - -permissions: read-all - -concurrency: - group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" - cancel-in-progress: true - -jobs: - unit-tests: - name: "Unit tests" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ${{ fromJSON(vars.DBT_TEST_PYTHON_ALL_VERSIONS) }} - steps: - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - persist-credentials: false - - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: "Setup environment" - uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows - with: - python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} - - - name: "Update development branches" - if: ${{ contains(github.event_name, 'workflow_') }} - uses: ./.github/actions/update-dev-branches - with: - dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} - dbt-common-branch: ${{ inputs.dbt-common-branch }} - dbt-core-branch: ${{ inputs.dbt-core-branch }} - - - name: "Run unit tests" - shell: bash - run: hatch run unit-tests:all - - aggregate-results: - name: "Successful unit tests" - needs: unit-tests - if: ${{ !failure() && !cancelled() }} - runs-on: ubuntu-latest - steps: - - name: "Unit tests completed successfully" - shell: bash - run: | - title="Unit tests" - message="Unit tests completed successfully!" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" From 30ac1bab991410a9f4e269f850ebc50c8f64b504 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 21 Apr 2024 21:12:16 -0400 Subject: [PATCH 07/15] clean up rebase artifacts --- .github/workflows/scheduled-stale-issues.yml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 .github/workflows/scheduled-stale-issues.yml diff --git a/.github/workflows/scheduled-stale-issues.yml b/.github/workflows/scheduled-stale-issues.yml deleted file mode 100644 index 51d0a1f49..000000000 --- a/.github/workflows/scheduled-stale-issues.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: "Close stale issues and PRs" - -on: - schedule: - - cron: "30 1 * * *" - workflow_dispatch: - -permissions: - issues: write - pull-requests: write - -jobs: - stale: - uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main From f511e4cf2f4392dfa795b81ecd8e9c5e5a9eba9b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 21 Apr 2024 21:14:01 -0400 Subject: [PATCH 08/15] clean up rebase artifacts --- .github/workflows/scheduled-tests.yml | 72 --------------------------- 1 file changed, 72 deletions(-) delete mode 100644 .github/workflows/scheduled-tests.yml diff --git a/.github/workflows/scheduled-tests.yml b/.github/workflows/scheduled-tests.yml deleted file mode 100644 index cd5099492..000000000 --- a/.github/workflows/scheduled-tests.yml +++ /dev/null @@ -1,72 +0,0 @@ -# **what?** -# The purpose of this workflow is to trigger CI to run for each release -# branch on a regular cadence. If the CI workflow fails for a branch, it -# will post to dev-core-alerts to raise awareness. -# -# **why?** -# Ensures release branches are always shippable and not broken. -# Also, can catch any dependencies shifting beneath us that might -# introduce breaking changes (could also impact Cloud). - -# **when?** -# - scheduled with the repository variable DBT_SCHEDULED_TESTS_SCHEDULE -# - manually -name: "Scheduled tests" - -on: - schedule: - - cron: ${{ vars.DBT_SCHEDULED_TESTS_SCHEDULE }} - workflow_dispatch: - -permissions: read-all - -jobs: - code-quality: - name: "Code quality" - uses: dbt-labs/dbt-redshift/.github/workflows/code-quality.yml - strategy: - matrix: - branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} - with: - branch: ${{ matrix.branch }} - - integration-tests: - name: "Integration tests" - uses: dbt-labs/dbt-redshift/.github/workflows/integration-tests.yml - strategy: - matrix: - branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} - with: - branch: ${{ matrix.branch }} - - unit-tests: - name: "Unit tests" - uses: dbt-labs/dbt-redshift/.github/workflows/unit-tests.yml - strategy: - matrix: - branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} - with: - branch: ${{ matrix.branch }} - - verify-build-artifacts: - name: "Unit tests" - uses: dbt-labs/dbt-redshift/.github/workflows/build-artifacts.yml - strategy: - matrix: - branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} - with: - branch: ${{ matrix.branch }} - - post-failure: - runs-on: ubuntu-latest - needs: test - if: ${{ failure() }} - steps: - - name: Posting scheduled run failures - uses: ravsamhq/notify-slack-action@v2 - if: ${{ github.event_name == 'schedule' }} - with: - notification_title: 'Redshift nightly integration test failed' - status: ${{ job.status }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} From 8ad441265d5cbf9deb8ccb11e85e16397898a97d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 21 Apr 2024 21:28:58 -0400 Subject: [PATCH 09/15] rename daily checks --- .../workflows/{scheduled-checks.yml => daily-checks.yml} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename .github/workflows/{scheduled-checks.yml => daily-checks.yml} (93%) diff --git a/.github/workflows/scheduled-checks.yml b/.github/workflows/daily-checks.yml similarity index 93% rename from .github/workflows/scheduled-checks.yml rename to .github/workflows/daily-checks.yml index e2af8d7e5..3143d2590 100644 --- a/.github/workflows/scheduled-checks.yml +++ b/.github/workflows/daily-checks.yml @@ -11,11 +11,11 @@ # **when?** # - scheduled with the repository variable DBT_SCHEDULED_TESTS_SCHEDULE # - manually -name: "Scheduled tests" +name: "Daily checks" on: schedule: - - cron: ${{ vars.DBT_SCHEDULED_TESTS_SCHEDULE }} + - cron: ${{ vars.DBT_DAILY_CHECKS_SCHEDULE }} workflow_dispatch: permissions: read-all @@ -48,8 +48,8 @@ jobs: with: branch: ${{ matrix.branch }} - verify-build-artifacts: - name: "Verify build artifacts" + build-artifacts: + name: "Build artifacts" uses: dbt-labs/actions/.github/workflows/build-artifacts.yml@add-hatch-actions strategy: matrix: From 154a89d5456370157084ccbd206ebed0cb96348b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 21 Apr 2024 21:44:11 -0400 Subject: [PATCH 10/15] rename scheduled workflows and variables --- .github/workflows/build-artifacts.yml | 61 ------------------- .github/workflows/daily-checks.yml | 10 +-- ...st-releases.yml => daily-dev-releases.yml} | 8 +-- .../dev-branch-integration-tests.yml | 6 +- .github/workflows/pull-request-checks.yml | 10 +-- .github/workflows/weekly-maintenance.yml | 2 +- 6 files changed, 18 insertions(+), 79 deletions(-) delete mode 100644 .github/workflows/build-artifacts.yml rename .github/workflows/{scheduled-test-releases.yml => daily-dev-releases.yml} (89%) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml deleted file mode 100644 index 2907f7f1e..000000000 --- a/.github/workflows/build-artifacts.yml +++ /dev/null @@ -1,61 +0,0 @@ -# **what?** -# Verifies python build on all code commited to the repository. This workflow -# should not require any secrets since it runs for PRs from forked repos. By -# default, secrets are not passed to workflows running from a forked repos. - -# **why?** -# Ensure code for dbt meets a certain quality standard. - -# **when?** -# This will run for all PRs, when code is pushed to main, and when manually triggered. -name: "Build release" - -on: - workflow_call: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - archive-name: - description: "The name to use for the upload archive, leave blank for no upload" - type: string - default: "" - workflow_dispatch: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - archive-name: - description: "The name to use for the upload archive, leave blank for no upload" - type: string - default: "" - -permissions: read-all - -concurrency: - group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.archive-name }}" - cancel-in-progress: true - -jobs: - build: - name: "Build a release" - runs-on: ubuntu-latest - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - steps: - - name: "Check out ${{ github.repository }}@${{ inputs.branch }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - - - name: "Setup environment" - uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows - with: - python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} - - - name: "Build ${{ github.event.repository.name }}" - uses: dbt-labs/dbt-adapters/.github/actions/build-artifacts@update-workflows - with: - archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/daily-checks.yml b/.github/workflows/daily-checks.yml index 3143d2590..0ee4b5c99 100644 --- a/.github/workflows/daily-checks.yml +++ b/.github/workflows/daily-checks.yml @@ -9,7 +9,7 @@ # introduce breaking changes (could also impact Cloud). # **when?** -# - scheduled with the repository variable DBT_SCHEDULED_TESTS_SCHEDULE +# - daily based on DBT_DAILY_CHECKS_SCHEDULE # - manually name: "Daily checks" @@ -26,7 +26,7 @@ jobs: uses: ./.github/workflows/code-quality.yml strategy: matrix: - branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + branch: ${{ fromJSON(vars.DBT_DAILY_CHECKS_BRANCHES) }} with: branch: ${{ matrix.branch }} @@ -35,7 +35,7 @@ jobs: uses: ./.github/workflows/unit-tests.yml strategy: matrix: - branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + branch: ${{ fromJSON(vars.DBT_DAILY_CHECKS_BRANCHES) }} with: branch: ${{ matrix.branch }} @@ -44,7 +44,7 @@ jobs: uses: ./.github/workflows/integration-tests.yml strategy: matrix: - branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + branch: ${{ fromJSON(vars.DBT_DAILY_CHECKS_BRANCHES) }} with: branch: ${{ matrix.branch }} @@ -53,7 +53,7 @@ jobs: uses: dbt-labs/actions/.github/workflows/build-artifacts.yml@add-hatch-actions strategy: matrix: - branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + branch: ${{ fromJSON(vars.DBT_DAILY_CHECKS_BRANCHES) }} with: branch: ${{ matrix.branch }} diff --git a/.github/workflows/scheduled-test-releases.yml b/.github/workflows/daily-dev-releases.yml similarity index 89% rename from .github/workflows/scheduled-test-releases.yml rename to .github/workflows/daily-dev-releases.yml index 296e2b193..e7c1bddda 100644 --- a/.github/workflows/scheduled-test-releases.yml +++ b/.github/workflows/daily-dev-releases.yml @@ -9,17 +9,17 @@ # Ensure an automated and tested release process for nightly builds # # **when?** -# - scheduled with the repository variable DBT_SCHEDULED_TEST_RELEASES_SCHEDULE +# - daily based on DBT_DAILY_DEV_RELEASE_SCHEDULE # - manually -name: "Scheduled test releases" +name: "Daily test releases" on: schedule: - - cron: ${{ vars.DBT_SCHEDULED_TEST_RELEASES_SCHEDULE }} + - cron: ${{ vars.DBT_DAILY_DEV_RELEASE_SCHEDULE }} workflow_dispatch: permissions: - contents: write # this is the permission that allows creating a new release + contents: write # allows creating a new release defaults: run: diff --git a/.github/workflows/dev-branch-integration-tests.yml b/.github/workflows/dev-branch-integration-tests.yml index 5e413b23c..62a1da210 100644 --- a/.github/workflows/dev-branch-integration-tests.yml +++ b/.github/workflows/dev-branch-integration-tests.yml @@ -56,12 +56,12 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ${{ fromJSON(vars.DBT_TEST_PYTHON_ALL_VERSIONS) }} + python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSIONS) }} include: - os: macos-latest - python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + python-version: ${{ vars.DBT_PYTHON_VERSION }} - os: windows-latest - python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + python-version: ${{ vars.DBT_PYTHON_VERSION }} env: REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} REDSHIFT_TEST_PORT: ${{ vars.REDSHIFT_TEST_PORT }} diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index a5b3358f1..8c1637324 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -19,7 +19,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.ref }} check-command: "hatch run code-quality" - python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + python-version: ${{ vars.DBT_PYTHON_VERSION }} unit-tests: name: "Run unit tests" @@ -27,7 +27,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.ref }} test-command: "hatch run unit-tests:all" - python-versions: ${{ vars.DBT_TEST_PYTHON_ALL_VERSIONS }} + python-versions: ${{ vars.DBT_PYTHON_VERSIONS }} integration-tests: name: "Run unit tests" @@ -35,9 +35,9 @@ jobs: with: ref: ${{ github.event.pull_request.head.ref }} test-command: "hatch run integration-tests:all" - python-versions-ubuntu: ${{ vars.DBT_TEST_PYTHON_ALL_VERSIONS }} - python-versions-macos: ${{ vars.DBT_TEST_PYTHON_VERSION }} - python-versions-windows: ${{ vars.DBT_TEST_PYTHON_VERSION }} + python-versions-ubuntu: ${{ vars.DBT_PYTHON_VERSIONS }} + python-versions-macos: ${{ vars.DBT_PYTHON_VERSION }} + python-versions-windows: ${{ vars.DBT_PYTHON_VERSION }} verify-build-artifacts: name: "Verify build artifacts" diff --git a/.github/workflows/weekly-maintenance.yml b/.github/workflows/weekly-maintenance.yml index c255a0536..7503fd52d 100644 --- a/.github/workflows/weekly-maintenance.yml +++ b/.github/workflows/weekly-maintenance.yml @@ -9,7 +9,7 @@ # - allow automated processes to leave artifacts for debugging and troubleshooting # # **when?** -# - weekly base on DBT_WEEKLY_MAINTENANCE_SCHEDULE +# - weekly based on DBT_WEEKLY_MAINTENANCE_SCHEDULE # - manually # # Runs on Saturday to avoid running at the same time as production release and breaking the release. From d4600b24ca422a6944b61e56b2d08bd37bf11b55 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 21 Apr 2024 22:20:09 -0400 Subject: [PATCH 11/15] typo --- .github/workflows/ci-test-suite.yml | 53 ++++++++++++++++++++++ .github/workflows/daily-checks.yml | 54 +++++------------------ .github/workflows/pull-request-checks.yml | 39 +++++----------- .github/workflows/release.yml | 44 +++--------------- 4 files changed, 81 insertions(+), 109 deletions(-) create mode 100644 .github/workflows/ci-test-suite.yml diff --git a/.github/workflows/ci-test-suite.yml b/.github/workflows/ci-test-suite.yml new file mode 100644 index 000000000..ada117771 --- /dev/null +++ b/.github/workflows/ci-test-suite.yml @@ -0,0 +1,53 @@ +name: "CI test suite" + +on: + workflow_call: + inputs: + ref: + description: "The branch/tag/commit to run integration tests on" + type: string + required: true + archive-name: + description: "Name of the archive for build artifacts, used during a release" + type: string + default: "" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.ref }}" + cancel-in-progress: true + +jobs: + code-quality: + name: "Run code quality" + uses: dbt-labs/actions/.github/workflows/code-quality.yml@add-hatch-actions + with: + ref: ${{ inputs.ref }} + check-command: "hatch run code-quality" + python-version: ${{ vars.DBT_PYTHON_VERSION }} + + unit-tests: + name: "Run unit tests" + uses: dbt-labs/actions/.github/workflows/unit-tests.yml@add-hatch-actions + with: + ref: ${{ inputs.ref }} + test-command: "hatch run unit-tests:all" + python-versions: ${{ vars.DBT_PYTHON_VERSIONS }} + + integration-tests: + name: "Run integration tests" + uses: dbt-labs/actions/.github/workflows/integration-tests.yml@add-hatch-actions + with: + ref: ${{ inputs.ref }} + test-command: "hatch run integration-tests:all" + python-versions-ubuntu: ${{ vars.DBT_PYTHON_VERSIONS }} + python-versions-macos: ${{ vars.DBT_PYTHON_VERSION }} + python-versions-windows: ${{ vars.DBT_PYTHON_VERSION }} + + build-artifacts: + name: "Verify build artifacts" + uses: dbt-labs/actions/.github/workflows/build-release-artifacts.yml@add-hatch-actions + with: + ref: ${{ inputs.ref }} + archive-name: ${{ needs.release-inputs.outputs.archive-name }} diff --git a/.github/workflows/daily-checks.yml b/.github/workflows/daily-checks.yml index 0ee4b5c99..35d39260c 100644 --- a/.github/workflows/daily-checks.yml +++ b/.github/workflows/daily-checks.yml @@ -1,61 +1,29 @@ # **what?** -# The purpose of this workflow is to trigger CI to run for each release -# branch on a regular cadence. If the CI workflow fails for a branch, it -# will post to dev-core-alerts to raise awareness. +# - run scheduled CI on protected branches +# - post a failure message in #dev-adapter-alerts for failures # # **why?** -# Ensures release branches are always shippable and not broken. -# Also, can catch any dependencies shifting beneath us that might -# introduce breaking changes (could also impact Cloud). - +# - ensure release branches are always shippable +# - raise awareness of dependencies shifting beneath us +# # **when?** -# - daily based on DBT_DAILY_CHECKS_SCHEDULE +# - daily at 5:00 AM UTC # - manually name: "Daily checks" on: schedule: - - cron: ${{ vars.DBT_DAILY_CHECKS_SCHEDULE }} + - cron: 0 5 * * * workflow_dispatch: permissions: read-all jobs: - code-quality: - name: "Code quality" - uses: ./.github/workflows/code-quality.yml - strategy: - matrix: - branch: ${{ fromJSON(vars.DBT_DAILY_CHECKS_BRANCHES) }} - with: - branch: ${{ matrix.branch }} - - unit-tests: - name: "Unit tests" - uses: ./.github/workflows/unit-tests.yml - strategy: - matrix: - branch: ${{ fromJSON(vars.DBT_DAILY_CHECKS_BRANCHES) }} - with: - branch: ${{ matrix.branch }} - - integration-tests: - name: "Integration tests" - uses: ./.github/workflows/integration-tests.yml - strategy: - matrix: - branch: ${{ fromJSON(vars.DBT_DAILY_CHECKS_BRANCHES) }} - with: - branch: ${{ matrix.branch }} - - build-artifacts: - name: "Build artifacts" - uses: dbt-labs/actions/.github/workflows/build-artifacts.yml@add-hatch-actions - strategy: - matrix: - branch: ${{ fromJSON(vars.DBT_DAILY_CHECKS_BRANCHES) }} + ci-test-suite: + name: "CI test suite" + uses: ./.github/workflows/ci-test-suite.yml with: - branch: ${{ matrix.branch }} + ref: "main" post-failure: runs-on: ubuntu-latest diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 8c1637324..5ee639352 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -1,3 +1,12 @@ +# **what?** +# - run CI on all PRs against protected branches +# +# **why?** +# - ensure code is verified prior to being merged into a protected branch +# +# **when?** +# - a PR is opened against a protected branch +# - code is pushed to a PR against a protected branch name: "Pull request checks" on: @@ -13,32 +22,8 @@ concurrency: cancel-in-progress: true jobs: - code-quality: - name: "Run code quality" - uses: dbt-labs/actions/.github/workflows/code-quality.yml + ci-test-suite: + name: "CI test suite" + uses: ./.github/workflows/ci-test-suite.yml with: ref: ${{ github.event.pull_request.head.ref }} - check-command: "hatch run code-quality" - python-version: ${{ vars.DBT_PYTHON_VERSION }} - - unit-tests: - name: "Run unit tests" - uses: dbt-labs/actions/.github/workflows/unit-tests.yml - with: - ref: ${{ github.event.pull_request.head.ref }} - test-command: "hatch run unit-tests:all" - python-versions: ${{ vars.DBT_PYTHON_VERSIONS }} - - integration-tests: - name: "Run unit tests" - uses: dbt-labs/actions/.github/workflows/integration-tests.yml - with: - ref: ${{ github.event.pull_request.head.ref }} - test-command: "hatch run integration-tests:all" - python-versions-ubuntu: ${{ vars.DBT_PYTHON_VERSIONS }} - python-versions-macos: ${{ vars.DBT_PYTHON_VERSION }} - python-versions-windows: ${{ vars.DBT_PYTHON_VERSION }} - - verify-build-artifacts: - name: "Verify build artifacts" - uses: dbt-labs/actions/.github/workflows/build-artifacts.yml@add-hatch-actions diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 272bb9c07..768805ac2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -135,56 +135,22 @@ jobs: version: ${{ inputs.version }} secrets: inherit - code-quality: - name: "Run code quality" - needs: - - release-branch - - version-bump - - changelog - uses: dbt-labs/actions/.github/workflows/code-quality.yml@add-hatch-actions - with: - branch: ${{ needs.release-branch.outputs.branch }} - check-command: "hatch run code-quality" - - unit-tests: - name: "Run unit tests" + ci-test-suite: + name: "Run code quality, unit tests, and integration tests, and build artifacts" needs: + - release-inputs - release-branch - version-bump - changelog - uses: dbt-labs/actions/.github/workflows/unit-tests.yml@add-hatch-actions + uses: ./.github/workflows/ci-test-suite.yml with: ref: ${{ needs.release-branch.outputs.branch }} - - integration-tests: - name: "Run unit tests" - needs: - - release-branch - - version-bump - - changelog - uses: dbt-labs/actions/.github/workflows/integration-tests.yml@add-hatch-actions - with: - branch: ${{ needs.release-branch.outputs.branch }} - test-command: "hatch run unit-tests:all" - - build-artifacts: - name: "Build artifacts" - if: ${{ !failure() && !cancelled() }} - needs: - - release-inputs - - release-branch - - code-quality - - unit-tests - - integration-tests - uses: dbt-labs/actions/.github/workflows/build-artifacts.yml@add-hatch-actions - with: - branch: ${{ needs.release-branch.outputs.branch }} archive-name: ${{ needs.release-inputs.outputs.archive-name }} merge-changes: name: "Merge the version bump and changelog updates" needs: - - build-artifacts + - ci-test-suite - release-branch runs-on: ubuntu-latest outputs: From e929fd94f8f27564033467fb44649553f20675f0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 21 Apr 2024 22:24:20 -0400 Subject: [PATCH 12/15] move schedule back into workflow --- .github/workflows/daily-dev-releases.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-dev-releases.yml b/.github/workflows/daily-dev-releases.yml index e7c1bddda..d288532bf 100644 --- a/.github/workflows/daily-dev-releases.yml +++ b/.github/workflows/daily-dev-releases.yml @@ -9,13 +9,13 @@ # Ensure an automated and tested release process for nightly builds # # **when?** -# - daily based on DBT_DAILY_DEV_RELEASE_SCHEDULE +# - daily 9:00 AM UTC # - manually name: "Daily test releases" on: schedule: - - cron: ${{ vars.DBT_DAILY_DEV_RELEASE_SCHEDULE }} + - cron: 0 9 * * * workflow_dispatch: permissions: From fb2b0bae48b285ab9ee936f2816ea33dfc9d1205 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 22 Apr 2024 16:24:59 -0400 Subject: [PATCH 13/15] merge aws iam user configuration into the integration tests workflow --- .../dev-branch-integration-tests.yml | 12 + .github/workflows/integration.yml | 259 ------------------ 2 files changed, 12 insertions(+), 259 deletions(-) delete mode 100644 .github/workflows/integration.yml diff --git a/.github/workflows/dev-branch-integration-tests.yml b/.github/workflows/dev-branch-integration-tests.yml index 62a1da210..ace39b778 100644 --- a/.github/workflows/dev-branch-integration-tests.yml +++ b/.github/workflows/dev-branch-integration-tests.yml @@ -68,6 +68,11 @@ jobs: REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }} REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }} REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_PROFILE }} + REDSHIFT_TEST_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} + REDSHIFT_TEST_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} DBT_TEST_USER_1: ${{ vars.REDSHIFT_TEST_USER_1 }} DBT_TEST_USER_2: ${{ vars.REDSHIFT_TEST_USER_2 }} DBT_TEST_USER_3: ${{ vars.REDSHIFT_TEST_USER_3 }} @@ -104,6 +109,13 @@ jobs: dbt-common-branch: ${{ inputs.dbt-common-branch }} dbt-core-branch: ${{ inputs.dbt-core-branch }} + - name: Create AWS IAM profile + run: | + aws configure set aws_access_key_id $REDSHIFT_TEST_ACCESS_KEY_ID + aws configure set aws_secret_access_key $REDSHIFT_TEST_SECRET_ACCESS_KEY + aws configure set region $REDSHIFT_TEST_REGION + aws configure set output json + - name: "Run integration tests" shell: bash run: hatch run integration-tests:all diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index 3fac0b0db..000000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,259 +0,0 @@ -# **what?** -# This workflow runs all integration tests for supported OS -# and python versions and core adapters. If triggered by PR, -# the workflow will only run tests for adapters related -# to code changes. Use the `test all` and `test ${adapter}` -# label to run all or additional tests. Use `ok to test` -# label to mark PRs from forked repositories that are safe -# to run integration tests for. Requires secrets to run -# against different warehouses. - -# **why?** -# This checks the functionality of dbt from a user's perspective -# and attempts to catch functional regressions. - -# **when?** -# This workflow will run on every push to a protected branch -# and when manually triggered. It will also run for all PRs, including -# PRs from forks. The workflow will be skipped until there is a label -# to mark the PR as safe to run. - -name: Adapter Integration Tests - -on: - # pushes to release branches - push: - branches: - - "main" - - "develop" - - "*.latest" - - "releases/*" - # all PRs, important to note that `pull_request_target` workflows - # will run in the context of the target branch of a PR - pull_request_target: - # manual trigger - workflow_dispatch: - inputs: - dbt-core-branch: - description: "branch of dbt-core to use in dev-requirements.txt" - required: false - type: string - -# explicitly turn off permissions for `GITHUB_TOKEN` -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -# sets default shell to bash, for all operating systems -defaults: - run: - shell: bash - -jobs: - # generate test metadata about what files changed and the testing matrix to use - test-metadata: - # run if not a PR from a forked repository or has a label to mark as safe to test - if: >- - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest - - outputs: - matrix: ${{ steps.generate-matrix.outputs.result }} - - steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Check if relevant files changed - if: github.event_name == 'pull_request_target' - # https://github.com/marketplace/actions/paths-changes-filter - # For each filter, it sets output variable named by the filter to the text: - # 'true' - if any of changed files matches any of filter rules - # 'false' - if none of changed files matches any of filter rules - # also, returns: - # `changes` - JSON array with names of all filters matching any of the changed files - uses: dorny/paths-filter@v2 - id: get-changes - with: - token: ${{ secrets.GITHUB_TOKEN }} - filters: | - redshift: - - 'dbt/**' - - 'tests/**' - - 'dev-requirements.txt' - - - name: Generate integration test matrix - id: generate-matrix - uses: actions/github-script@v6 - env: - CHANGES: ${{ steps.get-changes.outputs.changes }} - with: - script: | - const script = require('./.github/scripts/integration-test-matrix.js') - const matrix = script({ context }) - console.log(matrix) - return matrix - test: - name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} - - # run if not a PR from a forked repository or has a label to mark as safe to test - # also checks that the matrix generated is not empty - if: >- - needs.test-metadata.outputs.matrix && - fromJSON( needs.test-metadata.outputs.matrix ).include[0] && - ( - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - ) - runs-on: ${{ matrix.os }} - - needs: test-metadata - - strategy: - fail-fast: false - max-parallel: 3 - matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} - - env: - TOXENV: integration-${{ matrix.adapter }} - PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" - DBT_INVOCATION_ENV: github-actions - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: datadoghq.com - DD_ENV: ci - DD_SERVICE: ${{ github.event.repository.name }} - - steps: - - name: Check out the repository - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - - # explicity checkout the branch for the PR, - # this is necessary for the `pull_request_target` event - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' - run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} - - - name: Create AWS IAM profile - run: | - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region $AWS_REGION - aws configure set output json - env: - AWS_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} - - - name: Run tox (redshift) - if: matrix.adapter == 'redshift' - env: - REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} - REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} - REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} - REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} - REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} - REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} - REDSHIFT_TEST_IAM_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_PROFILE }} - REDSHIFT_TEST_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} - REDSHIFT_TEST_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} - DBT_TEST_USER_1: dbt_test_user_1 - DBT_TEST_USER_2: dbt_test_user_2 - DBT_TEST_USER_3: dbt_test_user_3 - run: tox -- --ddtrace - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: logs - path: ./logs - - - name: Get current date - if: always() - id: date - run: | - echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv - path: integration_results.csv - - require-label-comment: - runs-on: ubuntu-latest - - needs: test - - permissions: - pull-requests: write - - steps: - - name: Needs permission PR comment - if: >- - needs.test.result == 'skipped' && - github.event_name == 'pull_request_target' && - github.event.pull_request.head.repo.full_name != github.repository - uses: unsplash/comment-on-pr@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - msg: | - "You do not have permissions to run integration tests, @dbt-labs/core "\ - "needs to label this PR with `ok to test` in order to run integration tests!" - check_for_duplicate_msg: true - - post-failure: - runs-on: ubuntu-latest - needs: test - if: ${{ failure() }} - - steps: - - name: Posting scheduled run failures - uses: ravsamhq/notify-slack-action@v2 - if: ${{ github.event_name == 'schedule' }} - with: - notification_title: 'Redshift nightly integration test failed' - status: ${{ job.status }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} From 5b094f1d04efdca99db0f71e99efa0a598b65afc Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 22 Apr 2024 22:11:56 -0400 Subject: [PATCH 14/15] add ci workflows back at the adapter level --- .github/workflows/code-quality.yml | 61 +++++++++++++ .github/workflows/integration-tests.yml | 115 ++++++++++++++++++++++++ .github/workflows/unit-tests.yml | 72 +++++++++++++++ 3 files changed, 248 insertions(+) create mode 100644 .github/workflows/code-quality.yml create mode 100644 .github/workflows/integration-tests.yml create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 000000000..1acd20586 --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,61 @@ +# **what?** +# Run code quality checks, e.g. `black`, `flake8`, `mypy`, via `pre-commit` +# +# **why?** +# Ensure code quality meets dbt Labs standards +# +# **when?** +# - runs on all PRs into protected branches +# - runs indirectly during releases and release tests +# - can be manually run +name: "Code quality" + +on: + workflow_call: + inputs: + ref: + description: "The branch/tag/commit to run code quality on" + type: string + required: true + check-command: + description: "The command to run code quality checks" + type: string + required: true + python-version: + description: "The python version to test against" + type: string + required: true + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.ref }}" + cancel-in-progress: true + +jobs: + code-quality: + name: "Code quality" + runs-on: ubuntu-latest + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.ref }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + + - name: "Setup environment" + uses: ./setup-environment + with: + python-version: ${{ inputs.python-version }} + + - name: "Run code quality" + shell: bash + run: ${{ inputs.check-command }} + + - name: "[INFO] Run code quality" + shell: bash + run: | + echo "::notice title=$TITLE::$MESSAGE" + env: + TITLE: "Run code quality" + MESSAGE: "Code quality checks completed successfully!" diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..3024d3fb3 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,115 @@ +name: "Integration tests" + +on: + workflow_call: + inputs: + ref: + description: "The branch/tag/commit to run integration tests on" + type: string + required: true + test-command: + description: "The command to run unit tests" + type: string + required: true + python-versions-ubuntu: + description: "The python versions to test against for ubuntu" + type: string + required: true + python-versions-macos: + description: "The python version to test against for macos" + type: string + required: true + python-versions-windows: + description: "The python version to test against for windows" + type: string + required: true + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.ref }}" + cancel-in-progress: true + +jobs: + require-approval-on-forks: + # if it's a pull request from a fork that has not been approved yet, don't run tests + if: >- + github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name != github.repository && + !contains(github.event.pull_request.labels.*.name, 'ok to test') + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: "Author requires permission to run tests" + uses: unsplash/comment-on-pr@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + msg: | + "Thanks for your contribution! "\ + "This PR was created from a fork, integration tests need to be approved prior to running. "\ + "@dbt-labs/core will review this PR and approve running integration tests." + check_for_duplicate_msg: true + + integration-tests: + name: "Integration tests" + needs: require-approval-on-forks + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ${{ fromJSON(inputs.python-versions-ubuntu) }} + include: + - os: macos-latest + python-version: ${{ fromJSON(inputs.python-versions-macos) }} + - os: windows-latest + python-version: ${{ fromJSON(inputs.python-versions-windows) }} + env: + DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }} + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: ${{ vars.DD_SITE }} + DD_ENV: ${{ vars.DD_ENV }} + DD_SERVICE: ${{ github.event.repository.name }} + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.ref }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + + - name: "Setup environment" + uses: ./setup-environment@add-hatch-actions + with: + python-version: ${{ matrix.python-version }} + + - name: "Run integration tests" + shell: bash + run: ${{ inputs.test-command }} + + aggregate-results: + name: "Collect integration tests results" + needs: integration-tests + if: always() && !cancelled() + runs-on: ubuntu-latest + steps: + - name: "[INFO] Integration tests - FAILED" + if: failure() + shell: bash + run: | + echo "::notice title=$TITLE::$MESSAGE" + exit 1 + env: + TITLE: "Integration tests - FAILED" + MESSAGE: "Integration tests failed!" + + - name: "[INFO] Integration tests - PASSED" + if: ${{ !failure() }} # includes skipped + shell: bash + run: | + echo "::notice title=$TITLE::$MESSAGE" + env: + TITLE: "Integration tests - PASSED" + MESSAGE: "Integration tests completed successfully!" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 000000000..314cc5345 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,72 @@ +name: "Unit tests" + +on: + workflow_call: + inputs: + ref: + description: "The branch/tag/commit to run integration tests on" + type: string + required: true + test-command: + description: "The command to run unit tests" + type: string + required: true + python-versions: + description: "The python versions to test against" + type: string + required: true + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.ref }}" + cancel-in-progress: true + +jobs: + unit-tests: + name: "Unit tests" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJSON(inputs.python-versions) }} + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.ref }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + + - name: "Setup environment" + uses: ./setup-environment + with: + python-version: ${{ matrix.python-version }} + + - name: "Run unit tests" + shell: bash + run: ${{ inputs.test-command }} + + aggregate-results: + name: "Collect unit tests results" + needs: unit-tests + if: always() && !cancelled() + runs-on: ubuntu-latest + steps: + - name: "[INFO] Unit tests - FAILED" + if: failure() + shell: bash + run: | + echo "::notice title=$TITLE::$MESSAGE" + exit 1 + env: + TITLE: "Unit tests - FAILED" + MESSAGE: "Unit tests completed with failures!" + + - name: "[INFO] Unit tests - PASSED" + if: ${{ !failure() }} # includes skipped + shell: bash + run: | + echo "::notice title=$TITLE::$MESSAGE" + env: + TITLE: "Unit tests - PASSED" + MESSAGE: "Unit tests completed successfully!" From 8d27c1788da5be8aa3f22f3dce331edb7f78c4b4 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 23 Apr 2024 01:13:11 -0400 Subject: [PATCH 15/15] updates refleching changes in actions repo --- .github/workflows/build-artifacts.yml | 44 ++++++ .github/workflows/ci-test-suite.yml | 12 +- .github/workflows/code-quality.yml | 17 +- .github/workflows/daily-dev-releases.yml | 13 +- .../dev-branch-integration-tests.yml | 148 ------------------ .github/workflows/integration-tests.yml | 100 ++++++++---- .github/workflows/release.yml | 48 ++---- .github/workflows/unit-tests.yml | 21 +-- 8 files changed, 152 insertions(+), 251 deletions(-) create mode 100644 .github/workflows/build-artifacts.yml delete mode 100644 .github/workflows/dev-branch-integration-tests.yml diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 000000000..bb4659c8a --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,44 @@ +name: "Build artifacts" + +on: + workflow_call: + inputs: + ref: + description: "The branch/tag/commit to run integration tests on" + type: string + required: true + archive-name: + description: "Name of the archive for build artifacts, used during a release" + type: string + default: "" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.ref }}" + cancel-in-progress: true + +jobs: + build-artifacts: + name: "Build artifacts" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSIONS) }} + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.ref }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + + - name: "Setup environment" + uses: dbt-labs/actions/hatch/environment/create@add-hatch-actions + with: + python-version: ${{ matrix.python-version }} + + - name: "Build artifacts" + uses: dbt-labs/actions/hatch/artifacts/create.yml@add-hatch-actions + with: + archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/ci-test-suite.yml b/.github/workflows/ci-test-suite.yml index ada117771..8fde14f37 100644 --- a/.github/workflows/ci-test-suite.yml +++ b/.github/workflows/ci-test-suite.yml @@ -24,30 +24,22 @@ jobs: uses: dbt-labs/actions/.github/workflows/code-quality.yml@add-hatch-actions with: ref: ${{ inputs.ref }} - check-command: "hatch run code-quality" - python-version: ${{ vars.DBT_PYTHON_VERSION }} unit-tests: name: "Run unit tests" uses: dbt-labs/actions/.github/workflows/unit-tests.yml@add-hatch-actions with: ref: ${{ inputs.ref }} - test-command: "hatch run unit-tests:all" - python-versions: ${{ vars.DBT_PYTHON_VERSIONS }} integration-tests: name: "Run integration tests" uses: dbt-labs/actions/.github/workflows/integration-tests.yml@add-hatch-actions with: ref: ${{ inputs.ref }} - test-command: "hatch run integration-tests:all" - python-versions-ubuntu: ${{ vars.DBT_PYTHON_VERSIONS }} - python-versions-macos: ${{ vars.DBT_PYTHON_VERSION }} - python-versions-windows: ${{ vars.DBT_PYTHON_VERSION }} build-artifacts: name: "Verify build artifacts" - uses: dbt-labs/actions/.github/workflows/build-release-artifacts.yml@add-hatch-actions + uses: dbt-labs/actions/.github/workflows/build-artifacts.yml@add-hatch-actions with: ref: ${{ inputs.ref }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} + archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 1acd20586..7f32602e5 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -17,14 +17,6 @@ on: description: "The branch/tag/commit to run code quality on" type: string required: true - check-command: - description: "The command to run code quality checks" - type: string - required: true - python-version: - description: "The python version to test against" - type: string - required: true permissions: read-all @@ -44,18 +36,17 @@ jobs: persist-credentials: false - name: "Setup environment" - uses: ./setup-environment + uses: dbt-labs/actions/hatch/environment/create@add-hatch-actions with: - python-version: ${{ inputs.python-version }} + python-version: ${{ vars.DBT_PYTHON_VERSION }} - name: "Run code quality" shell: bash - run: ${{ inputs.check-command }} + run: hatch run code-quality - name: "[INFO] Run code quality" shell: bash - run: | - echo "::notice title=$TITLE::$MESSAGE" + run: echo "::notice title=$TITLE::$MESSAGE" env: TITLE: "Run code quality" MESSAGE: "Code quality checks completed successfully!" diff --git a/.github/workflows/daily-dev-releases.yml b/.github/workflows/daily-dev-releases.yml index d288532bf..3c961eb2b 100644 --- a/.github/workflows/daily-dev-releases.yml +++ b/.github/workflows/daily-dev-releases.yml @@ -36,6 +36,11 @@ jobs: - name: "Checkout `${{ github.repository }}`" uses: actions/checkout@v3 + - name: "Setup environment" + uses: dbt-labs/actions/hatch/environment/create@add-hatch-actions + with: + ref: ${{ vars.DBT_PYTHON_VERSION }} + - name: "Set: commit" id: commit run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT @@ -45,7 +50,7 @@ jobs: run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT - name: "Audit Version And Parse Into Parts" - id: semver + id: version uses: dbt-labs/actions/parse-semver@v1.1.0 with: version: ${{ steps.current-version.outputs.version }} @@ -53,9 +58,9 @@ jobs: - name: "Set: release version" id: release-version run: | - version="${{ steps.semver.outputs.major }}" - version+=".${{ steps.semver.outputs.minor }}" - version+=".$(${{ steps.semver.outputs.patch }}+1)" + version="${{ steps.version.outputs.major }}" + version+=".${{ steps.version.outputs.minor }}" + version+=".$(${{ steps.version.outputs.patch }}+1)" version+="dev$(date +'%m%d%Y')" echo "version=$version" >> $GITHUB_OUTPUT diff --git a/.github/workflows/dev-branch-integration-tests.yml b/.github/workflows/dev-branch-integration-tests.yml deleted file mode 100644 index ace39b778..000000000 --- a/.github/workflows/dev-branch-integration-tests.yml +++ /dev/null @@ -1,148 +0,0 @@ -name: "Dev branch integration tests" - -on: - workflow_dispatch: - inputs: - branch: - description: "The branch/tag to run integration tests on" - type: string - default: "main" - dbt-adapters-branch: - description: "The branch/tag of `dbt-adapters` to use" - type: string - default: "main" - dbt-common-branch: - description: "The branch/tag of `dbt-common` to use" - type: string - default: "main" - dbt-core-branch: - description: "The branch/tag of `dbt-core` to use" - type: string - default: "main" - -permissions: read-all - -concurrency: - group: "${{ github.workflow }}-${{ github.event_name }}-${{ github.sha }}" - cancel-in-progress: true - -jobs: - require-approval-on-forks: - # if it's a pull request from a fork that has not been approved yet, don't run tests - if: >- - github.event_name == 'pull_request_target' && - github.event.pull_request.head.repo.full_name != github.repository && - !contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest - permissions: - pull-requests: write - steps: - - name: "Author requires permission to run tests" - uses: unsplash/comment-on-pr@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - msg: | - "Thanks for your contribution! "\ - "This PR was created from a fork, integration tests need to be approved prior to running. "\ - "@dbt-labs/core will review this PR and approve running integration tests." - check_for_duplicate_msg: true - - integration-tests: - name: "Integration tests" - needs: require-approval-on-forks - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSIONS) }} - include: - - os: macos-latest - python-version: ${{ vars.DBT_PYTHON_VERSION }} - - os: windows-latest - python-version: ${{ vars.DBT_PYTHON_VERSION }} - env: - REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} - REDSHIFT_TEST_PORT: ${{ vars.REDSHIFT_TEST_PORT }} - REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }} - REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }} - REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} - REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} - REDSHIFT_TEST_IAM_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_PROFILE }} - REDSHIFT_TEST_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} - REDSHIFT_TEST_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} - REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} - DBT_TEST_USER_1: ${{ vars.REDSHIFT_TEST_USER_1 }} - DBT_TEST_USER_2: ${{ vars.REDSHIFT_TEST_USER_2 }} - DBT_TEST_USER_3: ${{ vars.REDSHIFT_TEST_USER_3 }} - DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }} - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: ${{ vars.DD_SITE }} - DD_ENV: ${{ vars.DD_ENV }} - DD_SERVICE: ${{ github.event.repository.name }} - steps: - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - persist-credentials: false - - - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: "Setup environment" - uses: dbt-labs/actions/setup-environment@add-hatch-actions - with: - python-version: ${{ matrix.python-version }} - - - name: "Update development branches" - if: ${{ contains(github.event_name, 'workflow_') }} - uses: ./.github/actions/update-dev-branches - with: - dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} - dbt-common-branch: ${{ inputs.dbt-common-branch }} - dbt-core-branch: ${{ inputs.dbt-core-branch }} - - - name: Create AWS IAM profile - run: | - aws configure set aws_access_key_id $REDSHIFT_TEST_ACCESS_KEY_ID - aws configure set aws_secret_access_key $REDSHIFT_TEST_SECRET_ACCESS_KEY - aws configure set region $REDSHIFT_TEST_REGION - aws configure set output json - - - name: "Run integration tests" - shell: bash - run: hatch run integration-tests:all - - aggregate-results: - name: "Collect integration tests results" - needs: integration-tests - if: always() && !cancelled() - runs-on: ubuntu-latest - steps: - - name: "[INFO] Integration tests - FAILED" - if: failure() - shell: bash - run: | - echo "::notice title=[$WORKFLOW]: $TITLE::$MESSAGE" - exit 1 - env: - WORKFLOW: ${{ env.NOTIFICATION_PREFIX }} - TITLE: "Integration tests - FAILED" - MESSAGE: "Integration tests failed!" - - - name: "[INFO] Integration tests - PASSED" - if: ${{ !failure() }} # includes skipped - shell: bash - run: | - echo "::notice title=[$WORKFLOW]: $TITLE::$MESSAGE" - env: - WORKFLOW: ${{ env.NOTIFICATION_PREFIX }} - TITLE: "Integration tests - PASSED" - MESSAGE: "Integration tests completed successfully!" diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3024d3fb3..e92168069 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -7,22 +7,36 @@ on: description: "The branch/tag/commit to run integration tests on" type: string required: true - test-command: - description: "The command to run unit tests" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" type: string - required: true - python-versions-ubuntu: - description: "The python versions to test against for ubuntu" + default: "" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" type: string - required: true - python-versions-macos: - description: "The python version to test against for macos" + default: "" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" type: string - required: true - python-versions-windows: - description: "The python version to test against for windows" + default: "" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" type: string - required: true + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "" permissions: read-all @@ -52,6 +66,10 @@ jobs: "@dbt-labs/core will review this PR and approve running integration tests." check_for_duplicate_msg: true + - name: "Exit" + shell: bash + run: exit 0 + integration-tests: name: "Integration tests" needs: require-approval-on-forks @@ -60,19 +78,12 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ${{ fromJSON(inputs.python-versions-ubuntu) }} + python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSIONS) }} include: - os: macos-latest - python-version: ${{ fromJSON(inputs.python-versions-macos) }} + python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSION) }} - os: windows-latest - python-version: ${{ fromJSON(inputs.python-versions-windows) }} - env: - DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }} - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: ${{ vars.DD_SITE }} - DD_ENV: ${{ vars.DD_ENV }} - DD_SERVICE: ${{ github.event.repository.name }} + python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSION) }} steps: - name: "Check out ${{ github.repository }}@${{ inputs.ref }}" uses: actions/checkout@v4 @@ -81,13 +92,47 @@ jobs: persist-credentials: false - name: "Setup environment" - uses: ./setup-environment@add-hatch-actions + uses: dbt-labs/actions/hatch/environment/create@add-hatch-actions with: python-version: ${{ matrix.python-version }} + - name: "Update development branches" + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + dbt-core-branch: ${{ inputs.dbt-core-branch }} + + - name: "Create AWS IAM profile" + run: | + aws configure set aws_access_key_id $REDSHIFT_TEST_ACCESS_KEY_ID + aws configure set aws_secret_access_key $REDSHIFT_TEST_SECRET_ACCESS_KEY + aws configure set region $REDSHIFT_TEST_REGION + aws configure set output json + - name: "Run integration tests" shell: bash - run: ${{ inputs.test-command }} + run: hatch run integration-tests:all + env: + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_PORT: ${{ vars.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_PROFILE }} + REDSHIFT_TEST_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} + REDSHIFT_TEST_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + DBT_TEST_USER_1: ${{ vars.REDSHIFT_TEST_USER_1 }} + DBT_TEST_USER_2: ${{ vars.REDSHIFT_TEST_USER_2 }} + DBT_TEST_USER_3: ${{ vars.REDSHIFT_TEST_USER_3 }} + DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }} + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: ${{ vars.DD_SITE }} + DD_ENV: ${{ vars.DD_ENV }} + DD_SERVICE: ${{ github.event.repository.name }} aggregate-results: name: "Collect integration tests results" @@ -98,9 +143,7 @@ jobs: - name: "[INFO] Integration tests - FAILED" if: failure() shell: bash - run: | - echo "::notice title=$TITLE::$MESSAGE" - exit 1 + run: echo "::notice title=$TITLE::$MESSAGE" && exit 1 env: TITLE: "Integration tests - FAILED" MESSAGE: "Integration tests failed!" @@ -108,8 +151,7 @@ jobs: - name: "[INFO] Integration tests - PASSED" if: ${{ !failure() }} # includes skipped shell: bash - run: | - echo "::notice title=$TITLE::$MESSAGE" + run: echo "::notice title=$TITLE::$MESSAGE" env: TITLE: "Integration tests - PASSED" MESSAGE: "Integration tests completed successfully!" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 768805ac2..1305d9c80 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,37 +111,27 @@ jobs: - name: "Create release branch" id: branch - uses: dbt-labs/actions/create-temp-branch@add-hatch-actions + uses: dbt-labs/actions/github/branch/create@add-hatch-actions with: - branch-stub: "release/${{ inputs.version }}/" + branch-group: "release/${{ inputs.version }}" - bump-version: - name: "Bump version" - needs: release-branch - uses: dbt-labs/actions/.github/workflows/build-bump-version.yml@add-hatch-actions - with: - branch: ${{ needs.release-branch.outputs.branch }} - version: ${{ inputs.version }} - secrets: inherit + - name: "Bump version" + uses: dbt-labs/actions/hatch/version/build.yml@add-hatch-actions + with: + branch: ${{ needs.release-branch.outputs.branch }} + version: ${{ inputs.version }} - changelog: - name: "Generate a new changelog" - needs: - - release-branch - - bump-version - uses: dbt-labs/actions/.github/workflows/build-changelog.yml@add-hatch-actions - with: - branch: ${{ needs.release-branch.outputs.branch }} - version: ${{ inputs.version }} - secrets: inherit + - name: "Generate a new changelog" + uses: dbt-labs/actions/changie/changelog/create.yml@add-hatch-actions + with: + branch: ${{ needs.release-branch.outputs.branch }} + version: ${{ inputs.version }} ci-test-suite: name: "Run code quality, unit tests, and integration tests, and build artifacts" needs: - release-inputs - release-branch - - version-bump - - changelog uses: ./.github/workflows/ci-test-suite.yml with: ref: ${{ needs.release-branch.outputs.branch }} @@ -164,7 +154,7 @@ jobs: - name: "Merge `${{ needs.release-branch.outputs.branch }}` into `${{ inputs.branch }}`" if: ${{ inputs.deploy-environment == 'prod' }} id: merge - uses: dbt-labs/actions/github-merge@add-hatch-actions + uses: dbt-labs/actions/github/branch/merge@add-hatch-actions with: source-branch: ${{ needs.release-branch.outputs.branch }} target-branch: ${{ inputs.branch }} @@ -195,9 +185,7 @@ jobs: publish-pypi: name: "Publish to PyPI" - needs: - - release-inputs - - merge-changes + needs: merge-changes if: ${{ !failure() && !cancelled() }} environment: name: ${{ inputs.deploy-environment }} @@ -205,17 +193,15 @@ jobs: runs-on: ubuntu-latest steps: - name: "Publish to PyPI" - uses: .github/actions/publish-pypi + uses: dbt-labs/actions/pypi/release/create with: - repository-url: ${{ vars.PYPI_REPOSITORY_URL }} archive-name: ${{ inputs.archive-name }} + index-url: ${{ vars.PYPI_REPOSITORY_URL }} publish-pypi-internal: name: "Publish to internal PyPI" if: ${{ !failure() && !cancelled() }} - needs: - - release-inputs - - merge-changes + needs: merge-changes uses: dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main with: version_number: "${{ inputs.version }}" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 314cc5345..00ae81917 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -7,14 +7,6 @@ on: description: "The branch/tag/commit to run integration tests on" type: string required: true - test-command: - description: "The command to run unit tests" - type: string - required: true - python-versions: - description: "The python versions to test against" - type: string - required: true permissions: read-all @@ -29,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ${{ fromJSON(inputs.python-versions) }} + python-version: ${{ fromJSON(vars.DBT_PYTHON_VERSIONS) }} steps: - name: "Check out ${{ github.repository }}@${{ inputs.ref }}" uses: actions/checkout@v4 @@ -38,13 +30,13 @@ jobs: persist-credentials: false - name: "Setup environment" - uses: ./setup-environment + uses: dbt-labs/actions/hatch/environment/create@add-hatch-actions with: python-version: ${{ matrix.python-version }} - name: "Run unit tests" shell: bash - run: ${{ inputs.test-command }} + run: hatch run unit-tests:all aggregate-results: name: "Collect unit tests results" @@ -55,9 +47,7 @@ jobs: - name: "[INFO] Unit tests - FAILED" if: failure() shell: bash - run: | - echo "::notice title=$TITLE::$MESSAGE" - exit 1 + run: echo "::notice title=$TITLE::$MESSAGE" && exit 1 env: TITLE: "Unit tests - FAILED" MESSAGE: "Unit tests completed with failures!" @@ -65,8 +55,7 @@ jobs: - name: "[INFO] Unit tests - PASSED" if: ${{ !failure() }} # includes skipped shell: bash - run: | - echo "::notice title=$TITLE::$MESSAGE" + run: echo "::notice title=$TITLE::$MESSAGE" env: TITLE: "Unit tests - PASSED" MESSAGE: "Unit tests completed successfully!"