From 04205d5e28b8bbc70fe4be702384d21b50ac8aee Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 5 Apr 2024 18:02:10 -0400 Subject: [PATCH 01/16] add docker release to release pipeline --- .github/workflows/release.yml | 25 +++++++++++++ docker/Dockerfile | 33 +++++++++++++++++ docker/README.md | 70 +++++++++++++++++++++++++++++++++++ docker/test.sh | 22 +++++++++++ 4 files changed, 150 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100755 docker/test.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f68eb3320..b3f34851d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -208,6 +208,31 @@ jobs: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + is-docker-release: + name: "Is Docker Release" + runs-on: ubuntu-latest + needs: [pypi-release] + outputs: + is-docker-release: ${{ steps.semver.outputs.is-pre-release == 0 }} + steps: + - name: "Audit version to determine if it is a pre-release" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version_number }} + + docker-release: + name: "Docker Release" + needs: [is-docker-release] + if: ${{ needs.is-docker-release.outputs.is-docker-release }} + permissions: + packages: write + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + with: + package: "dbt-redshift" + version_number: ${{ inputs.version_number }} + test_run: ${{ inputs.test_run }} + slack-notification: name: Slack Notification if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..dd1d2c6da --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,33 @@ +ARG build_for=linux/amd64 + +FROM --platform=$build_for python:3.10.7-slim-bullseye as base + +# ref is updated automatically every final release via bumpversion +ARG dbt_redshift_ref=dbt-redshift@v1.7.10 + +RUN apt-get update \ + && apt-get dist-upgrade -y \ + && apt-get install -y --no-install-recommends \ + git \ + ssh-client \ + software-properties-common \ + make \ + build-essential \ + ca-certificates \ + libpq-dev \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +ENV PYTHONIOENCODING=utf-8 +ENV LANG=C.UTF-8 + +RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir + +WORKDIR /usr/app/dbt/ +ENTRYPOINT ["dbt"] + +FROM base as dbt-redshift +RUN python -m pip install --no-cache-dir "dbt-redshift @ git+https://github.com/dbt-labs/${dbt_redshift_ref}" diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..714d493c1 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,70 @@ +# Docker for dbt +This docker file is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry. + + +## Building an image: +This Dockerfile can create images for the following target: `dbt-redshift` + +In order to build a new image, run the following docker command. +``` +docker build --tag --target dbt-redshift +``` +--- +> **Note:** Docker must be configured to use [BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/) in order for images to build properly! + +--- + +By default the images will be populated with the most recent release of `dbt-redshift`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +``` +docker build --tag \ + --target dbt-redshift \ + --build-arg dbt_redshift_ref= \ + +``` + +### Examples: +To build an image named "my-dbt" that supports redshift using the latest releases: +``` +cd dbt-core/docker +docker build --tag my-dbt --target dbt-redshift . +``` + +To build an image named "my-other-dbt" that supports redshift using the adapter version 1.0.0b1: +``` +cd dbt-core/docker +docker build \ + --tag my-other-dbt \ + --target dbt-redshift \ + --build-arg dbt_redshift_ref=dbt-redshift@v1.0.0b1 \ + . +``` + +## Special cases +There are a few special cases worth noting: + +* If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg: +``` +docker build --tag my_dbt \ + --target dbt-redshift \ + --build-arg build_for=linux/arm64 \ + +``` + +Supported architectures can be found in the python docker [dockerhub page](https://hub.docker.com/_/python). + +## Running an image in a container: +The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: +``` +docker run \ + --network=host \ + --mount type=bind,source=path/to/project,target=/usr/app \ + --mount type=bind,source=path/to/profiles.yml,target=/root/.dbt/profiles.yml \ + my-dbt \ + ls +``` +--- +**Notes:** +* Bind-mount sources _must_ be an absolute path +* You may need to make adjustments to the docker networking setting depending on the specifics of your data warehouse/database host. + +--- diff --git a/docker/test.sh b/docker/test.sh new file mode 100755 index 000000000..e3e62bd0c --- /dev/null +++ b/docker/test.sh @@ -0,0 +1,22 @@ +# - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` +# TODO: create a real test suite + +clear \ +&& echo "\n\n"\ +"########################################\n"\ +"##### Testing dbt-redshift latest #####\n"\ +"########################################\n"\ +&& docker build --tag dbt-redshift \ + --target dbt-redshift \ + docker \ +&& docker run dbt-redshift --version \ +\ +&& echo "\n\n"\ +"#########################################\n"\ +"##### Testing dbt-redshift-1.0.0b1 #####\n"\ +"#########################################\n"\ +&& docker build --tag dbt-redshift-1.0.0b1 \ + --target dbt-redshift \ + --build-arg dbt_redshift_ref=dbt-redshift@v1.0.0b1 \ + docker \ +&& docker run dbt-redshift-1.0.0b1 --version From 23772b237684773a5e31aebfcaa01101f36665af Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 5 Apr 2024 18:07:49 -0400 Subject: [PATCH 02/16] changelog --- .changes/unreleased/Under the Hood-20240405-180739.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20240405-180739.yaml diff --git a/.changes/unreleased/Under the Hood-20240405-180739.yaml b/.changes/unreleased/Under the Hood-20240405-180739.yaml new file mode 100644 index 000000000..9f5d45b37 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240405-180739.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Automate the Docker release as part of the PyPI release +time: 2024-04-05T18:07:39.492816-04:00 +custom: + Author: mikealfare + Issue: "758" From 34e4157c9bdb43155e74aa00d77dac2f2b9f06c1 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 5 Apr 2024 18:58:34 -0400 Subject: [PATCH 03/16] changelog --- docker/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/README.md b/docker/README.md index 714d493c1..c1e43b35e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,7 +6,7 @@ This docker file is suitable for building dbt Docker images locally or using wit This Dockerfile can create images for the following target: `dbt-redshift` In order to build a new image, run the following docker command. -``` +```shell docker build --tag --target dbt-redshift ``` --- @@ -15,7 +15,7 @@ docker build --tag --target dbt-redshift --- By default the images will be populated with the most recent release of `dbt-redshift`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag: -``` +```shell docker build --tag \ --target dbt-redshift \ --build-arg dbt_redshift_ref= \ @@ -24,13 +24,13 @@ docker build --tag \ ### Examples: To build an image named "my-dbt" that supports redshift using the latest releases: -``` +```shell cd dbt-core/docker docker build --tag my-dbt --target dbt-redshift . ``` To build an image named "my-other-dbt" that supports redshift using the adapter version 1.0.0b1: -``` +```shell cd dbt-core/docker docker build \ --tag my-other-dbt \ @@ -43,7 +43,7 @@ docker build \ There are a few special cases worth noting: * If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg: -``` +```shell docker build --tag my_dbt \ --target dbt-redshift \ --build-arg build_for=linux/arm64 \ @@ -54,7 +54,7 @@ Supported architectures can be found in the python docker [dockerhub page](https ## Running an image in a container: The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: -``` +```shell docker run \ --network=host \ --mount type=bind,source=path/to/project,target=/usr/app \ From d183cf0d5c3285292abaeab2cf5c9618cdbb1f4f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 10 Apr 2024 18:24:47 -0400 Subject: [PATCH 04/16] add docker to dependabot --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2a6f34492..285785506 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,3 +6,8 @@ updates: schedule: interval: "daily" rebase-strategy: "disabled" + - package-ecosystem: "docker" + directory: "/docker" + schedule: + interval: "weekly" + rebase-strategy: "disabled" From b4b3f0109f7a96284dc5566100d09c7ffa37b5b0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 22:32:14 -0400 Subject: [PATCH 05/16] update to align with other adapters --- .github/workflows/release.yml | 46 +++++--------------------------- Makefile | 10 +++++++ docker/Dockerfile | 34 +++++++++++++----------- docker/README.md | 24 +++++------------ docker/dev.Dockerfile | 50 +++++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 72 deletions(-) create mode 100644 docker/dev.Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b3f34851d..08b0fcff7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,8 @@ # # **when?** # This workflow can be run manually on demand or can be called by other workflows -name: Release to GitHub and PyPI +name: "Release to GitHub, PyPI, and Docker" +run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI, and Docker" on: workflow_dispatch: @@ -131,9 +132,7 @@ jobs: bump-version-generate-changelog: name: Bump package version, Generate changelog - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main - with: sha: ${{ inputs.sha }} version_number: ${{ inputs.version_number }} @@ -141,17 +140,13 @@ jobs: env_setup_script_path: ${{ inputs.env_setup_script_path }} test_run: ${{ inputs.test_run }} nightly_release: ${{ inputs.nightly_release }} - secrets: inherit log-outputs-bump-version-generate-changelog: name: "[Log output] Bump package version, Generate changelog" if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog] - runs-on: ubuntu-latest - steps: - name: Print variables run: | @@ -162,9 +157,7 @@ jobs: name: Build, Test, Package if: ${{ !failure() && !cancelled() }} needs: [bump-version-generate-changelog] - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} @@ -174,7 +167,6 @@ jobs: package_test_command: ${{ inputs.package_test_command }} test_run: ${{ inputs.test_run }} nightly_release: ${{ inputs.nightly_release }} - secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -182,11 +174,8 @@ jobs: github-release: name: GitHub Release if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main - with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} @@ -195,59 +184,38 @@ jobs: pypi-release: name: PyPI Release - needs: [github-release] - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - with: version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} - secrets: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - is-docker-release: - name: "Is Docker Release" - runs-on: ubuntu-latest - needs: [pypi-release] - outputs: - is-docker-release: ${{ steps.semver.outputs.is-pre-release == 0 }} - steps: - - name: "Audit version to determine if it is a pre-release" - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ inputs.version_number }} - docker-release: name: "Docker Release" - needs: [is-docker-release] - if: ${{ needs.is-docker-release.outputs.is-docker-release }} + if: ${{ !failure() && !cancelled() }} + needs: [bump-version-generate-changelog, build-test-package, github-release] permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release with: - package: "dbt-redshift" + package: ${{ github.event.repository.name }} version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} slack-notification: name: Slack Notification if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} - needs: [ - bump-version-generate-changelog, - build-test-package, github-release, pypi-release, + docker-release, ] - uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main with: status: "failure" - secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/Makefile b/Makefile index efd23b806..f32c3ba8f 100644 --- a/Makefile +++ b/Makefile @@ -65,3 +65,13 @@ help: ## Show this help message. @echo @echo 'targets:' @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + + +.PHONY: docker-dev +docker-dev: + docker build -f docker/dev.Dockerfile -t dbt-redshift-dev . + docker run --rm -it --name dbt-redshift-dev -v $(shell pwd):/opt/code dbt-redshift-dev + +.PHONY: docker-prod +docker-prod: + docker build -f docker/Dockerfile -t dbt-redshift . diff --git a/docker/Dockerfile b/docker/Dockerfile index dd1d2c6da..4b67efb72 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,20 +1,18 @@ -ARG build_for=linux/amd64 +# this image gets published to GHCR for production use +ARG py_version=3.10.7 -FROM --platform=$build_for python:3.10.7-slim-bullseye as base - -# ref is updated automatically every final release via bumpversion -ARG dbt_redshift_ref=dbt-redshift@v1.7.10 +FROM python:$py_version-slim-bullseye as base RUN apt-get update \ && apt-get dist-upgrade -y \ && apt-get install -y --no-install-recommends \ - git \ - ssh-client \ - software-properties-common \ - make \ - build-essential \ - ca-certificates \ - libpq-dev \ + build-essential=12.9 \ + ca-certificates=20210119 \ + git=1:2.30.2-1+deb11u2 \ + libpq-dev=13.14-0+deb11u1 \ + make=4.3-4.1 \ + openssh-client=1:8.4p1-5+deb11u3 \ + software-properties-common=0.96.20.2-2.1 \ && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ @@ -24,10 +22,16 @@ RUN apt-get update \ ENV PYTHONIOENCODING=utf-8 ENV LANG=C.UTF-8 -RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir +RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir + + +FROM base as dbt-redshift + +ARG commit_ref=main + +HEALTHCHECK CMD dbt --version || exit 1 WORKDIR /usr/app/dbt/ ENTRYPOINT ["dbt"] -FROM base as dbt-redshift -RUN python -m pip install --no-cache-dir "dbt-redshift @ git+https://github.com/dbt-labs/${dbt_redshift_ref}" +RUN python -m pip install --no-cache-dir "dbt-bigquery @ git+https://github.com/dbt-labs/dbt-redshift@${commit_ref}" diff --git a/docker/README.md b/docker/README.md index c1e43b35e..5be9e56ef 100644 --- a/docker/README.md +++ b/docker/README.md @@ -14,44 +14,32 @@ docker build --tag --target dbt-redshift --- -By default the images will be populated with the most recent release of `dbt-redshift`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +By default the image will be populated with the latest version of `dbt-redshift` on `main`. +If you need to use a different version you can specify it by git ref using the `--build-arg` flag: ```shell docker build --tag \ --target dbt-redshift \ - --build-arg dbt_redshift_ref= \ + --build-arg commit_ref= \ ``` ### Examples: -To build an image named "my-dbt" that supports redshift using the latest releases: +To build an image named "my-dbt" that supports Snowflake using the latest releases: ```shell cd dbt-core/docker docker build --tag my-dbt --target dbt-redshift . ``` -To build an image named "my-other-dbt" that supports redshift using the adapter version 1.0.0b1: +To build an image named "my-other-dbt" that supports Snowflake using the adapter version 1.0.0b1: ```shell cd dbt-core/docker docker build \ --tag my-other-dbt \ --target dbt-redshift \ - --build-arg dbt_redshift_ref=dbt-redshift@v1.0.0b1 \ + --build-arg commit_ref=v1.0.0b1 \ . ``` -## Special cases -There are a few special cases worth noting: - -* If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg: -```shell -docker build --tag my_dbt \ - --target dbt-redshift \ - --build-arg build_for=linux/arm64 \ - -``` - -Supported architectures can be found in the python docker [dockerhub page](https://hub.docker.com/_/python). - ## Running an image in a container: The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: ```shell diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile new file mode 100644 index 000000000..aac016320 --- /dev/null +++ b/docker/dev.Dockerfile @@ -0,0 +1,50 @@ +# this image does not get published, it is intended for local development only, see `Makefile` for usage +FROM ubuntu:22.04 as base + +# prevent python installation from asking for time zone region +ARG DEBIAN_FRONTEND=noninteractive + +# add python repository +RUN apt-get update \ + && apt-get install -y software-properties-common=0.99.22.9 \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# install python +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9ubuntu3 \ + git-all=1:2.34.1-1ubuntu1.10 \ + python3.8=3.8.19-1+jammy1 \ + python3.8-dev=3.8.19-1+jammy1 \ + python3.8-distutils=3.8.19-1+jammy1 \ + python3.8-venv=3.8.19-1+jammy1 \ + python3-pip=22.0.2+dfsg-1ubuntu0.4 \ + python3-wheel=0.37.1-2ubuntu0.22.04.1 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# update the default system interpreter to the newly installed version +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 + + +FROM base as dbt-redshift-dev + +HEALTHCHECK CMD python3 --version || exit 1 + +# send stdout/stderr to terminal +ENV PYTHONUNBUFFERED=1 + +# setup mount for local code +WORKDIR /opt/code +VOLUME /opt/code + +# create a virtual environment +RUN python3 -m venv /opt/venv From 7b31d7dbbfe4a98b29f3861fbc6e8f00625f2ed5 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 22:36:35 -0400 Subject: [PATCH 06/16] update build arg in test --- docker/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test.sh b/docker/test.sh index e3e62bd0c..99be17c76 100755 --- a/docker/test.sh +++ b/docker/test.sh @@ -17,6 +17,6 @@ clear \ "#########################################\n"\ && docker build --tag dbt-redshift-1.0.0b1 \ --target dbt-redshift \ - --build-arg dbt_redshift_ref=dbt-redshift@v1.0.0b1 \ + --build-arg commit_ref=v1.0.0b1 \ docker \ && docker run dbt-redshift-1.0.0b1 --version From 922c2af6cce8f783a8b8cfab5e55f83c500bdb0a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 10:27:46 -0400 Subject: [PATCH 07/16] removed defaulted input for docker package --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 08b0fcff7..6e0d885b5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -201,7 +201,6 @@ jobs: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release with: - package: ${{ github.event.repository.name }} version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} From a9c4bdb3b3a7b499c3309e10d2a142520b974cd9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 17:28:10 -0400 Subject: [PATCH 08/16] point back to main --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6e0d885b5..961c0c092 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -199,7 +199,7 @@ jobs: needs: [bump-version-generate-changelog, build-test-package, github-release] permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main with: version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} From b380cf30b08468d4c8f0ee57e39083ddd91fa953 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 17:34:22 -0400 Subject: [PATCH 09/16] remove changie entry --- .changes/unreleased/Under the Hood-20240405-180739.yaml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .changes/unreleased/Under the Hood-20240405-180739.yaml diff --git a/.changes/unreleased/Under the Hood-20240405-180739.yaml b/.changes/unreleased/Under the Hood-20240405-180739.yaml deleted file mode 100644 index 9f5d45b37..000000000 --- a/.changes/unreleased/Under the Hood-20240405-180739.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Automate the Docker release as part of the PyPI release -time: 2024-04-05T18:07:39.492816-04:00 -custom: - Author: mikealfare - Issue: "758" From 9aa1be9f0b6a58aae9749d2b28c4ba2e44e6cbf9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 21:28:49 -0400 Subject: [PATCH 10/16] fix repo reference --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4b67efb72..e1010c7c1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -34,4 +34,4 @@ HEALTHCHECK CMD dbt --version || exit 1 WORKDIR /usr/app/dbt/ ENTRYPOINT ["dbt"] -RUN python -m pip install --no-cache-dir "dbt-bigquery @ git+https://github.com/dbt-labs/dbt-redshift@${commit_ref}" +RUN python -m pip install --no-cache-dir "dbt-redshift @ git+https://github.com/dbt-labs/dbt-redshift@${commit_ref}" From 44e48b107da57d87a1f11bf72889354c0c03fbaf Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 17 Apr 2024 08:06:00 -0500 Subject: [PATCH 11/16] add permissions for nightly test release --- .github/workflows/nightly-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 46db5b749..3c36361d5 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -20,6 +20,7 @@ on: permissions: contents: write # this is the permission that allows creating a new release + packages: write # this is the permission that allows Docker release defaults: run: From da5acc576995150c7c6287b83fd24bce77e46677 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 17 Apr 2024 08:06:21 -0500 Subject: [PATCH 12/16] account for draft releases not havign docker, allow docker only release --- .github/workflows/release.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 961c0c092..be093d975 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,6 +61,11 @@ on: type: boolean default: false required: false + only_docker: + description: "Only release Docker image, skip GitHub & PyPI" + type: boolean + default: false + required: false workflow_call: inputs: sha: @@ -144,7 +149,7 @@ jobs: log-outputs-bump-version-generate-changelog: name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [bump-version-generate-changelog] runs-on: ubuntu-latest steps: @@ -155,7 +160,7 @@ jobs: build-test-package: name: Build, Test, Package - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [bump-version-generate-changelog] uses: dbt-labs/dbt-release/.github/workflows/build.yml@main with: @@ -173,7 +178,7 @@ jobs: github-release: name: GitHub Release - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [bump-version-generate-changelog, build-test-package] uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main with: @@ -184,6 +189,7 @@ jobs: pypi-release: name: PyPI Release + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [github-release] uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main with: @@ -195,7 +201,10 @@ jobs: docker-release: name: "Docker Release" - if: ${{ !failure() && !cancelled() }} + # We cannot release to docker on a test run because it uses the tag in GitHub as + # what we need to release but draft releases don't actually tag the commit so it + # finds nothing to release + if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }} needs: [bump-version-generate-changelog, build-test-package, github-release] permissions: packages: write From 184fab923b0fba6a19876e5a3310d16d9c96f6be Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 26 Apr 2024 15:57:12 -0400 Subject: [PATCH 13/16] Update docker/Dockerfile Co-authored-by: Emily Rockman --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e1010c7c1..5914e6396 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ # this image gets published to GHCR for production use -ARG py_version=3.10.7 +ARG py_version=3.11.2 FROM python:$py_version-slim-bullseye as base From 0145ba5d6b87b88ed1823bfca173a1d8f1d8f98d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sun, 28 Apr 2024 12:16:46 -0400 Subject: [PATCH 14/16] simplify test script --- docker/test.sh | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/docker/test.sh b/docker/test.sh index 99be17c76..74261233a 100755 --- a/docker/test.sh +++ b/docker/test.sh @@ -1,22 +1,19 @@ # - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` # TODO: create a real test suite +set -e -clear \ -&& echo "\n\n"\ -"########################################\n"\ -"##### Testing dbt-redshift latest #####\n"\ -"########################################\n"\ -&& docker build --tag dbt-redshift \ - --target dbt-redshift \ - docker \ -&& docker run dbt-redshift --version \ -\ -&& echo "\n\n"\ -"#########################################\n"\ -"##### Testing dbt-redshift-1.0.0b1 #####\n"\ -"#########################################\n"\ -&& docker build --tag dbt-redshift-1.0.0b1 \ - --target dbt-redshift \ - --build-arg commit_ref=v1.0.0b1 \ - docker \ -&& docker run dbt-redshift-1.0.0b1 --version +echo "\n\n" +echo "#######################################\n" +echo "##### Testing dbt-redshift latest #####\n" +echo "#######################################\n" + +docker build --tag dbt-redshift --target dbt-redshift docker +docker run dbt-redshift --version + +echo "\n\n" +echo "########################################\n" +echo "##### Testing dbt-redshift-1.0.0b1 #####\n" +echo "########################################\n" + +docker build --tag dbt-redshift-1.0.0b1 --target dbt-redshift --build-arg commit_ref=v1.0.0b1 docker +docker run dbt-redshift-1.0.0b1 --version From 6dbdbac9b5c9849a0541abc3ab927b166b8207f9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 14 May 2024 11:06:32 -0400 Subject: [PATCH 15/16] pull release-prep into this repo to install aws cli, use env vars, and deal with flaky tests --- .github/workflows/release-prep.yml | 702 +++++++++++++++++++++++++++++ .github/workflows/release.yml | 2 +- 2 files changed, 703 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release-prep.yml diff --git a/.github/workflows/release-prep.yml b/.github/workflows/release-prep.yml new file mode 100644 index 000000000..5c5ff0cd7 --- /dev/null +++ b/.github/workflows/release-prep.yml @@ -0,0 +1,702 @@ +# **what?** +# Perform the version bump, generate the changelog and run tests. +# +# Inputs: +# sha: The commit to attach to this release +# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) +# target_branch: The branch that we will release from +# env_setup_script_path: Path to the environment setup script +# test_run: Test run (The temp branch will be used for release) +# nightly_release: Identifier that this is nightly release +# +# Outputs: +# final_sha: The sha that will actually be released. This can differ from the +# input sha if adding a version bump and/or changelog +# changelog_path: Path to the changelog file (ex .changes/1.2.3-rc1.md) +# +# Branching strategy: +# - During execution workflow execution the temp branch will be generated. +# - For normal runs the temp branch will be removed once changes were merged to target branch; +# - For test runs we will keep temp branch and will use it for release; +# Naming strategy: +# - For normal runs: prep-release/${{ inputs.version_number }}_$GITHUB_RUN_ID +# - For test runs: prep-release/test-run/${{ inputs.version_number }}_$GITHUB_RUN_ID +# - For nightly releases: prep-release/nightly-release/${{ inputs.version_number }}_$GITHUB_RUN_ID +# +# **why?** +# Reusable and consistent GitHub release process. +# +# **when?** +# Call when ready to kick off a build and release +# +# Validation Checks +# +# 1. Bump the version if it has not been bumped +# 2. Generate the changelog (via changie) if there is no markdown file for this version +# + +name: Version Bump and Changelog Generation + +on: + workflow_call: + inputs: + sha: + required: true + type: string + version_number: + required: true + type: string + target_branch: + required: true + type: string + env_setup_script_path: + required: false + type: string + default: "" + test_run: + required: false + default: true + type: boolean + nightly_release: + type: boolean + default: false + required: false + outputs: + final_sha: + description: The new commit that includes the changelog and version bump. + value: ${{ jobs.determine-release-sha.outputs.final_sha }} + changelog_path: + description: The path to the changelog for this version + value: ${{ jobs.audit-changelog.outputs.changelog_path }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +env: + PYTHON_TARGET_VERSION: 3.8 + NOTIFICATION_PREFIX: "[Release Preparation]" + +jobs: + log-inputs: + runs-on: ubuntu-latest + + steps: + - name: "[DEBUG] Print Variables" + run: | + # WORKFLOW INPUTS + echo The last commit sha in the release: ${{ inputs.sha }} + echo The release version number: ${{ inputs.version_number }} + echo The branch that we will release from: ${{ inputs.target_branch }} + echo Path to the environment setup script: ${{ inputs.env_setup_script_path }} + echo Test run: ${{ inputs.test_run }} + echo Nightly release: ${{ inputs.nightly_release }} + # ENVIRONMENT VARIABLES + echo Python target version: ${{ env.PYTHON_TARGET_VERSION }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + audit-changelog: + runs-on: ubuntu-latest + + outputs: + changelog_path: ${{ steps.set_path.outputs.changelog_path }} + exists: ${{ steps.set_existence.outputs.exists }} + base_version: ${{ steps.semver.outputs.base-version }} + prerelease: ${{ steps.semver.outputs.pre-release }} + is_prerelease: ${{ steps.semver.outputs.is-pre-release }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Audit Version And Parse Into Parts" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version_number }} + + - name: "Set Changelog Path" + id: set_path + run: | + path=".changes/" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" + else + path+="${{ steps.semver.outputs.base-version }}.md" + fi + # Send notification + echo "changelog_path=$path" >> $GITHUB_OUTPUT + title="Changelog path" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$changelog_path" + + - name: "Set Changelog Existence For Subsequent Jobs" + id: set_existence + run: | + does_exist=false + if test -f ${{ steps.set_path.outputs.changelog_path }} + then + does_exist=true + fi + echo "exists=$does_exist">> $GITHUB_OUTPUT + + - name: "[Notification] Set Changelog Existence For Subsequent Jobs" + run: | + title="Changelog exists" + if [[ ${{ steps.set_existence.outputs.exists }} == true ]] + then + message="Changelog file ${{ steps.set_path.outputs.changelog_path }} already exists" + else + message="Changelog file ${{ steps.set_path.outputs.changelog_path }} doesn't exist" + fi + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo changelog_path: ${{ steps.set_path.outputs.changelog_path }} + echo exists: ${{ steps.set_existence.outputs.exists }} + echo base_version: ${{ steps.semver.outputs.base-version }} + echo prerelease: ${{ steps.semver.outputs.pre-release }} + echo is_prerelease: ${{ steps.semver.outputs.is-pre-release }} + + audit-version-in-code: + runs-on: ubuntu-latest + + outputs: + up_to_date: ${{ steps.version-check.outputs.up_to_date }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Check Current Version In Code" + id: version-check + run: | + is_updated=false + if grep -Fxq "current_version = ${{ inputs.version_number }}" .bumpversion.cfg + then + is_updated=true + fi + echo "up_to_date=$is_updated" >> $GITHUB_OUTPUT + + - name: "[Notification] Check Current Version In Code" + run: | + title="Version check" + if [[ ${{ steps.version-check.outputs.up_to_date }} == true ]] + then + message="The version in the codebase is equal to the provided version" + else + message="The version in the codebase differs from the provided version" + fi + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo up_to_date: ${{ steps.version-check.outputs.up_to_date }} + + skip-generate-changelog: + runs-on: ubuntu-latest + needs: [audit-changelog] + if: needs.audit-changelog.outputs.exists == 'true' + + steps: + - name: "Changelog Exists, Skip Generating New Changelog" + run: | + # Send notification + title="Skip changelog generation" + message="A changelog file already exists at ${{ needs.audit-changelog.outputs.changelog_path }}, skipping generating changelog" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + skip-version-bump: + runs-on: ubuntu-latest + needs: [audit-version-in-code] + if: needs.audit-version-in-code.outputs.up_to_date == 'true' + + steps: + - name: "Version Already Bumped" + run: | + # Send notification + title="Skip version bump" + message="The version has already been bumped to ${{ inputs.version_number }}, skipping version bump" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + create-temp-branch: + runs-on: ubuntu-latest + needs: [audit-changelog, audit-version-in-code] + if: needs.audit-changelog.outputs.exists == 'false' || needs.audit-version-in-code.outputs.up_to_date == 'false' + + outputs: + branch_name: ${{ steps.variables.outputs.branch_name }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Generate Branch Name" + id: variables + run: | + name="prep-release/" + if [[ ${{ inputs.nightly_release }} == true ]] + then + name+="nightly-release/" + elif [[ ${{ inputs.test_run }} == true ]] + then + name+="test-run/" + fi + name+="${{ inputs.version_number }}_$GITHUB_RUN_ID" + echo "branch_name=$name" >> $GITHUB_OUTPUT + + - name: "Create Branch - ${{ steps.variables.outputs.branch_name }}" + run: | + git checkout -b ${{ steps.variables.outputs.branch_name }} + git push -u origin ${{ steps.variables.outputs.branch_name }} + + - name: "[Notification] Temp branch created" + run: | + # Send notification + title="Temp branch generated" + message="The ${{ steps.variables.outputs.branch_name }} branch created" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo branch_name ${{ steps.variables.outputs.branch_name }} + + generate-changelog-bump-version: + runs-on: ubuntu-latest + needs: [audit-changelog, audit-version-in-code, create-temp-branch] + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Add Homebrew To PATH" + run: | + echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH + + - name: "Install Homebrew Packages" + run: | + brew install pre-commit + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + + - name: "Set json File Name" + id: json_file + run: | + echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT + + - name: "Get Core Team Membership" + run: | + gh api -H "Accept: application/vnd.github+json" orgs/dbt-labs/teams/core-group/members > ${{ steps.json_file.outputs.name }} + env: + GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} + + - name: "Set Core Team Membership for Changie Contributors exclusion" + id: set_team_membership + run: | + team_list=$(jq -r '.[].login' ${{ steps.json_file.outputs.name }}) + echo $team_list + team_list_single=$(echo $team_list | tr '\n' ' ') + echo "CHANGIE_CORE_TEAM=$team_list_single" >> $GITHUB_ENV + + - name: "Delete the json File" + run: | + rm ${{ steps.json_file.outputs.name }} + + - name: "Generate Release Changelog" + if: needs.audit-changelog.outputs.exists == 'false' + run: | + if [[ ${{ needs.audit-changelog.outputs.is_prerelease }} -eq 1 ]] + then + changie batch ${{ needs.audit-changelog.outputs.base_version }} --move-dir '${{ needs.audit-changelog.outputs.base_version }}' --prerelease ${{ needs.audit-changelog.outputs.prerelease }} + elif [[ -d ".changes/${{ needs.audit-changelog.outputs.base_version }}" ]] + then + changie batch ${{ needs.audit-changelog.outputs.base_version }} --include '${{ needs.audit-changelog.outputs.base_version }}' --remove-prereleases + else # releasing a final patch with no prereleases + changie batch ${{ needs.audit-changelog.outputs.base_version }} + fi + changie merge + git status + + - name: "Check Changelog Created Successfully" + if: needs.audit-changelog.outputs.exists == 'false' + run: | + title="Changelog" + if [[ -f ${{ needs.audit-changelog.outputs.changelog_path }} ]] + then + message="Changelog file created successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Changelog failed to generate" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + run: | + python3 -m venv env + source env/bin/activate + python -m pip install --upgrade pip + + - name: "Bump Version To ${{ inputs.version_number }}" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + # note: bumpversion is no longer supported, it actually points to bump2version now + run: | + source env/bin/activate + if [ -f "editable-requirements.txt" ] + then + python -m pip install -r dev-requirements.txt -r editable-requirements.txt + else + python -m pip install -r dev-requirements.txt + fi + env/bin/bumpversion --allow-dirty --new-version ${{ inputs.version_number }} major + git status + + - name: "[Notification] Bump Version To ${{ inputs.version_number }}" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + run: | + title="Version bump" + message="Version successfully bumped in codebase to ${{ inputs.version_number }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + # this step will fail on whitespace errors but also correct them + - name: "Remove Trailing Whitespace Via Pre-commit" + continue-on-error: true + run: | + pre-commit run trailing-whitespace --files .bumpversion.cfg CHANGELOG.md .changes/* + git status + + # this step will fail on newline errors but also correct them + - name: "Removing Extra Newlines Via Pre-commit" + continue-on-error: true + run: | + pre-commit run end-of-file-fixer --files .bumpversion.cfg CHANGELOG.md .changes/* + git status + + - name: "Commit & Push Changes" + run: | + #Data for commit + user="Github Build Bot" + email="buildbot@fishtownanalytics.com" + commit_message="Bumping version to ${{ inputs.version_number }} and generate changelog" + #Commit changes to branch + git config user.name "$user" + git config user.email "$email" + git pull + git add . + git commit -m "$commit_message" + git push + + run-unit-tests: + runs-on: ubuntu-latest + needs: [create-temp-branch, generate-changelog-bump-version] + + env: + TOXENV: unit + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + python -m tox --version + + - name: "Run Tox" + run: tox + + run-integration-tests: + runs-on: ubuntu-22.04 + needs: [create-temp-branch, generate-changelog-bump-version] + if: inputs.env_setup_script_path != '' + + env: + TOXENV: integration + PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" + DBT_INVOCATION_ENV: github-actions + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install python tools" + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + tox --version + + - name: Create AWS IAM profiles + run: | + aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID + aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY + aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION + aws configure --profile $AWS_USER_PROFILE set output json + + aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID + aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY + aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_SOURCE_PROFILE set output json + + aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE + aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN + aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_ROLE_PROFILE set output json + env: + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + + - name: Run tests + run: tox -- -m "not flaky" + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + + run-integration-tests-flaky: + runs-on: ubuntu-22.04 + needs: [run-integration-tests, create-temp-branch] + if: inputs.env_setup_script_path != '' + + env: + TOXENV: integration + PYTEST_ADDOPTS: "-v --color=yes -n1 --csv integration_results.csv" + DBT_INVOCATION_ENV: github-actions + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install python tools" + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + tox --version + + - name: Create AWS IAM profiles + run: | + aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID + aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY + aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION + aws configure --profile $AWS_USER_PROFILE set output json + + aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID + aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY + aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_SOURCE_PROFILE set output json + + aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE + aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN + aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_ROLE_PROFILE set output json + env: + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + + - name: Run tests + run: tox -- -m flaky + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + + merge-changes-into-target-branch: + runs-on: ubuntu-latest + needs: [run-unit-tests, run-integration-tests-flaky, create-temp-branch, audit-version-in-code, audit-changelog] + if: | + !failure() && !cancelled() && + inputs.test_run == false && + ( + needs.audit-changelog.outputs.exists == 'false' || + needs.audit-version-in-code.outputs.up_to_date == 'false' + ) + + steps: + - name: "[Debug] Print Variables" + run: | + echo target_branch: ${{ inputs.target_branch }} + echo branch_name: ${{ needs.create-temp-branch.outputs.branch_name }} + echo inputs.test_run: ${{ inputs.test_run }} + echo needs.audit-changelog.outputs.exists: ${{ needs.audit-changelog.outputs.exists }} + echo needs.audit-version-in-code.outputs.up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} + + - name: "Checkout Repo ${{ github.repository }}" + uses: actions/checkout@v4 + + - name: "Merge Changes Into ${{ inputs.target_branch }}" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.create-temp-branch.outputs.branch_name }} + target_branch: ${{ inputs.target_branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + + - name: "[Notification] Changes Merged into ${{ inputs.target_branch }}" + run: | + title="Changelog and Version Bump Branch Merge" + message="The ${{ needs.create-temp-branch.outputs.branch_name }} branch was merged into ${{ inputs.target_branch }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + determine-release-sha: + runs-on: ubuntu-latest + needs: + [ + create-temp-branch, + merge-changes-into-target-branch, + audit-changelog, + audit-version-in-code, + ] + # always run this job, regardless of if the dependant jobs were skipped + if: ${{ !failure() && !cancelled() }} + + # Get the sha that will be released. If the changelog already exists on the input sha and the version has already been bumped, + # then it is what we will release. Otherwise we generated a changelog and did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + final_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} + + steps: + - name: "[Debug] Print Variables" + run: | + echo target_branch: ${{ inputs.target_branch }} + echo new_branch: ${{ needs.create-temp-branch.outputs.branch_name }} + echo changelog_exists: ${{ needs.audit-changelog.outputs.exists }} + echo up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} + + - name: "Resolve Branch To Checkout" + id: resolve_branch + run: | + branch="" + if [[ ${{ inputs.test_run == true }} ]] + then + branch=${{ needs.create-temp-branch.outputs.branch_name }} + else + branch=${{ inputs.target_branch }} + fi + echo "target_branch=$branch" >> $GITHUB_OUTPUT + + - name: "[Notification] Resolve Branch To Checkout" + run: | + title="Branch pick" + message="The ${{ steps.resolve_branch.outputs.target_branch }} branch will be used for release" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Checkout Resolved Branch - ${{ steps.resolve_branch.outputs.target_branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ steps.resolve_branch.outputs.target_branch }} + + - name: "[Debug] Log Branch" + run: git status + + - name: "Resolve Commit SHA For Release" + id: resolve_commit_sha + run: | + commit_sha="" + if [[ ${{ needs.audit-changelog.outputs.exists }} == false ]] || [[ ${{ needs.audit-version-in-code.outputs.up_to_date }} == false ]] + then + commit_sha=$(git rev-parse HEAD) + else + commit_sha=${{ inputs.sha }} + fi + echo "release_sha=$commit_sha" >> $GITHUB_OUTPUT + + - name: "[Notification] Resolve Commit SHA For Release" + run: | + title="Release commit pick" + message="The ${{ steps.resolve_commit_sha.outputs.release_sha }} commit will be used for release" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Remove Temp Branch - ${{ needs.create-temp-branch.outputs.branch_name }}" + if: ${{ inputs.test_run == false && needs.create-temp-branch.outputs.branch_name != '' }} + run: | + git push origin -d ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "[Debug] Print Outputs" + run: | + echo release_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index be093d975..e01b961f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -137,7 +137,7 @@ jobs: bump-version-generate-changelog: name: Bump package version, Generate changelog - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main + uses: ./.github/workflows/release-prep.yml with: sha: ${{ inputs.sha }} version_number: ${{ inputs.version_number }} From f485f8fd3226323d266256ea26914dc30f6f5a02 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 20 May 2024 18:45:03 -0400 Subject: [PATCH 16/16] remove unused script --- docker/test.sh | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100755 docker/test.sh diff --git a/docker/test.sh b/docker/test.sh deleted file mode 100755 index 74261233a..000000000 --- a/docker/test.sh +++ /dev/null @@ -1,19 +0,0 @@ -# - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` -# TODO: create a real test suite -set -e - -echo "\n\n" -echo "#######################################\n" -echo "##### Testing dbt-redshift latest #####\n" -echo "#######################################\n" - -docker build --tag dbt-redshift --target dbt-redshift docker -docker run dbt-redshift --version - -echo "\n\n" -echo "########################################\n" -echo "##### Testing dbt-redshift-1.0.0b1 #####\n" -echo "########################################\n" - -docker build --tag dbt-redshift-1.0.0b1 --target dbt-redshift --build-arg commit_ref=v1.0.0b1 docker -docker run dbt-redshift-1.0.0b1 --version