From 3186972f425f61166f9eecff3694fdec39374035 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 5 Apr 2024 18:54:33 -0400 Subject: [PATCH 01/20] add docker release to release pipeline --- .github/workflows/release.yml | 25 +++++++++++ docker-release/Dockerfile | 46 ++++++++++++++++++++ docker-release/README.md | 80 +++++++++++++++++++++++++++++++++++ docker-release/test.sh | 22 ++++++++++ 4 files changed, 173 insertions(+) create mode 100644 docker-release/Dockerfile create mode 100644 docker-release/README.md create mode 100755 docker-release/test.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b2774f17..4038377ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -163,6 +163,31 @@ jobs: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + is-docker-release: + name: "Is Docker Release" + runs-on: ubuntu-latest + needs: [pypi-release] + outputs: + is-docker-release: ${{ steps.semver.outputs.is-pre-release == 0 }} + steps: + - name: "Audit version to determine if it is a pre-release" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version_number }} + + docker-release: + name: "Docker Release" + needs: [is-docker-release] + if: ${{ needs.is-docker-release.outputs.is-docker-release }} + permissions: + packages: write + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + with: + package: "dbt-spark" + version_number: ${{ inputs.version_number }} + test_run: ${{ inputs.test_run }} + slack-notification: name: Slack Notification if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} diff --git a/docker-release/Dockerfile b/docker-release/Dockerfile new file mode 100644 index 000000000..234c88fdf --- /dev/null +++ b/docker-release/Dockerfile @@ -0,0 +1,46 @@ +ARG build_for=linux/amd64 + +FROM --platform=$build_for python:3.10.7-slim-bullseye as base + +# ref is updated automatically every final release via bumpversion +ARG dbt_spark_ref=dbt-spark@v1.7.10 +ARG dbt_spark_version=all + +RUN apt-get update \ + && apt-get dist-upgrade -y \ + && apt-get install -y --no-install-recommends \ + git \ + ssh-client \ + software-properties-common \ + make \ + build-essential \ + ca-certificates \ + libpq-dev \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +ENV PYTHONIOENCODING=utf-8 +ENV LANG=C.UTF-8 + +RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir + +WORKDIR /usr/app/dbt/ +ENTRYPOINT ["dbt"] + +FROM base as dbt-spark +RUN apt-get update \ + && apt-get dist-upgrade -y \ + && apt-get install -y --no-install-recommends \ + python-dev \ + libsasl2-dev \ + gcc \ + unixodbc-dev \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* +RUN python -m pip install --no-cache-dir "dbt-spark[${dbt_spark_version}] @ git+https://github.com/dbt-labs/${dbt_spark_ref}" diff --git a/docker-release/README.md b/docker-release/README.md new file mode 100644 index 000000000..3698cf803 --- /dev/null +++ b/docker-release/README.md @@ -0,0 +1,80 @@ +# Docker for dbt +This docker file is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry. + + +## Building an image: +This Dockerfile can create images for the following target: `dbt-spark` + +In order to build a new image, run the following docker command. +```shell +docker build --tag --target dbt-spark +``` +--- +> **Note:** Docker must be configured to use [BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/) in order for images to build properly! + +--- + +By default the images will be populated with the most recent release of `dbt-spark`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +```shell +docker build --tag \ + --target dbt-spark \ + --build-arg dbt_spark_ref= \ + +``` + +### Examples: +To build an image named "my-dbt" that supports Snowflake using the latest releases: +```shell +cd dbt-core/docker +docker build --tag my-dbt --target dbt-spark . +``` + +To build an image named "my-other-dbt" that supports Snowflake using the adapter version 1.0.0b1: +```shell +cd dbt-core/docker +docker build \ + --tag my-other-dbt \ + --target dbt-spark \ + --build-arg dbt_spark_ref=dbt-spark@v1.0.0b1 \ + . +``` + +## Special cases +There are a few special cases worth noting: +* The `dbt-spark` database adapter comes in three different versions named `PyHive`, `ODBC`, and the default `all`. +If you wish to override this you can use the `--build-arg` flag with the value of `dbt_spark_version=`. +See the [docs](https://docs.getdbt.com/reference/warehouse-profiles/spark-profile) for more information. +```shell +docker build --tag my_dbt \ + --target dbt-spark \ + --build-arg dbt_spark_ref=dbt-spark@1.0.0b1 \ + --build-arg dbt_spark_version=PyHive \ + +``` + +* If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg: +```shell +docker build --tag my_dbt \ + --target dbt-spark \ + --build-arg build_for=linux/arm64 \ + +``` + +Supported architectures can be found in the python docker [dockerhub page](https://hub.docker.com/_/python). + +## Running an image in a container: +The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: +```shell +docker run \ + --network=host \ + --mount type=bind,source=path/to/project,target=/usr/app \ + --mount type=bind,source=path/to/profiles.yml,target=/root/.dbt/profiles.yml \ + my-dbt \ + ls +``` +--- +**Notes:** +* Bind-mount sources _must_ be an absolute path +* You may need to make adjustments to the docker networking setting depending on the specifics of your data warehouse/database host. + +--- diff --git a/docker-release/test.sh b/docker-release/test.sh new file mode 100755 index 000000000..04435846a --- /dev/null +++ b/docker-release/test.sh @@ -0,0 +1,22 @@ +# - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` +# TODO: create a real test suite + +clear \ +&& echo "\n\n"\ +"########################################\n"\ +"##### Testing dbt-spark latest #####\n"\ +"########################################\n"\ +&& docker build --tag dbt-spark \ + --target dbt-spark \ + docker \ +&& docker run dbt-spark --version \ +\ +&& echo "\n\n"\ +"#########################################\n"\ +"##### Testing dbt-spark-1.0.0b1 #####\n"\ +"#########################################\n"\ +&& docker build --tag dbt-spark-1.0.0b1 \ + --target dbt-spark \ + --build-arg dbt_spark_ref=dbt-spark@v1.0.0b1 \ + docker \ +&& docker run dbt-spark-1.0.0b1 --version From 446bb0c3c4691bad8d8abea41e495a1130497167 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 5 Apr 2024 18:56:19 -0400 Subject: [PATCH 02/20] changelog --- .changes/unreleased/Under the Hood-20240405-185609.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20240405-185609.yaml diff --git a/.changes/unreleased/Under the Hood-20240405-185609.yaml b/.changes/unreleased/Under the Hood-20240405-185609.yaml new file mode 100644 index 000000000..e453a63f8 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240405-185609.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Automate the Docker release as part of the PyPI release +time: 2024-04-05T18:56:09.981637-04:00 +custom: + Author: mikealfare + Issue: "1004" From 548f7df3cdaa879963e8858d14a49262dd2bbba5 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 13 Apr 2024 02:02:46 -0400 Subject: [PATCH 03/20] update docker release to align with other adapters, add dev docker --- .github/workflows/release.yml | 47 +++++----------------------- Makefile | 10 ++++++ docker-dev/Dockerfile | 59 +++++++++++++++++++++++++++++++++++ docker-release/Dockerfile | 52 ++++++++++++++---------------- docker-release/README.md | 23 +++++--------- docker-release/test.sh | 2 +- 6 files changed, 108 insertions(+), 85 deletions(-) create mode 100644 docker-dev/Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4038377ac..f7c9a08d0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,8 +13,8 @@ # This will only run manually. Run this workflow only after the # version bump workflow is completed and related changes are reviewed and merged. # - -name: Release to GitHub and PyPI +name: "Release to GitHub, PyPI, and Docker" +run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI, and Docker" on: workflow_dispatch: @@ -88,26 +88,20 @@ jobs: # so that the integration tests stage will be skipped. audit-version-and-changelog: name: Bump package version, Generate changelog - uses: dbt-labs/dbt-spark/.github/workflows/release-prep.yml@main - with: sha: ${{ inputs.sha }} version_number: ${{ inputs.version_number }} target_branch: ${{ inputs.target_branch }} env_setup_script_path: "" test_run: ${{ inputs.test_run }} - secrets: inherit log-outputs-audit-version-and-changelog: name: "[Log output] Bump package version, Generate changelog" if: ${{ !failure() && !cancelled() }} - needs: [audit-version-and-changelog] - runs-on: ubuntu-latest - steps: - name: Print variables run: | @@ -118,9 +112,7 @@ jobs: name: Build, Test, Package if: ${{ !failure() && !cancelled() }} needs: [audit-version-and-changelog] - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - with: sha: ${{ needs.audit-version-and-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} @@ -129,7 +121,6 @@ jobs: s3_bucket_name: ${{ inputs.s3_bucket_name }} package_test_command: ${{ inputs.package_test_command }} test_run: ${{ inputs.test_run }} - secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -137,11 +128,8 @@ jobs: github-release: name: GitHub Release if: ${{ !failure() && !cancelled() }} - needs: [audit-version-and-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main - with: sha: ${{ needs.audit-version-and-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} @@ -150,59 +138,38 @@ jobs: pypi-release: name: PyPI Release - needs: [github-release] - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - with: version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} - secrets: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - is-docker-release: - name: "Is Docker Release" - runs-on: ubuntu-latest - needs: [pypi-release] - outputs: - is-docker-release: ${{ steps.semver.outputs.is-pre-release == 0 }} - steps: - - name: "Audit version to determine if it is a pre-release" - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ inputs.version_number }} - docker-release: name: "Docker Release" - needs: [is-docker-release] - if: ${{ needs.is-docker-release.outputs.is-docker-release }} + if: ${{ !failure() && !cancelled() }} + needs: [bump-version-generate-changelog, build-test-package, github-release] permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release with: - package: "dbt-spark" + package: ${{ github.event.repository.name }} version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} slack-notification: name: Slack Notification if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} - needs: [ - audit-version-and-changelog, - build-test-package, github-release, pypi-release, + docker-release, ] - uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main with: status: "failure" - secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_CORE_ALERTS }} diff --git a/Makefile b/Makefile index 2bd1055fa..c497ba3b8 100644 --- a/Makefile +++ b/Makefile @@ -61,3 +61,13 @@ help: ## Show this help message. @echo @echo 'targets:' @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + + +.PHONY: docker-dev +docker-dev: + docker build -f docker-dev/Dockerfile -t dbt-spark-dev . + docker run --rm -it --name dbt-spark-dev -v $(shell pwd):/opt/code dbt-spark-dev + +.PHONY: docker-prod +docker-prod: + docker build -f docker-release/Dockerfile -t dbt-spark . diff --git a/docker-dev/Dockerfile b/docker-dev/Dockerfile new file mode 100644 index 000000000..5f1f8e3f0 --- /dev/null +++ b/docker-dev/Dockerfile @@ -0,0 +1,59 @@ +# this image does not get published, it is intended for local development only, see `Makefile` for usage +FROM ubuntu:22.04 as base + +# prevent python installation from asking for time zone region +ARG DEBIAN_FRONTEND=noninteractive + +# add python repository +RUN apt-get update \ + && apt-get install -y software-properties-common=0.99.22.9 \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# install python +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9ubuntu3 \ + gcc=4:11.2.0-1ubuntu1 \ + git-all=1:2.34.1-1ubuntu1.10 \ + libsasl2-dev=2.1.27+dfsg2-3ubuntu1.2 \ + python3.10=3.10.12-1~22.04.3 \ + python3.10-dev=3.10.12-1~22.04.3 \ + python3.10-venv=3.10.12-1~22.04.3 \ + python3-distutils=3.10.8-1~22.04 \ + python3-pip=22.0.2+dfsg-1ubuntu0.4 \ + python3-wheel=0.37.1-2ubuntu0.22.04.1 \ + unixodbc-dev=2.3.9-5ubuntu0.1 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# update the default system interpreter to the newly installed version +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 + +# install python dependencies +RUN python3 -m pip install --upgrade --no-cache-dir \ + "beartype==0.17.2" \ + "dagger-io==0.10.3" \ + "python-dotenv==1.0.1" + + +FROM base as dbt-spark-dev + +HEALTHCHECK CMD python3 --version || exit 1 + +# send stdout/stderr to terminal +ENV PYTHONUNBUFFERED=1 + +# setup mount for local code +WORKDIR /opt/code +VOLUME /opt/code + +# create a virtual environment +RUN python3 -m venv /opt/venv diff --git a/docker-release/Dockerfile b/docker-release/Dockerfile index 234c88fdf..588d5a256 100644 --- a/docker-release/Dockerfile +++ b/docker-release/Dockerfile @@ -1,21 +1,22 @@ -ARG build_for=linux/amd64 +# this image gets published to GHCR for production use +ARG py_version=3.10.7 -FROM --platform=$build_for python:3.10.7-slim-bullseye as base - -# ref is updated automatically every final release via bumpversion -ARG dbt_spark_ref=dbt-spark@v1.7.10 -ARG dbt_spark_version=all +FROM python:$py_version-slim-bullseye as base RUN apt-get update \ && apt-get dist-upgrade -y \ && apt-get install -y --no-install-recommends \ - git \ - ssh-client \ - software-properties-common \ - make \ - build-essential \ - ca-certificates \ - libpq-dev \ + build-essential=12.9 \ + ca-certificates=20210119 \ + gcc=4:10.2.1-1 \ + git=1:2.30.2-1+deb11u2 \ + libpq-dev=13.14-0+deb11u1 \ + libsasl2-dev=2.1.27+dfsg-2.1+deb11u1 \ + make=4.3-4.1 \ + openssh-client=1:8.4p1-5+deb11u3 \ + python-dev-is-python2=2.7.18-9 \ + software-properties-common=0.96.20.2-2.1 \ + unixodbc-dev=2.3.6-0.1+b1 \ && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ @@ -25,22 +26,17 @@ RUN apt-get update \ ENV PYTHONIOENCODING=utf-8 ENV LANG=C.UTF-8 -RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir +RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir + + +FROM base as dbt-spark + +ARG commit_ref=main +ARG extras=all + +HEALTHCHECK CMD dbt --version || exit 1 WORKDIR /usr/app/dbt/ ENTRYPOINT ["dbt"] -FROM base as dbt-spark -RUN apt-get update \ - && apt-get dist-upgrade -y \ - && apt-get install -y --no-install-recommends \ - python-dev \ - libsasl2-dev \ - gcc \ - unixodbc-dev \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* -RUN python -m pip install --no-cache-dir "dbt-spark[${dbt_spark_version}] @ git+https://github.com/dbt-labs/${dbt_spark_ref}" +RUN python -m pip install --no-cache-dir "dbt-spark[${extras}] @ git+https://github.com/dbt-labs/dbt-spark@${commit_ref}" diff --git a/docker-release/README.md b/docker-release/README.md index 3698cf803..1b6bf6b6c 100644 --- a/docker-release/README.md +++ b/docker-release/README.md @@ -14,11 +14,12 @@ docker build --tag --target dbt-spark --- -By default the images will be populated with the most recent release of `dbt-spark`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +By default the image will be populated with the latest version of `dbt-spark` on `main`. +If you need to use a different version you can specify it by git ref using the `--build-arg` flag: ```shell docker build --tag \ --target dbt-spark \ - --build-arg dbt_spark_ref= \ + --build-arg commit_ref= \ ``` @@ -35,33 +36,23 @@ cd dbt-core/docker docker build \ --tag my-other-dbt \ --target dbt-spark \ - --build-arg dbt_spark_ref=dbt-spark@v1.0.0b1 \ + --build-arg commit_ref=v1.0.0b1 \ . ``` ## Special cases There are a few special cases worth noting: * The `dbt-spark` database adapter comes in three different versions named `PyHive`, `ODBC`, and the default `all`. -If you wish to override this you can use the `--build-arg` flag with the value of `dbt_spark_version=`. +If you wish to override this you can use the `--build-arg` flag with the value of `extras=`. See the [docs](https://docs.getdbt.com/reference/warehouse-profiles/spark-profile) for more information. ```shell docker build --tag my_dbt \ --target dbt-spark \ - --build-arg dbt_spark_ref=dbt-spark@1.0.0b1 \ - --build-arg dbt_spark_version=PyHive \ + --build-arg commit_ref=v1.0.0b1 \ + --build-arg extras=PyHive \ ``` -* If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg: -```shell -docker build --tag my_dbt \ - --target dbt-spark \ - --build-arg build_for=linux/arm64 \ - -``` - -Supported architectures can be found in the python docker [dockerhub page](https://hub.docker.com/_/python). - ## Running an image in a container: The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: ```shell diff --git a/docker-release/test.sh b/docker-release/test.sh index 04435846a..c51b681bc 100755 --- a/docker-release/test.sh +++ b/docker-release/test.sh @@ -17,6 +17,6 @@ clear \ "#########################################\n"\ && docker build --tag dbt-spark-1.0.0b1 \ --target dbt-spark \ - --build-arg dbt_spark_ref=dbt-spark@v1.0.0b1 \ + --build-arg commit_ref=v1.0.0b1 \ docker \ && docker run dbt-spark-1.0.0b1 --version From 8e31cc446f34a274bb156a83da28b48ab4ee1a2e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 13 Apr 2024 02:05:16 -0400 Subject: [PATCH 04/20] update docker release to align with other adapters, add dev docker --- .github/dependabot.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2a6f34492..4299a42a0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,8 +1,22 @@ version: 2 updates: - # python dependencies - package-ecosystem: "pip" directory: "/" schedule: interval: "daily" rebase-strategy: "disabled" + - package-ecosystem: "docker" + directory: "/docker" + schedule: + interval: "weekly" + rebase-strategy: "disabled" + - package-ecosystem: "docker" + directory: "/docker-dev" + schedule: + interval: "weekly" + rebase-strategy: "disabled" + - package-ecosystem: "docker" + directory: "/docker-release" + schedule: + interval: "weekly" + rebase-strategy: "disabled" From e9420b5f6727ebc1e1202ac995a88cf2f8ad1086 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 10:29:59 -0400 Subject: [PATCH 05/20] remove defaulted input for docker package, override default for docker release image --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f7c9a08d0..7a183c97c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -155,8 +155,8 @@ jobs: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release with: - package: ${{ github.event.repository.name }} version_number: ${{ inputs.version_number }} + dockerfile: "docker-release/Dockerfile" test_run: ${{ inputs.test_run }} slack-notification: From acdc453e4ca8f06cf7f38643b45d34730cf2d5f0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 17:29:31 -0400 Subject: [PATCH 06/20] point back to main --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a183c97c..95f1aa611 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -153,7 +153,7 @@ jobs: needs: [bump-version-generate-changelog, build-test-package, github-release] permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main with: version_number: ${{ inputs.version_number }} dockerfile: "docker-release/Dockerfile" From fb197f0e2380273d85d1e6f181e8cdc8a546ff2d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 17:37:12 -0400 Subject: [PATCH 07/20] remove changie entry --- .changes/unreleased/Under the Hood-20240405-185609.yaml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .changes/unreleased/Under the Hood-20240405-185609.yaml diff --git a/.changes/unreleased/Under the Hood-20240405-185609.yaml b/.changes/unreleased/Under the Hood-20240405-185609.yaml deleted file mode 100644 index e453a63f8..000000000 --- a/.changes/unreleased/Under the Hood-20240405-185609.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Automate the Docker release as part of the PyPI release -time: 2024-04-05T18:56:09.981637-04:00 -custom: - Author: mikealfare - Issue: "1004" From fc7de15fc69214132510125ea828db7e2b4a744c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 21:26:12 -0400 Subject: [PATCH 08/20] fix docker release dependent steps --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 95f1aa611..1750c25bf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -150,7 +150,7 @@ jobs: docker-release: name: "Docker Release" if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog, build-test-package, github-release] + needs: [github-release] permissions: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main From d9d27b0b989e0133fb94f35708101a6439afad1d Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 17 Apr 2024 08:31:51 -0500 Subject: [PATCH 09/20] only release docker when not testing, allow to only release to docker --- .github/workflows/release.yml | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1750c25bf..a6fb830b1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,6 +56,11 @@ on: type: boolean default: true required: false + only_docker: + description: "Only release Docker image, skip GitHub & PyPI" + type: boolean + default: false + required: false permissions: contents: write # this is the permission that allows creating a new release @@ -66,7 +71,7 @@ defaults: jobs: log-inputs: - name: Log Inputs + name: "Log Inputs" runs-on: ubuntu-latest steps: - name: "[DEBUG] Print Variables" @@ -79,6 +84,7 @@ jobs: echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }} echo Package test command: ${{ inputs.package_test_command }} echo Test run: ${{ inputs.test_run }} + echo Only Docker: ${{ inputs.only_docker }} # The Spark repository uses CircleCI to run integration tests. # Because of this, the process of version bumps will be manual @@ -87,7 +93,7 @@ jobs: # We are passing `env_setup_script_path` as an empty string # so that the integration tests stage will be skipped. audit-version-and-changelog: - name: Bump package version, Generate changelog + name: "Bump package version, Generate changelog" uses: dbt-labs/dbt-spark/.github/workflows/release-prep.yml@main with: sha: ${{ inputs.sha }} @@ -99,7 +105,7 @@ jobs: log-outputs-audit-version-and-changelog: name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [audit-version-and-changelog] runs-on: ubuntu-latest steps: @@ -109,8 +115,8 @@ jobs: echo Changelog path: ${{ needs.audit-version-and-changelog.outputs.changelog_path }} build-test-package: - name: Build, Test, Package - if: ${{ !failure() && !cancelled() }} + name: "Build, Test, Package" + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [audit-version-and-changelog] uses: dbt-labs/dbt-release/.github/workflows/build.yml@main with: @@ -126,8 +132,8 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} github-release: - name: GitHub Release - if: ${{ !failure() && !cancelled() }} + name: "GitHub Release" + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [audit-version-and-changelog, build-test-package] uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main with: @@ -137,7 +143,8 @@ jobs: test_run: ${{ inputs.test_run }} pypi-release: - name: PyPI Release + name: "PyPI Release" + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: [github-release] uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main with: @@ -149,7 +156,10 @@ jobs: docker-release: name: "Docker Release" - if: ${{ !failure() && !cancelled() }} + # We cannot release to docker on a test run because it uses the tag in GitHub as + # what we need to release but draft releases don't actually tag the commit so it + # finds nothing to release + if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }} needs: [github-release] permissions: packages: write From a82838a81c15fcc7c29e050101aa0a6e96f63251 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 12:24:19 -0400 Subject: [PATCH 10/20] remove dev container --- Makefile | 6 ----- docker-dev/Dockerfile | 59 ------------------------------------------- 2 files changed, 65 deletions(-) delete mode 100644 docker-dev/Dockerfile diff --git a/Makefile b/Makefile index c497ba3b8..cb9436cb6 100644 --- a/Makefile +++ b/Makefile @@ -62,12 +62,6 @@ help: ## Show this help message. @echo 'targets:' @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' - -.PHONY: docker-dev -docker-dev: - docker build -f docker-dev/Dockerfile -t dbt-spark-dev . - docker run --rm -it --name dbt-spark-dev -v $(shell pwd):/opt/code dbt-spark-dev - .PHONY: docker-prod docker-prod: docker build -f docker-release/Dockerfile -t dbt-spark . diff --git a/docker-dev/Dockerfile b/docker-dev/Dockerfile deleted file mode 100644 index 5f1f8e3f0..000000000 --- a/docker-dev/Dockerfile +++ /dev/null @@ -1,59 +0,0 @@ -# this image does not get published, it is intended for local development only, see `Makefile` for usage -FROM ubuntu:22.04 as base - -# prevent python installation from asking for time zone region -ARG DEBIAN_FRONTEND=noninteractive - -# add python repository -RUN apt-get update \ - && apt-get install -y software-properties-common=0.99.22.9 \ - && add-apt-repository -y ppa:deadsnakes/ppa \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* - -# install python -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential=12.9ubuntu3 \ - gcc=4:11.2.0-1ubuntu1 \ - git-all=1:2.34.1-1ubuntu1.10 \ - libsasl2-dev=2.1.27+dfsg2-3ubuntu1.2 \ - python3.10=3.10.12-1~22.04.3 \ - python3.10-dev=3.10.12-1~22.04.3 \ - python3.10-venv=3.10.12-1~22.04.3 \ - python3-distutils=3.10.8-1~22.04 \ - python3-pip=22.0.2+dfsg-1ubuntu0.4 \ - python3-wheel=0.37.1-2ubuntu0.22.04.1 \ - unixodbc-dev=2.3.9-5ubuntu0.1 \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* - -# update the default system interpreter to the newly installed version -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 - -# install python dependencies -RUN python3 -m pip install --upgrade --no-cache-dir \ - "beartype==0.17.2" \ - "dagger-io==0.10.3" \ - "python-dotenv==1.0.1" - - -FROM base as dbt-spark-dev - -HEALTHCHECK CMD python3 --version || exit 1 - -# send stdout/stderr to terminal -ENV PYTHONUNBUFFERED=1 - -# setup mount for local code -WORKDIR /opt/code -VOLUME /opt/code - -# create a virtual environment -RUN python3 -m venv /opt/venv From c1f735907856aea0f67d982050bebf1e31cd4f72 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 12:24:29 -0400 Subject: [PATCH 11/20] clean up test script --- docker-release/test.sh | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/docker-release/test.sh b/docker-release/test.sh index c51b681bc..7b277b5b4 100755 --- a/docker-release/test.sh +++ b/docker-release/test.sh @@ -1,22 +1,19 @@ # - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` # TODO: create a real test suite +set -e -clear \ -&& echo "\n\n"\ -"########################################\n"\ -"##### Testing dbt-spark latest #####\n"\ -"########################################\n"\ -&& docker build --tag dbt-spark \ - --target dbt-spark \ - docker \ -&& docker run dbt-spark --version \ -\ -&& echo "\n\n"\ -"#########################################\n"\ -"##### Testing dbt-spark-1.0.0b1 #####\n"\ -"#########################################\n"\ -&& docker build --tag dbt-spark-1.0.0b1 \ - --target dbt-spark \ - --build-arg commit_ref=v1.0.0b1 \ - docker \ -&& docker run dbt-spark-1.0.0b1 --version +echo "\n\n" +echo "####################################" +echo "##### Testing dbt-spark latest #####" +echo "####################################" + +docker build --tag dbt-spark --target dbt-spark docker +docker run dbt-spark --version + +echo "\n\n" +echo "#####################################" +echo "##### Testing dbt-spark-1.0.0b1 #####" +echo "#####################################" + +docker build --tag dbt-spark-1.0.0b1 --target dbt-spark --build-arg commit_ref=v1.0.0b1 docker +docker run dbt-spark-1.0.0b1 --version From 1d9fe5d1eee6571ded62d0ea4556c5fb8bf5d6fa Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 2 May 2024 12:25:04 -0400 Subject: [PATCH 12/20] Update docker-release/Dockerfile Co-authored-by: Emily Rockman --- docker-release/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-release/Dockerfile b/docker-release/Dockerfile index 588d5a256..ef4574ddd 100644 --- a/docker-release/Dockerfile +++ b/docker-release/Dockerfile @@ -1,5 +1,5 @@ # this image gets published to GHCR for production use -ARG py_version=3.10.7 +ARG py_version=3.11.2 FROM python:$py_version-slim-bullseye as base From 6c544c63e65ac0149ef12b910a98bc7a33848ee9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 12:37:09 -0400 Subject: [PATCH 13/20] rename the spark Dockerfile to make space for the release Dockerfile --- docker-compose.yml | 4 +++- docker/spark.Dockerfile | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 docker/spark.Dockerfile diff --git a/docker-compose.yml b/docker-compose.yml index ad083eaf4..cd3e1c776 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,9 @@ version: "3.7" services: dbt-spark3-thrift: - build: docker/ + build: + context: ./docker + dockerfile: spark.Dockerfile ports: - "10000:10000" - "4040:4040" diff --git a/docker/spark.Dockerfile b/docker/spark.Dockerfile new file mode 100644 index 000000000..bb4d378ed --- /dev/null +++ b/docker/spark.Dockerfile @@ -0,0 +1,30 @@ +ARG OPENJDK_VERSION=8 +FROM eclipse-temurin:${OPENJDK_VERSION}-jre + +ARG BUILD_DATE +ARG SPARK_VERSION=3.3.2 +ARG HADOOP_VERSION=3 + +LABEL org.label-schema.name="Apache Spark ${SPARK_VERSION}" \ + org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.version=$SPARK_VERSION + +ENV SPARK_HOME /usr/spark +ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}" + +RUN apt-get update && \ + apt-get install -y wget netcat procps libpostgresql-jdbc-java && \ + wget -q "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ + tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ + rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ + mv "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" /usr/spark && \ + ln -s /usr/share/java/postgresql-jdbc4.jar /usr/spark/jars/postgresql-jdbc4.jar && \ + apt-get remove -y wget && \ + apt-get autoremove -y && \ + apt-get clean + +COPY entrypoint.sh /scripts/ +RUN chmod +x /scripts/entrypoint.sh + +ENTRYPOINT ["/scripts/entrypoint.sh"] +CMD ["--help"] From 52ad6cba21211419b0803fc1af037b5d1f2077e9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 12:37:25 -0400 Subject: [PATCH 14/20] move the release Dockerfile into ./docker --- docker-release/Dockerfile | 42 ---------------- docker/Dockerfile | 72 ++++++++++++++++------------ {docker-release => docker}/README.md | 3 +- {docker-release => docker}/test.sh | 0 4 files changed, 43 insertions(+), 74 deletions(-) delete mode 100644 docker-release/Dockerfile rename {docker-release => docker}/README.md (94%) rename {docker-release => docker}/test.sh (100%) diff --git a/docker-release/Dockerfile b/docker-release/Dockerfile deleted file mode 100644 index ef4574ddd..000000000 --- a/docker-release/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# this image gets published to GHCR for production use -ARG py_version=3.11.2 - -FROM python:$py_version-slim-bullseye as base - -RUN apt-get update \ - && apt-get dist-upgrade -y \ - && apt-get install -y --no-install-recommends \ - build-essential=12.9 \ - ca-certificates=20210119 \ - gcc=4:10.2.1-1 \ - git=1:2.30.2-1+deb11u2 \ - libpq-dev=13.14-0+deb11u1 \ - libsasl2-dev=2.1.27+dfsg-2.1+deb11u1 \ - make=4.3-4.1 \ - openssh-client=1:8.4p1-5+deb11u3 \ - python-dev-is-python2=2.7.18-9 \ - software-properties-common=0.96.20.2-2.1 \ - unixodbc-dev=2.3.6-0.1+b1 \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* - -ENV PYTHONIOENCODING=utf-8 -ENV LANG=C.UTF-8 - -RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir - - -FROM base as dbt-spark - -ARG commit_ref=main -ARG extras=all - -HEALTHCHECK CMD dbt --version || exit 1 - -WORKDIR /usr/app/dbt/ -ENTRYPOINT ["dbt"] - -RUN python -m pip install --no-cache-dir "dbt-spark[${extras}] @ git+https://github.com/dbt-labs/dbt-spark@${commit_ref}" diff --git a/docker/Dockerfile b/docker/Dockerfile index bb4d378ed..ef4574ddd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,30 +1,42 @@ -ARG OPENJDK_VERSION=8 -FROM eclipse-temurin:${OPENJDK_VERSION}-jre - -ARG BUILD_DATE -ARG SPARK_VERSION=3.3.2 -ARG HADOOP_VERSION=3 - -LABEL org.label-schema.name="Apache Spark ${SPARK_VERSION}" \ - org.label-schema.build-date=$BUILD_DATE \ - org.label-schema.version=$SPARK_VERSION - -ENV SPARK_HOME /usr/spark -ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}" - -RUN apt-get update && \ - apt-get install -y wget netcat procps libpostgresql-jdbc-java && \ - wget -q "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ - mv "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" /usr/spark && \ - ln -s /usr/share/java/postgresql-jdbc4.jar /usr/spark/jars/postgresql-jdbc4.jar && \ - apt-get remove -y wget && \ - apt-get autoremove -y && \ - apt-get clean - -COPY entrypoint.sh /scripts/ -RUN chmod +x /scripts/entrypoint.sh - -ENTRYPOINT ["/scripts/entrypoint.sh"] -CMD ["--help"] +# this image gets published to GHCR for production use +ARG py_version=3.11.2 + +FROM python:$py_version-slim-bullseye as base + +RUN apt-get update \ + && apt-get dist-upgrade -y \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9 \ + ca-certificates=20210119 \ + gcc=4:10.2.1-1 \ + git=1:2.30.2-1+deb11u2 \ + libpq-dev=13.14-0+deb11u1 \ + libsasl2-dev=2.1.27+dfsg-2.1+deb11u1 \ + make=4.3-4.1 \ + openssh-client=1:8.4p1-5+deb11u3 \ + python-dev-is-python2=2.7.18-9 \ + software-properties-common=0.96.20.2-2.1 \ + unixodbc-dev=2.3.6-0.1+b1 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +ENV PYTHONIOENCODING=utf-8 +ENV LANG=C.UTF-8 + +RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir + + +FROM base as dbt-spark + +ARG commit_ref=main +ARG extras=all + +HEALTHCHECK CMD dbt --version || exit 1 + +WORKDIR /usr/app/dbt/ +ENTRYPOINT ["dbt"] + +RUN python -m pip install --no-cache-dir "dbt-spark[${extras}] @ git+https://github.com/dbt-labs/dbt-spark@${commit_ref}" diff --git a/docker-release/README.md b/docker/README.md similarity index 94% rename from docker-release/README.md rename to docker/README.md index 1b6bf6b6c..42ca5e227 100644 --- a/docker-release/README.md +++ b/docker/README.md @@ -1,6 +1,5 @@ # Docker for dbt -This docker file is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry. - +`Dockerfile` is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry. ## Building an image: This Dockerfile can create images for the following target: `dbt-spark` diff --git a/docker-release/test.sh b/docker/test.sh similarity index 100% rename from docker-release/test.sh rename to docker/test.sh From 5f3e52dcbfa4635bb359cdc230e9d9fd05225d5c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 12:46:18 -0400 Subject: [PATCH 15/20] move the release Dockerfile into ./docker --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cb9436cb6..548a95cb7 100644 --- a/Makefile +++ b/Makefile @@ -64,4 +64,4 @@ help: ## Show this help message. .PHONY: docker-prod docker-prod: - docker build -f docker-release/Dockerfile -t dbt-spark . + docker build -f docker/Dockerfile -t dbt-spark . From 2326b2da4b1e1fef51c52ce6aeab2a49c5a64cec Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 12:46:49 -0400 Subject: [PATCH 16/20] move the release Dockerfile into ./docker --- .github/dependabot.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 67ff2222a..fc44c9fe5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -20,8 +20,3 @@ updates: schedule: interval: "weekly" rebase-strategy: "disabled" - - package-ecosystem: "docker" - directory: "/docker-release" - schedule: - interval: "weekly" - rebase-strategy: "disabled" From b87999b3dccca05b16c1c3b6040b5fcffca76714 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 13:35:47 -0400 Subject: [PATCH 17/20] move the release Dockerfile into ./docker --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a6fb830b1..cdbdaa13f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -166,7 +166,7 @@ jobs: uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main with: version_number: ${{ inputs.version_number }} - dockerfile: "docker-release/Dockerfile" + dockerfile: "docker/Dockerfile" test_run: ${{ inputs.test_run }} slack-notification: From 4674f9f65807a7ed0dd701ca64de4cacc8661cb3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 14 May 2024 18:20:59 -0400 Subject: [PATCH 18/20] point to dev branch for now --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cdbdaa13f..0331712a6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -163,7 +163,7 @@ jobs: needs: [github-release] permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@latest-wrangler-new-docker with: version_number: ${{ inputs.version_number }} dockerfile: "docker/Dockerfile" From c7731c34520b5dacbe9f9e8944ad5b355f3b683a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 15 May 2024 15:39:36 -0400 Subject: [PATCH 19/20] point back to main --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0331712a6..cdbdaa13f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -163,7 +163,7 @@ jobs: needs: [github-release] permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@latest-wrangler-new-docker + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main with: version_number: ${{ inputs.version_number }} dockerfile: "docker/Dockerfile" From 562bebddc9ba8b955e1365c7abdd5e12fa8f3dd3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 20 May 2024 18:43:51 -0400 Subject: [PATCH 20/20] remove unused script --- docker/test.sh | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100755 docker/test.sh diff --git a/docker/test.sh b/docker/test.sh deleted file mode 100755 index 7b277b5b4..000000000 --- a/docker/test.sh +++ /dev/null @@ -1,19 +0,0 @@ -# - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` -# TODO: create a real test suite -set -e - -echo "\n\n" -echo "####################################" -echo "##### Testing dbt-spark latest #####" -echo "####################################" - -docker build --tag dbt-spark --target dbt-spark docker -docker run dbt-spark --version - -echo "\n\n" -echo "#####################################" -echo "##### Testing dbt-spark-1.0.0b1 #####" -echo "#####################################" - -docker build --tag dbt-spark-1.0.0b1 --target dbt-spark --build-arg commit_ref=v1.0.0b1 docker -docker run dbt-spark-1.0.0b1 --version