From 35bd77c959b379096de1d3975aca37544cc860c1 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 2 Apr 2024 23:26:28 -0400 Subject: [PATCH 01/28] add docker release to the full release process for final releases --- .github/workflows/release.yml | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 88942e251..d4a009cd8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,8 @@ # # **when?** # This workflow can be run manually on demand or can be called by other workflows -name: Release to GitHub and PyPI +name: "Release to GitHub, PyPI & Docker" +run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI & Docker" on: workflow_dispatch: @@ -208,6 +209,31 @@ jobs: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + is-docker-release: + name: "Is Docker Release" + runs-on: ubuntu-latest + needs: [pypi-release] + outputs: + is-docker-release: ${{ steps.semver.outputs.is-pre-release == 0 }} + steps: + - name: "Audit version to determine if it is a pre-release" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version_number }} + + docker-release: + name: "Docker Release" + needs: [is-docker-release] + if: ${{ needs.is-docker-release.outputs.is-docker-release }} + permissions: + packages: write + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + with: + package: "dbt-snowflake" + version_number: ${{ inputs.version_number }} + test_run: ${{ inputs.test_run }} + slack-notification: name: Slack Notification if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} @@ -218,6 +244,7 @@ jobs: build-test-package, github-release, pypi-release, + docker-release, ] uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main From 08234515aeb62fea8940efcaefcd727c367e4686 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 2 Apr 2024 23:29:03 -0400 Subject: [PATCH 02/28] changelog --- .changes/unreleased/Under the Hood-20240402-232851.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20240402-232851.yaml diff --git a/.changes/unreleased/Under the Hood-20240402-232851.yaml b/.changes/unreleased/Under the Hood-20240402-232851.yaml new file mode 100644 index 000000000..c022c5d38 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240402-232851.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Automate the Docker release as part of the PyPI release +time: 2024-04-02T23:28:51.524626-04:00 +custom: + Author: mikealfare + Issue: "963" From ed33b0c4f3f10919c27d6bd652a47bf1e0c15c3d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 3 Apr 2024 12:27:56 -0400 Subject: [PATCH 03/28] add Dockerfile --- docker/Dockerfile | 48 ++++++++++++++++++++++++++++++++ docker/README.md | 70 +++++++++++++++++++++++++++++++++++++++++++++++ docker/test.sh | 22 +++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100755 docker/test.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..9d092ed07 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,48 @@ +## +# Generic dockerfile for dbt image building. +# See README for operational details +## + +# Top level build args +ARG build_for=linux/amd64 + +## +# base image (abstract) +## +# Please do not upgrade beyond python3.10.7 currently as dbt-spark does not support +# 3.11py and images do not get made properly +FROM --platform=$build_for python:3.10.7-slim-bullseye as base + +# N.B. The refs updated automagically every release via bumpversion +ARG dbt_snowflake_ref=dbt-snowflake@v1.7.10 + +# System setup +RUN apt-get update \ + && apt-get dist-upgrade -y \ + && apt-get install -y --no-install-recommends \ + git \ + ssh-client \ + software-properties-common \ + make \ + build-essential \ + ca-certificates \ + libpq-dev \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# Env vars +ENV PYTHONIOENCODING=utf-8 +ENV LANG=C.UTF-8 + +# Update python +RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir + +# Set docker basics +WORKDIR /usr/app/dbt/ +ENTRYPOINT ["dbt"] + +FROM base as dbt-snowflake +RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_snowflake_ref}#egg=dbt-snowflake" diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..4db5423c4 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,70 @@ +# Docker for dbt +This docker file is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry. + + +## Building an image: +This Dockerfile can create images for the following target: `dbt-snowflake` + +In order to build a new image, run the following docker command. +``` +docker build --tag --target dbt-snowflake +``` +--- +> **Note:** Docker must be configured to use [BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/) in order for images to build properly! + +--- + +By default the images will be populated with the most recent release of `dbt-snowflake`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +``` +docker build --tag \ + --target dbt-snowflake \ + --build-arg dbt_snowflake_ref= \ + +``` + +### Examples: +To build an image named "my-dbt" that supports Snowflake using the latest releases: +``` +cd dbt-core/docker +docker build --tag my-dbt --target dbt-snowflake . +``` + +To build an image named "my-other-dbt" that supports Snowflake using the adapter version 1.0.0b1: +``` +cd dbt-core/docker +docker build \ + --tag my-other-dbt \ + --target dbt-snowflake \ + --build-arg dbt_snowflake_ref=dbt-snowflake@v1.0.0b1 \ + . +``` + +## Special cases +There are a few special cases worth noting: + +* If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg: +``` +docker build --tag my_dbt \ + --target dbt-snowflake \ + --build-arg build_for=linux/arm64 \ + +``` + +Supported architectures can be found in the python docker [dockerhub page](https://hub.docker.com/_/python). + +## Running an image in a container: +The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: +``` +docker run \ + --network=host \ + --mount type=bind,source=path/to/project,target=/usr/app \ + --mount type=bind,source=path/to/profiles.yml,target=/root/.dbt/profiles.yml \ + my-dbt \ + ls +``` +--- +**Notes:** +* Bind-mount sources _must_ be an absolute path +* You may need to make adjustments to the docker networking setting depending on the specifics of your data warehouse/database host. + +--- diff --git a/docker/test.sh b/docker/test.sh new file mode 100755 index 000000000..e0a4649e9 --- /dev/null +++ b/docker/test.sh @@ -0,0 +1,22 @@ +# - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` +# TODO: create a real test suite + +clear \ +&& echo "\n\n"\ +"########################################\n"\ +"##### Testing dbt-snowflake latest #####\n"\ +"########################################\n"\ +&& docker build --tag dbt-snowflake \ + --target dbt-snowflake \ + docker \ +&& docker run dbt-snowflake --version \ +\ +&& echo "\n\n"\ +"#########################################\n"\ +"##### Testing dbt-snowflake-1.0.0b1 #####\n"\ +"#########################################\n"\ +&& docker build --tag dbt-snowflake-1.0.0b1 \ + --target dbt-snowflake \ + --build-arg dbt_snowflake_ref=dbt-snowflake@v1.0.0b1 \ + docker \ +&& docker run dbt-snowflake-1.0.0b1 --version From 50a27de6e9075d487cf58f33284ad2a5ad67f5d4 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 5 Apr 2024 18:09:50 -0400 Subject: [PATCH 04/28] simplify dockerfile --- docker/Dockerfile | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9d092ed07..8368a0aa2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,22 +1,10 @@ -## -# Generic dockerfile for dbt image building. -# See README for operational details -## - -# Top level build args ARG build_for=linux/amd64 -## -# base image (abstract) -## -# Please do not upgrade beyond python3.10.7 currently as dbt-spark does not support -# 3.11py and images do not get made properly FROM --platform=$build_for python:3.10.7-slim-bullseye as base -# N.B. The refs updated automagically every release via bumpversion +# ref is updated automatically every final release via bumpversion ARG dbt_snowflake_ref=dbt-snowflake@v1.7.10 -# System setup RUN apt-get update \ && apt-get dist-upgrade -y \ && apt-get install -y --no-install-recommends \ @@ -33,16 +21,13 @@ RUN apt-get update \ /tmp/* \ /var/tmp/* -# Env vars ENV PYTHONIOENCODING=utf-8 ENV LANG=C.UTF-8 -# Update python RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir -# Set docker basics WORKDIR /usr/app/dbt/ ENTRYPOINT ["dbt"] FROM base as dbt-snowflake -RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_snowflake_ref}#egg=dbt-snowflake" +RUN python -m pip install --no-cache-dir "dbt-snowflake @ git+https://github.com/dbt-labs/${dbt_snowflake_ref}" From cf97d6790ca5aeb933f98c3a0b8f84111de8f87a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 5 Apr 2024 18:59:30 -0400 Subject: [PATCH 05/28] simplify dockerfile --- docker/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/README.md b/docker/README.md index 4db5423c4..e7066a42d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,7 +6,7 @@ This docker file is suitable for building dbt Docker images locally or using wit This Dockerfile can create images for the following target: `dbt-snowflake` In order to build a new image, run the following docker command. -``` +```shell docker build --tag --target dbt-snowflake ``` --- @@ -15,7 +15,7 @@ docker build --tag --target dbt-snowflake --- By default the images will be populated with the most recent release of `dbt-snowflake`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag: -``` +```shell docker build --tag \ --target dbt-snowflake \ --build-arg dbt_snowflake_ref= \ @@ -24,13 +24,13 @@ docker build --tag \ ### Examples: To build an image named "my-dbt" that supports Snowflake using the latest releases: -``` +```shell cd dbt-core/docker docker build --tag my-dbt --target dbt-snowflake . ``` To build an image named "my-other-dbt" that supports Snowflake using the adapter version 1.0.0b1: -``` +```shell cd dbt-core/docker docker build \ --tag my-other-dbt \ @@ -43,7 +43,7 @@ docker build \ There are a few special cases worth noting: * If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg: -``` +```shell docker build --tag my_dbt \ --target dbt-snowflake \ --build-arg build_for=linux/arm64 \ @@ -54,7 +54,7 @@ Supported architectures can be found in the python docker [dockerhub page](https ## Running an image in a container: The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: -``` +```shell docker run \ --network=host \ --mount type=bind,source=path/to/project,target=/usr/app \ From a962d6a79ee9606d25b738f6d12956c6e570ea55 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 10 Apr 2024 14:22:29 -0400 Subject: [PATCH 06/28] move dev dockerfile into main dockerfile --- Makefile | 31 +++++++---------- docker/Dockerfile | 36 ++++++++++++++++--- docker_dev/README.md | 5 --- docker_dev/debian.Dockerfile | 49 -------------------------- docker_dev/dev-env-default.Dockerfile | 25 -------------- docker_dev/ubuntu.Dockerfile | 50 --------------------------- 6 files changed, 45 insertions(+), 151 deletions(-) delete mode 100644 docker_dev/README.md delete mode 100644 docker_dev/debian.Dockerfile delete mode 100644 docker_dev/dev-env-default.Dockerfile delete mode 100644 docker_dev/ubuntu.Dockerfile diff --git a/Makefile b/Makefile index c8f682a1c..907c4231e 100644 --- a/Makefile +++ b/Makefile @@ -66,32 +66,27 @@ help: ## Show this help message. @echo 'targets:' @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +.PHONY: ubuntu +ubuntu: + docker build -f docker/Dockerfile -t dbt-snowflake-dev . + docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev + .PHONY: ubuntu-py38 ubuntu-py38: - docker build -f docker_dev/ubuntu.Dockerfile -t dbt-snowflake-ubuntu-py38 . --build-arg version=3.8 - docker run --rm -it --name dbt-snowflake-ubuntu-py38 -v $(shell pwd):/opt/code dbt-snowflake-ubuntu-py38 + docker build -f docker/Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.8 + docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev .PHONY: ubuntu-py39 ubuntu-py39: - docker build -f docker_dev/ubuntu.Dockerfile -t dbt-snowflake-ubuntu-py39 . --build-arg version=3.9 - docker run --rm -it --name dbt-snowflake-ubuntu-py39 -v $(shell pwd):/opt/code dbt-snowflake-ubuntu-py39 + docker build -f docker/Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.9 + docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev .PHONY: ubuntu-py310 ubuntu-py310: - docker build -f docker_dev/ubuntu.Dockerfile -t dbt-snowflake-ubuntu-py310 . --build-arg version=3.10 - docker run --rm -it --name dbt-snowflake-ubuntu-py310 -v $(shell pwd):/opt/code dbt-snowflake-ubuntu-py310 + docker build -f docker/Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.10 + docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev .PHONY: ubuntu-py311 ubuntu-py311: - docker build -f docker_dev/ubuntu.Dockerfile -t dbt-snowflake-ubuntu-py311 . --build-arg version=3.11 - docker run --rm -it --name dbt-snowflake-ubuntu-py311 -v $(shell pwd):/opt/code dbt-snowflake-ubuntu-py311 - -.PHONY: debian-py38 -debian-py38: - docker build -f docker/debian-py38.Dockerfile -t dbt-snowflake-debian-py38 . --build-arg version=3.8.15 - docker run --rm -it --name dbt-snowflake-debian-py38 -v $(shell pwd):/opt/code dbt-snowflake-debian-py38 - -.PHONY: dev-env-default -dev-env-default: - docker build -f docker/dev-env-default.Dockerfile -t dbt-snowflake-dev-env-default . - docker run --rm -it --name dbt-snowflake-dev-env-default -v $(shell pwd):/opt/code dbt-snowflake-dev-env-default + docker build -f docker/Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.11 + docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev diff --git a/docker/Dockerfile b/docker/Dockerfile index 8368a0aa2..e42184d0d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,9 +1,9 @@ ARG build_for=linux/amd64 +ARG py_version=3.10.7 +ARG dbt_snowflake_ref=dbt-snowflake@main -FROM --platform=$build_for python:3.10.7-slim-bullseye as base -# ref is updated automatically every final release via bumpversion -ARG dbt_snowflake_ref=dbt-snowflake@v1.7.10 +FROM --platform=$build_for python:$py_version-slim-bullseye as base RUN apt-get update \ && apt-get dist-upgrade -y \ @@ -26,8 +26,36 @@ ENV LANG=C.UTF-8 RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir + +FROM base as dbt-snowflake + WORKDIR /usr/app/dbt/ ENTRYPOINT ["dbt"] -FROM base as dbt-snowflake RUN python -m pip install --no-cache-dir "dbt-snowflake @ git+https://github.com/dbt-labs/${dbt_snowflake_ref}" + + +FROM base as dbt-snowflake-dev + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git-all \ + libbz2-dev \ + libffi-dev \ + libgdbm-dev \ + libncurses5-dev \ + libnss3-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + wget \ + zlib1g-dev + +# setup mount for local code +WORKDIR /opt/code +VOLUME /opt/code + +ENTRYPOINT python -m pip install --no-cache-dir -e . -r dev-requirements.txt \ + && pre-commit install + +ENV PYTHONUNBUFFERED=1 diff --git a/docker_dev/README.md b/docker_dev/README.md deleted file mode 100644 index dd487fea7..000000000 --- a/docker_dev/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Docker Dev Images - -These images are solely for development purposes. They are -saved here for convenience. There should be no expectation -of stability or maintenance. diff --git a/docker_dev/debian.Dockerfile b/docker_dev/debian.Dockerfile deleted file mode 100644 index 9c3415be5..000000000 --- a/docker_dev/debian.Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -FROM debian:latest - -# default to py3.11.1, this can be overridden at build, e.g. `docker build ... --build-arg version=3.10.8` -ARG version=3.11.1 - -# install python dependencies -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - build-essential \ - zlib1g-dev \ - libncurses5-dev \ - libgdbm-dev \ - libnss3-dev \ - libssl-dev \ - libreadline-dev \ - libffi-dev \ - libsqlite3-dev \ - wget \ - libbz2-dev \ - git-all - -# download, extract, and install python -RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && \ - tar -xvf Python-$version.tgz && \ - cd Python-$version && \ - ./configure --enable-optimizations && \ - make -j $(shell nproc) && \ - make altinstall - -# clean up -RUN apt-get clean && \ - rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* \ - /Python-$version.tgz - -# add this installation to the path and update the default system interpreter to the newly installed version -RUN export PATH="/Python-$version:$PATH" && \ - update-alternatives --install /usr/bin/python3 python3 /Python-$version/python 1 - -# update python build tools -RUN python3 -m pip install --upgrade pip setuptools wheel --no-cache-dir - -# setup mount for our code -WORKDIR /opt/code -VOLUME /opt/code - -ENV PYTHONUNBUFFERED=1 diff --git a/docker_dev/dev-env-default.Dockerfile b/docker_dev/dev-env-default.Dockerfile deleted file mode 100644 index ed90889a7..000000000 --- a/docker_dev/dev-env-default.Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM docker/dev-environments-default:latest - -# install python and git (for installing dbt-core) -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - python3-pip \ - python3-wheel \ - build-essential - -# clean up -RUN apt-get clean && \ - rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* - -# update python build tools -RUN python3 -m pip install --upgrade pip setuptools wheel --no-cache-dir - -# setup mount for our code -WORKDIR /opt/code -VOLUME /opt/code - -# send stdout/stderr to terminal -ENV PYTHONUNBUFFERED=1 diff --git a/docker_dev/ubuntu.Dockerfile b/docker_dev/ubuntu.Dockerfile deleted file mode 100644 index bac3f5993..000000000 --- a/docker_dev/ubuntu.Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -FROM ubuntu:latest - -# default to py3.11, this can be overridden at build, e.g. `docker build ... --build-arg version=3.10` -ARG version=3.11 - -# prevent python installation from asking for time zone region -ARG DEBIAN_FRONTEND=noninteractive - -# get add-apt-repository -RUN apt-get update && \ - apt-get install -y software-properties-common - -# add the python repository -RUN apt-get update && \ - add-apt-repository -y ppa:deadsnakes/ppa - -# install python and git (for installing dbt-core) -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - python$version \ - python$version-dev \ - python$version-distutils \ - python$version-venv \ - python3-pip \ - python3-wheel \ - build-essential \ - git-all - -# clean up -RUN apt-get clean && \ - rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* - -# update the default system interpreter to the newly installed version -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python$version 1 - -# setup mount for our code -WORKDIR /opt/code -VOLUME /opt/code - -# install tox in the system interpreter (it creates it's own virtual environments) -RUN pip install tox - -# explicitly create a virtual environment as well for interactive testing -RUN python3 -m venv /opt/venv - -# send stdout/stderr to terminal -ENV PYTHONUNBUFFERED=1 From 402077caefce465978a813fe281bfe870b2b3c02 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 10 Apr 2024 18:14:52 -0400 Subject: [PATCH 07/28] add docker to dependabot config --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2a6f34492..285785506 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,3 +6,8 @@ updates: schedule: interval: "daily" rebase-strategy: "disabled" + - package-ecosystem: "docker" + directory: "/docker" + schedule: + interval: "weekly" + rebase-strategy: "disabled" From 5cd975b8d4ae1ce4231ce6df754fd17d16d46d20 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 11 Apr 2024 15:24:40 -0400 Subject: [PATCH 08/28] resolve release.yml feedback --- .github/workflows/release.yml | 43 ++++------------------------------- 1 file changed, 5 insertions(+), 38 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4a009cd8..a9e453e63 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -132,9 +132,7 @@ jobs: bump-version-generate-changelog: name: Bump package version, Generate changelog - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main - with: sha: ${{ inputs.sha }} version_number: ${{ inputs.version_number }} @@ -142,17 +140,13 @@ jobs: env_setup_script_path: ${{ inputs.env_setup_script_path }} test_run: ${{ inputs.test_run }} nightly_release: ${{ inputs.nightly_release }} - secrets: inherit log-outputs-bump-version-generate-changelog: name: "[Log output] Bump package version, Generate changelog" if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog] - runs-on: ubuntu-latest - steps: - name: Print variables run: | @@ -163,9 +157,7 @@ jobs: name: Build, Test, Package if: ${{ !failure() && !cancelled() }} needs: [bump-version-generate-changelog] - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} @@ -175,7 +167,6 @@ jobs: package_test_command: ${{ inputs.package_test_command }} test_run: ${{ inputs.test_run }} nightly_release: ${{ inputs.nightly_release }} - secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -183,11 +174,8 @@ jobs: github-release: name: GitHub Release if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main - with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} @@ -196,60 +184,39 @@ jobs: pypi-release: name: PyPI Release - - needs: [github-release] - + if: ${{ !failure() && !cancelled() }} + needs: [bump-version-generate-changelog, build-test-package] uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - with: version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} - secrets: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - is-docker-release: - name: "Is Docker Release" - runs-on: ubuntu-latest - needs: [pypi-release] - outputs: - is-docker-release: ${{ steps.semver.outputs.is-pre-release == 0 }} - steps: - - name: "Audit version to determine if it is a pre-release" - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ inputs.version_number }} - docker-release: name: "Docker Release" - needs: [is-docker-release] - if: ${{ needs.is-docker-release.outputs.is-docker-release }} + if: ${{ !failure() && !cancelled() }} + needs: [bump-version-generate-changelog, build-test-package] permissions: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main with: - package: "dbt-snowflake" + package: ${{ github.event.repository.name }} version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} slack-notification: name: Slack Notification if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} - needs: [ - bump-version-generate-changelog, - build-test-package, github-release, pypi-release, docker-release, ] - uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main with: status: "failure" - secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} From 383cf7e3c6a9b86f7685a72bb445dccbf520651c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 11 Apr 2024 15:28:18 -0400 Subject: [PATCH 09/28] add clarifying comments, distinguishing the dev and prod docker images --- docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index e42184d0d..13e3900f5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -27,6 +27,7 @@ ENV LANG=C.UTF-8 RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir +# this image gets published to GHCR for production use FROM base as dbt-snowflake WORKDIR /usr/app/dbt/ @@ -35,6 +36,7 @@ ENTRYPOINT ["dbt"] RUN python -m pip install --no-cache-dir "dbt-snowflake @ git+https://github.com/dbt-labs/${dbt_snowflake_ref}" +# this image does not get published, it is intended for local development only, see `Makefile` for usage FROM base as dbt-snowflake-dev RUN apt-get update && \ From 3464e2762b1e80536a68d740c7ecebf66f952f06 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 11 Apr 2024 15:41:21 -0400 Subject: [PATCH 10/28] docker linter updates --- docker/Dockerfile | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 13e3900f5..8a0e1b518 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -35,12 +35,14 @@ ENTRYPOINT ["dbt"] RUN python -m pip install --no-cache-dir "dbt-snowflake @ git+https://github.com/dbt-labs/${dbt_snowflake_ref}" +HEALTHCHECK CMD dbt --version || exit 1 + # this image does not get published, it is intended for local development only, see `Makefile` for usage FROM base as dbt-snowflake-dev -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ git-all \ libbz2-dev \ libffi-dev \ @@ -51,13 +53,19 @@ RUN apt-get update && \ libsqlite3-dev \ libssl-dev \ wget \ - zlib1g-dev + zlib1g-dev \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* # setup mount for local code WORKDIR /opt/code VOLUME /opt/code -ENTRYPOINT python -m pip install --no-cache-dir -e . -r dev-requirements.txt \ - && pre-commit install +ENTRYPOINT ["python", "-m", "pip", "install", "--no-cache-dir", "-e", ".", "-r dev-requirements.txt", "&&", "pre-commit install"] + +HEALTHCHECK CMD dbt --version || exit 1 ENV PYTHONUNBUFFERED=1 From 6083d8884336093cb3ddd53d983b411f253c7937 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 00:31:13 -0400 Subject: [PATCH 11/28] update dev Dockerfile --- docker/dev.Dockerfile | 49 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docker/dev.Dockerfile diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile new file mode 100644 index 000000000..09d97437d --- /dev/null +++ b/docker/dev.Dockerfile @@ -0,0 +1,49 @@ +# this image does not get published, it is intended for local development only, see `Makefile` for usage +FROM ubuntu:latest as base + +ARG py_version=3.11 + +# prevent python installation from asking for time zone region +ARG DEBIAN_FRONTEND=noninteractive + +# add python repository +RUN apt-get update \ + && apt-get install -y software-properties-common=0.99.22.9 \ + && add-apt-repository -y ppa:deadsnakes/ppa + +# install python +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9ubuntu3 \ + git-all=1:2.34.1-1ubuntu1.10 \ + python$py_version \ + python$py_version-dev \ + python$py_version-distutils \ + python$py_version-venv \ + python3-pip=22.0.2+dfsg-1ubuntu0.4 \ + python3-wheel=0.37.1-2ubuntu0.22.04.1 + +# cleanup +RUN apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# update the default system interpreter to the newly installed version +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python$py_version 1 + + +FROM base as dbt-snowflake-dev + +HEALTHCHECK CMD python3 --version || exit 1 + +# send stdout/stderr to terminal +ENV PYTHONUNBUFFERED=1 + +# setup mount for local code +WORKDIR /opt/code +VOLUME /opt/code + +# create a virtual environment +RUN python3 -m venv /opt/venv From e7066412b90e4e19187b36a5e024470b69ecebcc Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 00:31:33 -0400 Subject: [PATCH 12/28] add version pins to Dockerfile --- docker/Dockerfile | 61 ++++++++++++----------------------------------- 1 file changed, 15 insertions(+), 46 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8a0e1b518..563d69079 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,21 +1,22 @@ +# this image gets published to GHCR for production use ARG build_for=linux/amd64 ARG py_version=3.10.7 -ARG dbt_snowflake_ref=dbt-snowflake@main - FROM --platform=$build_for python:$py_version-slim-bullseye as base RUN apt-get update \ && apt-get dist-upgrade -y \ && apt-get install -y --no-install-recommends \ - git \ - ssh-client \ - software-properties-common \ - make \ - build-essential \ - ca-certificates \ - libpq-dev \ - && apt-get clean \ + build-essential=12.9 \ + ca-certificates=20210119 \ + git=1:2.30.2-1+deb11u2 \ + libpq-dev=13.14-0+deb11u1 \ + make=4.3-4.1 \ + openssh-client=1:8.4p1-5+deb11u3 \ + software-properties-common=0.96.20.2-2.1 + +# cleanup +RUN apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /tmp/* \ @@ -27,45 +28,13 @@ ENV LANG=C.UTF-8 RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir -# this image gets published to GHCR for production use FROM base as dbt-snowflake -WORKDIR /usr/app/dbt/ -ENTRYPOINT ["dbt"] - -RUN python -m pip install --no-cache-dir "dbt-snowflake @ git+https://github.com/dbt-labs/${dbt_snowflake_ref}" +ARG dbt_snowflake_ref=dbt-snowflake@main HEALTHCHECK CMD dbt --version || exit 1 +WORKDIR /usr/app/dbt/ +ENTRYPOINT ["dbt"] -# this image does not get published, it is intended for local development only, see `Makefile` for usage -FROM base as dbt-snowflake-dev - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - git-all \ - libbz2-dev \ - libffi-dev \ - libgdbm-dev \ - libncurses5-dev \ - libnss3-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - wget \ - zlib1g-dev \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* - -# setup mount for local code -WORKDIR /opt/code -VOLUME /opt/code - -ENTRYPOINT ["python", "-m", "pip", "install", "--no-cache-dir", "-e", ".", "-r dev-requirements.txt", "&&", "pre-commit install"] - -HEALTHCHECK CMD dbt --version || exit 1 - -ENV PYTHONUNBUFFERED=1 +RUN python -m pip install --no-cache-dir "dbt-snowflake @ git+https://github.com/dbt-labs/${dbt_snowflake_ref}" From 17ee3e13e8ceab750a0b381bb74614a349025603 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 00:32:08 -0400 Subject: [PATCH 13/28] add and update make recipes for both versions --- Makefile | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 907c4231e..0cb565f4b 100644 --- a/Makefile +++ b/Makefile @@ -68,25 +68,30 @@ help: ## Show this help message. .PHONY: ubuntu ubuntu: - docker build -f docker/Dockerfile -t dbt-snowflake-dev . + docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev .PHONY: ubuntu-py38 ubuntu-py38: - docker build -f docker/Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.8 + docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.8 docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev .PHONY: ubuntu-py39 ubuntu-py39: - docker build -f docker/Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.9 + docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.9 docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev .PHONY: ubuntu-py310 ubuntu-py310: - docker build -f docker/Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.10 + docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.10 docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev .PHONY: ubuntu-py311 ubuntu-py311: - docker build -f docker/Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.11 + docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.11 docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev + +.PHONY: docker-prod +docker-prod: + docker build -f docker/Dockerfile -t dbt-snowflake . --build-arg build_for=linux/arm64/v8 + docker run --rm -it --name dbt-snowflake -v $(shell pwd):/opt/code dbt-snowflake From 818c80f757673e9bbb6a2920a33416d9265913b5 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 00:51:06 -0400 Subject: [PATCH 14/28] address wiz linter recommendations --- docker/Dockerfile | 6 ++---- docker/dev.Dockerfile | 15 +++++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 563d69079..f86ee154b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,10 +13,8 @@ RUN apt-get update \ libpq-dev=13.14-0+deb11u1 \ make=4.3-4.1 \ openssh-client=1:8.4p1-5+deb11u3 \ - software-properties-common=0.96.20.2-2.1 - -# cleanup -RUN apt-get clean \ + software-properties-common=0.96.20.2-2.1 \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /tmp/* \ diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile index 09d97437d..53570673e 100644 --- a/docker/dev.Dockerfile +++ b/docker/dev.Dockerfile @@ -1,5 +1,5 @@ # this image does not get published, it is intended for local development only, see `Makefile` for usage -FROM ubuntu:latest as base +FROM ubuntu:22.04 as base ARG py_version=3.11 @@ -9,7 +9,12 @@ ARG DEBIAN_FRONTEND=noninteractive # add python repository RUN apt-get update \ && apt-get install -y software-properties-common=0.99.22.9 \ - && add-apt-repository -y ppa:deadsnakes/ppa + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* # install python RUN apt-get update \ @@ -21,10 +26,8 @@ RUN apt-get update \ python$py_version-distutils \ python$py_version-venv \ python3-pip=22.0.2+dfsg-1ubuntu0.4 \ - python3-wheel=0.37.1-2ubuntu0.22.04.1 - -# cleanup -RUN apt-get clean \ + python3-wheel=0.37.1-2ubuntu0.22.04.1 \ + && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ /tmp/* \ From ae8b88364662e0ff4e8af1276e7d5945aa7346cc Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 01:01:31 -0400 Subject: [PATCH 15/28] pin dev docker to py3.8 --- Makefile | 24 ++---------------------- docker/dev.Dockerfile | 12 +++++------- 2 files changed, 7 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 0cb565f4b..3059025b7 100644 --- a/Makefile +++ b/Makefile @@ -66,31 +66,11 @@ help: ## Show this help message. @echo 'targets:' @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -.PHONY: ubuntu -ubuntu: +.PHONY: docker-dev +docker-dev: docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev -.PHONY: ubuntu-py38 -ubuntu-py38: - docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.8 - docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev - -.PHONY: ubuntu-py39 -ubuntu-py39: - docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.9 - docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev - -.PHONY: ubuntu-py310 -ubuntu-py310: - docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.10 - docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev - -.PHONY: ubuntu-py311 -ubuntu-py311: - docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . --build-arg py_version=3.11 - docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev - .PHONY: docker-prod docker-prod: docker build -f docker/Dockerfile -t dbt-snowflake . --build-arg build_for=linux/arm64/v8 diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile index 53570673e..0fc667048 100644 --- a/docker/dev.Dockerfile +++ b/docker/dev.Dockerfile @@ -1,8 +1,6 @@ # this image does not get published, it is intended for local development only, see `Makefile` for usage FROM ubuntu:22.04 as base -ARG py_version=3.11 - # prevent python installation from asking for time zone region ARG DEBIAN_FRONTEND=noninteractive @@ -21,10 +19,10 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential=12.9ubuntu3 \ git-all=1:2.34.1-1ubuntu1.10 \ - python$py_version \ - python$py_version-dev \ - python$py_version-distutils \ - python$py_version-venv \ + python3.8=3.8.19-1+jammy1 \ + python3.8-dev=3.8.19-1+jammy1 \ + python3.8-distutils=3.8.19-1+jammy1 \ + python3.8-venv=3.8.19-1+jammy1 \ python3-pip=22.0.2+dfsg-1ubuntu0.4 \ python3-wheel=0.37.1-2ubuntu0.22.04.1 \ && apt-get clean \ @@ -34,7 +32,7 @@ RUN apt-get update \ /var/tmp/* # update the default system interpreter to the newly installed version -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python$py_version 1 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 FROM base as dbt-snowflake-dev From a7fef7ffbc63242530f11cf07896714cf4245373 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 01:08:29 -0400 Subject: [PATCH 16/28] point to dev branch for docker release - revert this --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a9e453e63..9d80dcd00 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -200,7 +200,7 @@ jobs: needs: [bump-version-generate-changelog, build-test-package] permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@er/update-local-run-docker with: package: ${{ github.event.repository.name }} version_number: ${{ inputs.version_number }} From 6d83dc2191b0d3d71e0cef070f9dfd7080f257c0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 01:10:19 -0400 Subject: [PATCH 17/28] update release title --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9d80dcd00..f6596d2ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,8 +13,8 @@ # # **when?** # This workflow can be run manually on demand or can be called by other workflows -name: "Release to GitHub, PyPI & Docker" -run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI & Docker" +name: "Release to GitHub, PyPI, and Docker" +run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI, and Docker" on: workflow_dispatch: From a07f85338ae3f2ebf05f8a45d11163a5cc30addf Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 12:21:00 -0400 Subject: [PATCH 18/28] point back to main --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f6596d2ee..33df839be 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -200,7 +200,7 @@ jobs: needs: [bump-version-generate-changelog, build-test-package] permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@er/update-local-run-docker + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main with: package: ${{ github.event.repository.name }} version_number: ${{ inputs.version_number }} From d11a563408749ed566ffdb8be4b915845e6c310a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 16:58:19 -0400 Subject: [PATCH 19/28] remove platform build arg, pin python deps, update repo ref to commit ref --- docker/Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f86ee154b..952d8dc39 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,8 +1,7 @@ # this image gets published to GHCR for production use -ARG build_for=linux/amd64 ARG py_version=3.10.7 -FROM --platform=$build_for python:$py_version-slim-bullseye as base +FROM python:$py_version-slim-bullseye as base RUN apt-get update \ && apt-get dist-upgrade -y \ @@ -23,16 +22,16 @@ RUN apt-get update \ ENV PYTHONIOENCODING=utf-8 ENV LANG=C.UTF-8 -RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir +RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir FROM base as dbt-snowflake -ARG dbt_snowflake_ref=dbt-snowflake@main +ARG commit_ref=main HEALTHCHECK CMD dbt --version || exit 1 WORKDIR /usr/app/dbt/ ENTRYPOINT ["dbt"] -RUN python -m pip install --no-cache-dir "dbt-snowflake @ git+https://github.com/dbt-labs/${dbt_snowflake_ref}" +RUN python -m pip install --no-cache-dir "dbt-snowflake @ git+https://github.com/dbt-labs/dbt-snowflake@${commit_ref}" From 8ab5be19364a4afe194ceb3a9599e3bdb6e6aeca Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 17:08:29 -0400 Subject: [PATCH 20/28] docker release requires the github release tag --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 33df839be..4a2842180 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -197,7 +197,7 @@ jobs: docker-release: name: "Docker Release" if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog, build-test-package] + needs: [bump-version-generate-changelog, build-test-package, github-release] permissions: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main From 699638313d491300085a5e5e17711673b3b67251 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 17:08:56 -0400 Subject: [PATCH 21/28] point to dev branch - revert this --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4a2842180..423203fc1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -200,7 +200,7 @@ jobs: needs: [bump-version-generate-changelog, build-test-package, github-release] permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release with: package: ${{ github.event.repository.name }} version_number: ${{ inputs.version_number }} From d14e119f7e1bbab533ee02be019e69bcb2a70172 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 21:48:57 -0400 Subject: [PATCH 22/28] remove unused build arg --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3059025b7..8eae9ca8c 100644 --- a/Makefile +++ b/Makefile @@ -73,5 +73,5 @@ docker-dev: .PHONY: docker-prod docker-prod: - docker build -f docker/Dockerfile -t dbt-snowflake . --build-arg build_for=linux/arm64/v8 + docker build -f docker/Dockerfile -t dbt-snowflake . docker run --rm -it --name dbt-snowflake -v $(shell pwd):/opt/code dbt-snowflake From cb1f580f6faa58642ffd46ffd95eb81202364f46 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 21:54:17 -0400 Subject: [PATCH 23/28] remove unused docker run --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 8eae9ca8c..fc1511a74 100644 --- a/Makefile +++ b/Makefile @@ -74,4 +74,3 @@ docker-dev: .PHONY: docker-prod docker-prod: docker build -f docker/Dockerfile -t dbt-snowflake . - docker run --rm -it --name dbt-snowflake -v $(shell pwd):/opt/code dbt-snowflake From 8e467e64af4d0a01ec5ee504148e8874f2e3bdaa Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 22:01:07 -0400 Subject: [PATCH 24/28] update readme to align with the new container --- docker/README.md | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/docker/README.md b/docker/README.md index e7066a42d..95ecde101 100644 --- a/docker/README.md +++ b/docker/README.md @@ -14,11 +14,12 @@ docker build --tag --target dbt-snowflake --- -By default the images will be populated with the most recent release of `dbt-snowflake`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +By default the image will be populated with the latest version of `dbt-snowflake` on `main`. +If you need to use a different version you can specify it by git ref using the `--build-arg` flag: ```shell docker build --tag \ --target dbt-snowflake \ - --build-arg dbt_snowflake_ref= \ + --build-arg commit_ref= \ ``` @@ -35,23 +36,10 @@ cd dbt-core/docker docker build \ --tag my-other-dbt \ --target dbt-snowflake \ - --build-arg dbt_snowflake_ref=dbt-snowflake@v1.0.0b1 \ + --build-arg commit_ref=v1.0.0b1 \ . ``` -## Special cases -There are a few special cases worth noting: - -* If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg: -```shell -docker build --tag my_dbt \ - --target dbt-snowflake \ - --build-arg build_for=linux/arm64 \ - -``` - -Supported architectures can be found in the python docker [dockerhub page](https://hub.docker.com/_/python). - ## Running an image in a container: The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: ```shell From 26afa2ac65ae9dde71ba1878682fce4f36138cb2 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 22:35:13 -0400 Subject: [PATCH 25/28] update build arg in test --- docker/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test.sh b/docker/test.sh index e0a4649e9..79311b6c9 100755 --- a/docker/test.sh +++ b/docker/test.sh @@ -17,6 +17,6 @@ clear \ "#########################################\n"\ && docker build --tag dbt-snowflake-1.0.0b1 \ --target dbt-snowflake \ - --build-arg dbt_snowflake_ref=dbt-snowflake@v1.0.0b1 \ + --build-arg commit_ref=v1.0.0b1 \ docker \ && docker run dbt-snowflake-1.0.0b1 --version From 351df924ce1f993695aad37eedba7d54fffada0d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 10:25:37 -0400 Subject: [PATCH 26/28] remove defaulted package name for docker release --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 423203fc1..9355c9297 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -202,7 +202,6 @@ jobs: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release with: - package: ${{ github.event.repository.name }} version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} From a3fe51f369b171b84fd195189c500673d0560ba0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 17:27:16 -0400 Subject: [PATCH 27/28] point back to main --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9355c9297..6ff3d950e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -200,7 +200,7 @@ jobs: needs: [bump-version-generate-changelog, build-test-package, github-release] permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main with: version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} From 36a5a56731b4fa5abfa3fd08704dc38e262ad6f0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 17:32:01 -0400 Subject: [PATCH 28/28] remove changie entry --- .changes/unreleased/Under the Hood-20240402-232851.yaml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .changes/unreleased/Under the Hood-20240402-232851.yaml diff --git a/.changes/unreleased/Under the Hood-20240402-232851.yaml b/.changes/unreleased/Under the Hood-20240402-232851.yaml deleted file mode 100644 index c022c5d38..000000000 --- a/.changes/unreleased/Under the Hood-20240402-232851.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Automate the Docker release as part of the PyPI release -time: 2024-04-02T23:28:51.524626-04:00 -custom: - Author: mikealfare - Issue: "963"