diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4673f47cf..ae2be43aa 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,3 +10,8 @@ updates: schedule: interval: "weekly" rebase-strategy: "disabled" + - package-ecosystem: "docker" + directory: "/docker" + schedule: + interval: "weekly" + rebase-strategy: "disabled" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 88942e251..6ff3d950e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,8 @@ # # **when?** # This workflow can be run manually on demand or can be called by other workflows -name: Release to GitHub and PyPI +name: "Release to GitHub, PyPI, and Docker" +run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI, and Docker" on: workflow_dispatch: @@ -131,9 +132,7 @@ jobs: bump-version-generate-changelog: name: Bump package version, Generate changelog - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main - with: sha: ${{ inputs.sha }} version_number: ${{ inputs.version_number }} @@ -141,17 +140,13 @@ jobs: env_setup_script_path: ${{ inputs.env_setup_script_path }} test_run: ${{ inputs.test_run }} nightly_release: ${{ inputs.nightly_release }} - secrets: inherit log-outputs-bump-version-generate-changelog: name: "[Log output] Bump package version, Generate changelog" if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog] - runs-on: ubuntu-latest - steps: - name: Print variables run: | @@ -162,9 +157,7 @@ jobs: name: Build, Test, Package if: ${{ !failure() && !cancelled() }} needs: [bump-version-generate-changelog] - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} @@ -174,7 +167,6 @@ jobs: package_test_command: ${{ inputs.package_test_command }} test_run: ${{ inputs.test_run }} nightly_release: ${{ inputs.nightly_release }} - secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -182,11 +174,8 @@ jobs: github-release: name: GitHub Release if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog, build-test-package] - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main - with: sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} version_number: ${{ inputs.version_number }} @@ -195,34 +184,38 @@ jobs: pypi-release: name: PyPI Release - - needs: [github-release] - + if: ${{ !failure() && !cancelled() }} + needs: [bump-version-generate-changelog, build-test-package] uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - with: version_number: ${{ inputs.version_number }} test_run: ${{ inputs.test_run }} - secrets: PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + docker-release: + name: "Docker Release" + if: ${{ !failure() && !cancelled() }} + needs: [bump-version-generate-changelog, build-test-package, github-release] + permissions: + packages: write + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + with: + version_number: ${{ inputs.version_number }} + test_run: ${{ inputs.test_run }} + slack-notification: name: Slack Notification if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} - needs: [ - bump-version-generate-changelog, - build-test-package, github-release, pypi-release, + docker-release, ] - uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main with: status: "failure" - secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/Makefile b/Makefile index c8f682a1c..fc1511a74 100644 --- a/Makefile +++ b/Makefile @@ -66,32 +66,11 @@ help: ## Show this help message. @echo 'targets:' @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -.PHONY: ubuntu-py38 -ubuntu-py38: - docker build -f docker_dev/ubuntu.Dockerfile -t dbt-snowflake-ubuntu-py38 . --build-arg version=3.8 - docker run --rm -it --name dbt-snowflake-ubuntu-py38 -v $(shell pwd):/opt/code dbt-snowflake-ubuntu-py38 - -.PHONY: ubuntu-py39 -ubuntu-py39: - docker build -f docker_dev/ubuntu.Dockerfile -t dbt-snowflake-ubuntu-py39 . --build-arg version=3.9 - docker run --rm -it --name dbt-snowflake-ubuntu-py39 -v $(shell pwd):/opt/code dbt-snowflake-ubuntu-py39 - -.PHONY: ubuntu-py310 -ubuntu-py310: - docker build -f docker_dev/ubuntu.Dockerfile -t dbt-snowflake-ubuntu-py310 . --build-arg version=3.10 - docker run --rm -it --name dbt-snowflake-ubuntu-py310 -v $(shell pwd):/opt/code dbt-snowflake-ubuntu-py310 - -.PHONY: ubuntu-py311 -ubuntu-py311: - docker build -f docker_dev/ubuntu.Dockerfile -t dbt-snowflake-ubuntu-py311 . --build-arg version=3.11 - docker run --rm -it --name dbt-snowflake-ubuntu-py311 -v $(shell pwd):/opt/code dbt-snowflake-ubuntu-py311 - -.PHONY: debian-py38 -debian-py38: - docker build -f docker/debian-py38.Dockerfile -t dbt-snowflake-debian-py38 . --build-arg version=3.8.15 - docker run --rm -it --name dbt-snowflake-debian-py38 -v $(shell pwd):/opt/code dbt-snowflake-debian-py38 - -.PHONY: dev-env-default -dev-env-default: - docker build -f docker/dev-env-default.Dockerfile -t dbt-snowflake-dev-env-default . - docker run --rm -it --name dbt-snowflake-dev-env-default -v $(shell pwd):/opt/code dbt-snowflake-dev-env-default +.PHONY: docker-dev +docker-dev: + docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev . + docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev + +.PHONY: docker-prod +docker-prod: + docker build -f docker/Dockerfile -t dbt-snowflake . diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..952d8dc39 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,37 @@ +# this image gets published to GHCR for production use +ARG py_version=3.10.7 + +FROM python:$py_version-slim-bullseye as base + +RUN apt-get update \ + && apt-get dist-upgrade -y \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9 \ + ca-certificates=20210119 \ + git=1:2.30.2-1+deb11u2 \ + libpq-dev=13.14-0+deb11u1 \ + make=4.3-4.1 \ + openssh-client=1:8.4p1-5+deb11u3 \ + software-properties-common=0.96.20.2-2.1 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +ENV PYTHONIOENCODING=utf-8 +ENV LANG=C.UTF-8 + +RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir + + +FROM base as dbt-snowflake + +ARG commit_ref=main + +HEALTHCHECK CMD dbt --version || exit 1 + +WORKDIR /usr/app/dbt/ +ENTRYPOINT ["dbt"] + +RUN python -m pip install --no-cache-dir "dbt-snowflake @ git+https://github.com/dbt-labs/dbt-snowflake@${commit_ref}" diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..95ecde101 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,58 @@ +# Docker for dbt +This docker file is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry. + + +## Building an image: +This Dockerfile can create images for the following target: `dbt-snowflake` + +In order to build a new image, run the following docker command. +```shell +docker build --tag --target dbt-snowflake +``` +--- +> **Note:** Docker must be configured to use [BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/) in order for images to build properly! + +--- + +By default the image will be populated with the latest version of `dbt-snowflake` on `main`. +If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +```shell +docker build --tag \ + --target dbt-snowflake \ + --build-arg commit_ref= \ + +``` + +### Examples: +To build an image named "my-dbt" that supports Snowflake using the latest releases: +```shell +cd dbt-core/docker +docker build --tag my-dbt --target dbt-snowflake . +``` + +To build an image named "my-other-dbt" that supports Snowflake using the adapter version 1.0.0b1: +```shell +cd dbt-core/docker +docker build \ + --tag my-other-dbt \ + --target dbt-snowflake \ + --build-arg commit_ref=v1.0.0b1 \ + . +``` + +## Running an image in a container: +The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: +```shell +docker run \ + --network=host \ + --mount type=bind,source=path/to/project,target=/usr/app \ + --mount type=bind,source=path/to/profiles.yml,target=/root/.dbt/profiles.yml \ + my-dbt \ + ls +``` +--- +**Notes:** +* Bind-mount sources _must_ be an absolute path +* You may need to make adjustments to the docker networking setting depending on the specifics of your data warehouse/database host. + +--- diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile new file mode 100644 index 000000000..0fc667048 --- /dev/null +++ b/docker/dev.Dockerfile @@ -0,0 +1,50 @@ +# this image does not get published, it is intended for local development only, see `Makefile` for usage +FROM ubuntu:22.04 as base + +# prevent python installation from asking for time zone region +ARG DEBIAN_FRONTEND=noninteractive + +# add python repository +RUN apt-get update \ + && apt-get install -y software-properties-common=0.99.22.9 \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# install python +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9ubuntu3 \ + git-all=1:2.34.1-1ubuntu1.10 \ + python3.8=3.8.19-1+jammy1 \ + python3.8-dev=3.8.19-1+jammy1 \ + python3.8-distutils=3.8.19-1+jammy1 \ + python3.8-venv=3.8.19-1+jammy1 \ + python3-pip=22.0.2+dfsg-1ubuntu0.4 \ + python3-wheel=0.37.1-2ubuntu0.22.04.1 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# update the default system interpreter to the newly installed version +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 + + +FROM base as dbt-snowflake-dev + +HEALTHCHECK CMD python3 --version || exit 1 + +# send stdout/stderr to terminal +ENV PYTHONUNBUFFERED=1 + +# setup mount for local code +WORKDIR /opt/code +VOLUME /opt/code + +# create a virtual environment +RUN python3 -m venv /opt/venv diff --git a/docker/test.sh b/docker/test.sh new file mode 100755 index 000000000..79311b6c9 --- /dev/null +++ b/docker/test.sh @@ -0,0 +1,22 @@ +# - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` +# TODO: create a real test suite + +clear \ +&& echo "\n\n"\ +"########################################\n"\ +"##### Testing dbt-snowflake latest #####\n"\ +"########################################\n"\ +&& docker build --tag dbt-snowflake \ + --target dbt-snowflake \ + docker \ +&& docker run dbt-snowflake --version \ +\ +&& echo "\n\n"\ +"#########################################\n"\ +"##### Testing dbt-snowflake-1.0.0b1 #####\n"\ +"#########################################\n"\ +&& docker build --tag dbt-snowflake-1.0.0b1 \ + --target dbt-snowflake \ + --build-arg commit_ref=v1.0.0b1 \ + docker \ +&& docker run dbt-snowflake-1.0.0b1 --version diff --git a/docker_dev/README.md b/docker_dev/README.md deleted file mode 100644 index dd487fea7..000000000 --- a/docker_dev/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Docker Dev Images - -These images are solely for development purposes. They are -saved here for convenience. There should be no expectation -of stability or maintenance. diff --git a/docker_dev/debian.Dockerfile b/docker_dev/debian.Dockerfile deleted file mode 100644 index 9c3415be5..000000000 --- a/docker_dev/debian.Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -FROM debian:latest - -# default to py3.11.1, this can be overridden at build, e.g. `docker build ... --build-arg version=3.10.8` -ARG version=3.11.1 - -# install python dependencies -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - build-essential \ - zlib1g-dev \ - libncurses5-dev \ - libgdbm-dev \ - libnss3-dev \ - libssl-dev \ - libreadline-dev \ - libffi-dev \ - libsqlite3-dev \ - wget \ - libbz2-dev \ - git-all - -# download, extract, and install python -RUN wget https://www.python.org/ftp/python/$version/Python-$version.tgz && \ - tar -xvf Python-$version.tgz && \ - cd Python-$version && \ - ./configure --enable-optimizations && \ - make -j $(shell nproc) && \ - make altinstall - -# clean up -RUN apt-get clean && \ - rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* \ - /Python-$version.tgz - -# add this installation to the path and update the default system interpreter to the newly installed version -RUN export PATH="/Python-$version:$PATH" && \ - update-alternatives --install /usr/bin/python3 python3 /Python-$version/python 1 - -# update python build tools -RUN python3 -m pip install --upgrade pip setuptools wheel --no-cache-dir - -# setup mount for our code -WORKDIR /opt/code -VOLUME /opt/code - -ENV PYTHONUNBUFFERED=1 diff --git a/docker_dev/dev-env-default.Dockerfile b/docker_dev/dev-env-default.Dockerfile deleted file mode 100644 index ed90889a7..000000000 --- a/docker_dev/dev-env-default.Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM docker/dev-environments-default:latest - -# install python and git (for installing dbt-core) -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - python3-pip \ - python3-wheel \ - build-essential - -# clean up -RUN apt-get clean && \ - rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* - -# update python build tools -RUN python3 -m pip install --upgrade pip setuptools wheel --no-cache-dir - -# setup mount for our code -WORKDIR /opt/code -VOLUME /opt/code - -# send stdout/stderr to terminal -ENV PYTHONUNBUFFERED=1 diff --git a/docker_dev/ubuntu.Dockerfile b/docker_dev/ubuntu.Dockerfile deleted file mode 100644 index bac3f5993..000000000 --- a/docker_dev/ubuntu.Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -FROM ubuntu:latest - -# default to py3.11, this can be overridden at build, e.g. `docker build ... --build-arg version=3.10` -ARG version=3.11 - -# prevent python installation from asking for time zone region -ARG DEBIAN_FRONTEND=noninteractive - -# get add-apt-repository -RUN apt-get update && \ - apt-get install -y software-properties-common - -# add the python repository -RUN apt-get update && \ - add-apt-repository -y ppa:deadsnakes/ppa - -# install python and git (for installing dbt-core) -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - python$version \ - python$version-dev \ - python$version-distutils \ - python$version-venv \ - python3-pip \ - python3-wheel \ - build-essential \ - git-all - -# clean up -RUN apt-get clean && \ - rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* - -# update the default system interpreter to the newly installed version -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python$version 1 - -# setup mount for our code -WORKDIR /opt/code -VOLUME /opt/code - -# install tox in the system interpreter (it creates it's own virtual environments) -RUN pip install tox - -# explicitly create a virtual environment as well for interactive testing -RUN python3 -m venv /opt/venv - -# send stdout/stderr to terminal -ENV PYTHONUNBUFFERED=1