update docker release to align with other adapters, add dev docker

dbt-labs · Apr 13, 2024 · 548f7df · 548f7df
1 parent 43e100c
commit 548f7df
Show file tree

Hide file tree

Showing 6 changed files with 108 additions and 85 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -13,8 +13,8 @@
 # This will only run manually. Run this workflow only after the
 # version bump workflow is completed and related changes are reviewed and merged.
 #
-
-name: Release to GitHub and PyPI
+name: "Release to GitHub, PyPI, and Docker"
+run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI, and Docker"
 
 on:
   workflow_dispatch:
@@ -88,26 +88,20 @@ jobs:
   # so that the integration tests stage will be skipped.
   audit-version-and-changelog:
     name: Bump package version, Generate changelog
-
     uses: dbt-labs/dbt-spark/.github/workflows/release-prep.yml@main
-
     with:
       sha: ${{ inputs.sha }}
       version_number: ${{ inputs.version_number }}
       target_branch: ${{ inputs.target_branch }}
       env_setup_script_path: ""
       test_run: ${{ inputs.test_run }}
-
     secrets: inherit
 
   log-outputs-audit-version-and-changelog:
     name: "[Log output] Bump package version, Generate changelog"
     if: ${{ !failure() && !cancelled() }}
-
     needs: [audit-version-and-changelog]
-
     runs-on: ubuntu-latest
-
     steps:
       - name: Print variables
         run: |
@@ -118,9 +112,7 @@ jobs:
     name: Build, Test, Package
     if: ${{ !failure() && !cancelled() }}
     needs: [audit-version-and-changelog]
-
     uses: dbt-labs/dbt-release/.github/workflows/build.yml@main
-
     with:
       sha: ${{ needs.audit-version-and-changelog.outputs.final_sha }}
       version_number: ${{ inputs.version_number }}
@@ -129,19 +121,15 @@ jobs:
       s3_bucket_name: ${{ inputs.s3_bucket_name }}
       package_test_command: ${{ inputs.package_test_command }}
       test_run: ${{ inputs.test_run }}
-
     secrets:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
   github-release:
     name: GitHub Release
     if: ${{ !failure() && !cancelled() }}
-
     needs: [audit-version-and-changelog, build-test-package]
-
     uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main
-
     with:
       sha: ${{ needs.audit-version-and-changelog.outputs.final_sha }}
       version_number: ${{ inputs.version_number }}
@@ -150,59 +138,38 @@ jobs:
 
   pypi-release:
     name: PyPI Release
-
     needs: [github-release]
-
     uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main
-
     with:
       version_number: ${{ inputs.version_number }}
       test_run: ${{ inputs.test_run }}
-
     secrets:
       PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
       TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
 
-  is-docker-release:
-    name: "Is Docker Release"
-    runs-on: ubuntu-latest
-    needs: [pypi-release]
-    outputs:
-      is-docker-release: ${{ steps.semver.outputs.is-pre-release == 0 }}
-    steps:
-      - name: "Audit version to determine if it is a pre-release"
-        id: semver
-        uses: dbt-labs/actions/[email protected]
-        with:
-          version: ${{ inputs.version_number }}
-
   docker-release:
     name: "Docker Release"
-    needs: [is-docker-release]
-    if: ${{ needs.is-docker-release.outputs.is-docker-release }}
+    if: ${{ !failure() && !cancelled() }}
+    needs: [bump-version-generate-changelog, build-test-package, github-release]
     permissions:
       packages: write
-    uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main
+    uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release
     with:
-      package: "dbt-spark"
+      package: ${{ github.event.repository.name }}
       version_number: ${{ inputs.version_number }}
       test_run: ${{ inputs.test_run }}
 
   slack-notification:
     name: Slack Notification
     if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }}
-
     needs:
       [
-        audit-version-and-changelog,
-        build-test-package,
         github-release,
         pypi-release,
+        docker-release,
       ]
-
     uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main
     with:
       status: "failure"
-
     secrets:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_CORE_ALERTS }}
diff --git a/Makefile b/Makefile
@@ -61,3 +61,13 @@ help: ## Show this help message.
 	@echo
 	@echo 'targets:'
 	@grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+
+.PHONY: docker-dev
+docker-dev:
+	docker build -f docker-dev/Dockerfile -t dbt-spark-dev .
+	docker run --rm -it --name dbt-spark-dev -v $(shell pwd):/opt/code dbt-spark-dev
+
+.PHONY: docker-prod
+docker-prod:
+	docker build -f docker-release/Dockerfile -t dbt-spark .
diff --git a/docker-dev/Dockerfile b/docker-dev/Dockerfile
@@ -0,0 +1,59 @@
+# this image does not get published, it is intended for local development only, see `Makefile` for usage
+FROM ubuntu:22.04 as base
+
+# prevent python installation from asking for time zone region
+ARG DEBIAN_FRONTEND=noninteractive
+
+# add python repository
+RUN apt-get update \
+  && apt-get install -y software-properties-common=0.99.22.9 \
+  && add-apt-repository -y ppa:deadsnakes/ppa \
+  && apt-get clean \
+  && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
+
+# install python
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+    build-essential=12.9ubuntu3 \
+    gcc=4:11.2.0-1ubuntu1 \
+    git-all=1:2.34.1-1ubuntu1.10 \
+    libsasl2-dev=2.1.27+dfsg2-3ubuntu1.2 \
+    python3.10=3.10.12-1~22.04.3 \
+    python3.10-dev=3.10.12-1~22.04.3 \
+    python3.10-venv=3.10.12-1~22.04.3 \
+    python3-distutils=3.10.8-1~22.04 \
+    python3-pip=22.0.2+dfsg-1ubuntu0.4 \
+    python3-wheel=0.37.1-2ubuntu0.22.04.1 \
+    unixodbc-dev=2.3.9-5ubuntu0.1 \
+  && apt-get clean \
+  && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
+
+# update the default system interpreter to the newly installed version
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1
+
+# install python dependencies
+RUN python3 -m pip install --upgrade --no-cache-dir  \
+    "beartype==0.17.2" \
+    "dagger-io==0.10.3" \
+    "python-dotenv==1.0.1"
+
+
+FROM base as dbt-spark-dev
+
+HEALTHCHECK CMD python3 --version || exit 1
+
+# send stdout/stderr to terminal
+ENV PYTHONUNBUFFERED=1
+
+# setup mount for local code
+WORKDIR /opt/code
+VOLUME /opt/code
+
+# create a virtual environment
+RUN python3 -m venv /opt/venv
diff --git a/docker-release/Dockerfile b/docker-release/Dockerfile
@@ -1,21 +1,22 @@
-ARG build_for=linux/amd64
+# this image gets published to GHCR for production use
+ARG py_version=3.10.7
 
-FROM --platform=$build_for python:3.10.7-slim-bullseye as base
-
-# ref is updated automatically every final release via bumpversion
-ARG [email protected]
-ARG dbt_spark_version=all
+FROM python:$py_version-slim-bullseye as base
 
 RUN apt-get update \
   && apt-get dist-upgrade -y \
   && apt-get install -y --no-install-recommends \
-    git \
-    ssh-client \
-    software-properties-common \
-    make \
-    build-essential \
-    ca-certificates \
-    libpq-dev \
+    build-essential=12.9 \
+    ca-certificates=20210119 \
+    gcc=4:10.2.1-1 \
+    git=1:2.30.2-1+deb11u2 \
+    libpq-dev=13.14-0+deb11u1 \
+    libsasl2-dev=2.1.27+dfsg-2.1+deb11u1 \
+    make=4.3-4.1 \
+    openssh-client=1:8.4p1-5+deb11u3 \
+    python-dev-is-python2=2.7.18-9 \
+    software-properties-common=0.96.20.2-2.1 \
+    unixodbc-dev=2.3.6-0.1+b1 \
   && apt-get clean \
   && rm -rf \
     /var/lib/apt/lists/* \
@@ -25,22 +26,17 @@ RUN apt-get update \
 ENV PYTHONIOENCODING=utf-8
 ENV LANG=C.UTF-8
 
-RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir
+RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir
+
+
+FROM base as dbt-spark
+
+ARG commit_ref=main
+ARG extras=all
+
+HEALTHCHECK CMD dbt --version || exit 1
 
 WORKDIR /usr/app/dbt/
 ENTRYPOINT ["dbt"]
 
-FROM base as dbt-spark
-RUN apt-get update \
-  && apt-get dist-upgrade -y \
-  && apt-get install -y --no-install-recommends \
-    python-dev \
-    libsasl2-dev \
-    gcc \
-    unixodbc-dev \
-  && apt-get clean \
-  && rm -rf \
-    /var/lib/apt/lists/* \
-    /tmp/* \
-    /var/tmp/*
-RUN python -m pip install --no-cache-dir "dbt-spark[${dbt_spark_version}] @ git+https://github.com/dbt-labs/${dbt_spark_ref}"
+RUN python -m pip install --no-cache-dir "dbt-spark[${extras}] @ git+https://github.com/dbt-labs/dbt-spark@${commit_ref}"
diff --git a/docker-release/README.md b/docker-release/README.md
@@ -14,11 +14,12 @@ docker build --tag <your_image_name> --target dbt-spark <path/to/dockerfile>
 
 ---
 
-By default the images will be populated with the most recent release of `dbt-spark`.  If you need to use a different version you can specify it by git ref using the `--build-arg` flag:
+By default the image will be populated with the latest version of `dbt-spark` on `main`.
+If you need to use a different version you can specify it by git ref using the `--build-arg` flag:
 ```shell
 docker build --tag <your_image_name> \
   --target dbt-spark \
-  --build-arg dbt_spark_ref=<git_ref> \
+  --build-arg commit_ref=<commit_ref> \
   <path/to/dockerfile>
 ```
 
@@ -35,33 +36,23 @@ cd dbt-core/docker
 docker build \
   --tag my-other-dbt \
   --target dbt-spark \
-  --build-arg dbt_spark_ref=dbt-spark@v1.0.0b1 \
+  --build-arg commit_ref=v1.0.0b1 \
  .
 ```
 
 ## Special cases
 There are a few special cases worth noting:
 * The `dbt-spark` database adapter comes in three different versions named `PyHive`, `ODBC`, and the default `all`.
-If you wish to override this you can use the `--build-arg` flag with the value of `dbt_spark_version=<version_name>`.
+If you wish to override this you can use the `--build-arg` flag with the value of `extras=<extras_name>`.
 See the [docs](https://docs.getdbt.com/reference/warehouse-profiles/spark-profile) for more information.
 ```shell
 docker build --tag my_dbt \
   --target dbt-spark \
-  --build-arg dbt_spark_ref=dbt-spark@1.0.0b1 \
-  --build-arg dbt_spark_version=PyHive \
+  --build-arg commit_ref=v1.0.0b1 \
+  --build-arg extras=PyHive \
   <path/to/dockerfile>
 ```
 
-* If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg:
-```shell
-docker build --tag my_dbt \
-  --target dbt-spark \
-  --build-arg build_for=linux/arm64 \
-  <path/to/dockerfile>
-```
-
-Supported architectures can be found in the python docker [dockerhub page](https://hub.docker.com/_/python).
-
 ## Running an image in a container:
 The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal:
 ```shell

diff --git a/docker-release/test.sh b/docker-release/test.sh
@@ -17,6 +17,6 @@ clear \
 "#########################################\n"\
 && docker build --tag dbt-spark-1.0.0b1 \
   --target dbt-spark \
-  --build-arg dbt_spark_ref=dbt-spark@v1.0.0b1 \
+  --build-arg commit_ref=v1.0.0b1 \
   docker \
 && docker run dbt-spark-1.0.0b1 --version