Skip to content

Commit

Permalink
update to align with other adapters
Browse files Browse the repository at this point in the history
  • Loading branch information
mikealfare committed Apr 13, 2024
1 parent dced9c5 commit b4b3f01
Show file tree
Hide file tree
Showing 5 changed files with 92 additions and 72 deletions.
46 changes: 7 additions & 39 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
#
# **when?**
# This workflow can be run manually on demand or can be called by other workflows
name: Release to GitHub and PyPI
name: "Release to GitHub, PyPI, and Docker"
run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI, and Docker"

on:
workflow_dispatch:
Expand Down Expand Up @@ -131,27 +132,21 @@ jobs:
bump-version-generate-changelog:
name: Bump package version, Generate changelog

uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main

with:
sha: ${{ inputs.sha }}
version_number: ${{ inputs.version_number }}
target_branch: ${{ inputs.target_branch }}
env_setup_script_path: ${{ inputs.env_setup_script_path }}
test_run: ${{ inputs.test_run }}
nightly_release: ${{ inputs.nightly_release }}

secrets: inherit

log-outputs-bump-version-generate-changelog:
name: "[Log output] Bump package version, Generate changelog"
if: ${{ !failure() && !cancelled() }}

needs: [bump-version-generate-changelog]

runs-on: ubuntu-latest

steps:
- name: Print variables
run: |
Expand All @@ -162,9 +157,7 @@ jobs:
name: Build, Test, Package
if: ${{ !failure() && !cancelled() }}
needs: [bump-version-generate-changelog]

uses: dbt-labs/dbt-release/.github/workflows/build.yml@main

with:
sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
version_number: ${{ inputs.version_number }}
Expand All @@ -174,19 +167,15 @@ jobs:
package_test_command: ${{ inputs.package_test_command }}
test_run: ${{ inputs.test_run }}
nightly_release: ${{ inputs.nightly_release }}

secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

github-release:
name: GitHub Release
if: ${{ !failure() && !cancelled() }}

needs: [bump-version-generate-changelog, build-test-package]

uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main

with:
sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
version_number: ${{ inputs.version_number }}
Expand All @@ -195,59 +184,38 @@ jobs:

pypi-release:
name: PyPI Release

needs: [github-release]

uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main

with:
version_number: ${{ inputs.version_number }}
test_run: ${{ inputs.test_run }}

secrets:
PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}

is-docker-release:
name: "Is Docker Release"
runs-on: ubuntu-latest
needs: [pypi-release]
outputs:
is-docker-release: ${{ steps.semver.outputs.is-pre-release == 0 }}
steps:
- name: "Audit version to determine if it is a pre-release"
id: semver
uses: dbt-labs/actions/[email protected]
with:
version: ${{ inputs.version_number }}

docker-release:
name: "Docker Release"
needs: [is-docker-release]
if: ${{ needs.is-docker-release.outputs.is-docker-release }}
if: ${{ !failure() && !cancelled() }}
needs: [bump-version-generate-changelog, build-test-package, github-release]
permissions:
packages: write
uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main
uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release
with:
package: "dbt-redshift"
package: ${{ github.event.repository.name }}
version_number: ${{ inputs.version_number }}
test_run: ${{ inputs.test_run }}

slack-notification:
name: Slack Notification
if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }}

needs:
[
bump-version-generate-changelog,
build-test-package,
github-release,
pypi-release,
docker-release,
]

uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main
with:
status: "failure"

secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }}
10 changes: 10 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,13 @@ help: ## Show this help message.
@echo
@echo 'targets:'
@grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'


.PHONY: docker-dev
docker-dev:
docker build -f docker/dev.Dockerfile -t dbt-redshift-dev .
docker run --rm -it --name dbt-redshift-dev -v $(shell pwd):/opt/code dbt-redshift-dev

.PHONY: docker-prod
docker-prod:
docker build -f docker/Dockerfile -t dbt-redshift .
34 changes: 19 additions & 15 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,20 +1,18 @@
ARG build_for=linux/amd64
# this image gets published to GHCR for production use
ARG py_version=3.10.7

FROM --platform=$build_for python:3.10.7-slim-bullseye as base

# ref is updated automatically every final release via bumpversion
ARG [email protected]
FROM python:$py_version-slim-bullseye as base

RUN apt-get update \
&& apt-get dist-upgrade -y \
&& apt-get install -y --no-install-recommends \
git \
ssh-client \
software-properties-common \
make \
build-essential \
ca-certificates \
libpq-dev \
build-essential=12.9 \
ca-certificates=20210119 \
git=1:2.30.2-1+deb11u2 \
libpq-dev=13.14-0+deb11u1 \
make=4.3-4.1 \
openssh-client=1:8.4p1-5+deb11u3 \
software-properties-common=0.96.20.2-2.1 \
&& apt-get clean \
&& rm -rf \
/var/lib/apt/lists/* \
Expand All @@ -24,10 +22,16 @@ RUN apt-get update \
ENV PYTHONIOENCODING=utf-8
ENV LANG=C.UTF-8

RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir
RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir


FROM base as dbt-redshift

Check failure on line 28 in docker/Dockerfile

View check run for this annotation

Wiz Inc. (266a8a9c32) / Wiz IaC Scanner

Missing User Instruction

Rule ID: e54afcf9-dc71-484a-8967-d930e3044062 Severity: High Resource: FROM={{base as dbt-redshift}} A user should be specified in the dockerfile, otherwise the image will run as root
Raw output
Expected: The 'Dockerfile' should contain the 'USER' instruction
Found: The 'Dockerfile' does not contain any 'USER' instruction

ARG commit_ref=main

HEALTHCHECK CMD dbt --version || exit 1

WORKDIR /usr/app/dbt/
ENTRYPOINT ["dbt"]

FROM base as dbt-redshift
RUN python -m pip install --no-cache-dir "dbt-redshift @ git+https://github.com/dbt-labs/${dbt_redshift_ref}"
RUN python -m pip install --no-cache-dir "dbt-bigquery @ git+https://github.com/dbt-labs/dbt-redshift@${commit_ref}"

Check warning on line 37 in docker/Dockerfile

View check run for this annotation

Wiz Inc. (266a8a9c32) / Wiz IaC Scanner

Unpinned Package Version in Pip Install

Rule ID: 1f0d05d7-8caf-4f04-bc60-332d472de5a9 Severity: Medium Resource: FROM={{base as dbt-redshift}}.{{RUN python -m pip install --no-cache-dir "dbt-bigquery @ git+https://github.com/dbt-labs/dbt-redshift@${commit_ref}"}} Package version pinning reduces the range of versions that can be installed, reducing the chances of failure due to unanticipated changes
Raw output
Expected: RUN instruction with 'pip/pip3 install <package>' should use package pinning form 'pip/pip3 install <package>=<version>'
Found: RUN instruction python -m pip install --no-cache-dir "dbt-bigquery @ git+https://github.com/dbt-labs/dbt-redshift@main" does not use package pinning form
24 changes: 6 additions & 18 deletions docker/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,44 +14,32 @@ docker build --tag <your_image_name> --target dbt-redshift <path/to/dockerfile>
---

By default the images will be populated with the most recent release of `dbt-redshift`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag:
By default the image will be populated with the latest version of `dbt-redshift` on `main`.
If you need to use a different version you can specify it by git ref using the `--build-arg` flag:
```shell
docker build --tag <your_image_name> \
--target dbt-redshift \
--build-arg dbt_redshift_ref=<git_ref> \
--build-arg commit_ref=<commit_ref> \
<path/to/dockerfile>
```

### Examples:
To build an image named "my-dbt" that supports redshift using the latest releases:
To build an image named "my-dbt" that supports Snowflake using the latest releases:
```shell
cd dbt-core/docker
docker build --tag my-dbt --target dbt-redshift .
```

To build an image named "my-other-dbt" that supports redshift using the adapter version 1.0.0b1:
To build an image named "my-other-dbt" that supports Snowflake using the adapter version 1.0.0b1:
```shell
cd dbt-core/docker
docker build \
--tag my-other-dbt \
--target dbt-redshift \
--build-arg dbt_redshift_ref=dbt-redshift@v1.0.0b1 \
--build-arg commit_ref=v1.0.0b1 \
.
```

## Special cases
There are a few special cases worth noting:

* If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg:
```shell
docker build --tag my_dbt \
--target dbt-redshift \
--build-arg build_for=linux/arm64 \
<path/to/dockerfile>
```

Supported architectures can be found in the python docker [dockerhub page](https://hub.docker.com/_/python).

## Running an image in a container:
The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal:
```shell
Expand Down
50 changes: 50 additions & 0 deletions docker/dev.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# this image does not get published, it is intended for local development only, see `Makefile` for usage
FROM ubuntu:22.04 as base

# prevent python installation from asking for time zone region
ARG DEBIAN_FRONTEND=noninteractive

# add python repository
RUN apt-get update \

Check notice on line 8 in docker/dev.Dockerfile

View check run for this annotation

Wiz Inc. (266a8a9c32) / Wiz IaC Scanner

APT-GET Not Avoiding Additional Packages

Rule ID: 0cbafd91-7f35-4000-b40a-bebedb7bb5f8 Severity: None Resource: FROM={{ubuntu:22.04 as base}}.{{RUN apt-get update && apt-get install -y software-properties-common=0.99.22.9 && add-apt-repository -y ppa:deadsnakes/ppa && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*}} Check if any apt-get installs don't use '--no-install-recommends' flag to avoid installing additional packages.
Raw output
Expected: 'RUN apt-get update   && apt-get install -y software-properties-common=0.99.22.9   && add-apt-repository -y ppa:deadsnakes/ppa   && apt-get clean   && rm -rf     /var/lib/apt/lists/*     /tmp/*     /var/tmp/*' uses '--no-install-recommends' flag to avoid installing additional packages
Found: 'RUN apt-get update   && apt-get install -y software-properties-common=0.99.22.9   && add-apt-repository -y ppa:deadsnakes/ppa   && apt-get clean   && rm -rf     /var/lib/apt/lists/*     /tmp/*     /var/tmp/*' does not use '--no-install-recommends' flag to avoid installing additional packages
&& apt-get install -y software-properties-common=0.99.22.9 \
&& add-apt-repository -y ppa:deadsnakes/ppa \
&& apt-get clean \
&& rm -rf \
/var/lib/apt/lists/* \
/tmp/* \
/var/tmp/*

# install python
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
build-essential=12.9ubuntu3 \
git-all=1:2.34.1-1ubuntu1.10 \
python3.8=3.8.19-1+jammy1 \
python3.8-dev=3.8.19-1+jammy1 \
python3.8-distutils=3.8.19-1+jammy1 \
python3.8-venv=3.8.19-1+jammy1 \
python3-pip=22.0.2+dfsg-1ubuntu0.4 \
python3-wheel=0.37.1-2ubuntu0.22.04.1 \
&& apt-get clean \
&& rm -rf \
/var/lib/apt/lists/* \
/tmp/* \
/var/tmp/*

# update the default system interpreter to the newly installed version
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1


FROM base as dbt-redshift-dev

Check failure on line 38 in docker/dev.Dockerfile

View check run for this annotation

Wiz Inc. (266a8a9c32) / Wiz IaC Scanner

Missing User Instruction

Rule ID: e54afcf9-dc71-484a-8967-d930e3044062 Severity: High Resource: FROM={{base as dbt-redshift-dev}} A user should be specified in the dockerfile, otherwise the image will run as root
Raw output
Expected: The 'Dockerfile' should contain the 'USER' instruction
Found: The 'Dockerfile' does not contain any 'USER' instruction

HEALTHCHECK CMD python3 --version || exit 1

# send stdout/stderr to terminal
ENV PYTHONUNBUFFERED=1

# setup mount for local code
WORKDIR /opt/code
VOLUME /opt/code

# create a virtual environment
RUN python3 -m venv /opt/venv

0 comments on commit b4b3f01

Please sign in to comment.