From 02e41c033a4fcc37e8833d60a887eb3560d43c0d Mon Sep 17 00:00:00 2001 From: Vit Zikmund Date: Thu, 2 Jan 2025 07:03:12 +0100 Subject: [PATCH 1/3] chore(cleanup): uv is not a runtime requirement --- requirements/dev.txt | 4 +--- requirements/main.in | 3 --- requirements/main.txt | 2 -- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 76454ae..7fb7f99 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -248,9 +248,7 @@ urllib3==2.0.7 # responses # types-requests uv==0.5.8 - # via - # -c requirements/main.txt - # -r requirements/dev.in + # via -r requirements/dev.in vcrpy==6.0.1 # via pytest-vcr virtualenv==20.25.1 diff --git a/requirements/main.in b/requirements/main.in index 611cda2..20dbb4e 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -22,7 +22,4 @@ boto3~=1.34 # GitHub AA Provider cachetools~=5.3 -# uv: fast pip replacement -uv - importlib-metadata; python_version < '3.13' diff --git a/requirements/main.txt b/requirements/main.txt index 7b5fdec..490e194 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -132,8 +132,6 @@ urllib3==2.0.7 # via # botocore # requests -uv==0.5.8 - # via -r requirements/main.in webargs==8.4.0 # via -r requirements/main.in werkzeug==3.0.3 From 6d70618aabfe4bb063e4573acfe68689384e4cd1 Mon Sep 17 00:00:00 2001 From: Vit Zikmund Date: Thu, 2 Jan 2025 11:46:58 +0100 Subject: [PATCH 2/3] feat(docker): Cut docker image size down to a half Using a couple best practices for saving space and build cache optimizations to get better build times and much smaller final image. Notably: - Python packages are now kept in a virtual environment, which is fully prepared in the builder stage, later simply copied to the same path of the final image - Any python cache .pyc files are not part of the final image - Git binary needed to automatically determine the package version is not part of the final image - Many auxiliary project source files (incl. the local .git dir(!)) are also omitted from the final image - Use RUN --mount instead of COPY where the files are needed only for the command - Keep various COPY operations to the end of a stage not to invalidate semantically unrelated cached layers - Remove deprecated (and misplaced) MAINTAINER directive in favor of a particular "authors" LABEL --- Dockerfile | 136 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 89 insertions(+), 47 deletions(-) diff --git a/Dockerfile b/Dockerfile index 37509bf..d89a530 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,66 +1,108 @@ # Dockerfile for uWSGI wrapped Giftless Git LFS Server +# Shared build ARGs among stages +ARG WORKDIR=/app +ARG VENV="$WORKDIR/.venv" ### --- Build Depdendencies --- +FROM python:3.12 AS builder +ARG UWSGI_VERSION=2.0.23 +# Common WSGI middleware modules to be pip-installed +# These are not required in every Giftless installation but are common enough +ARG EXTRA_PACKAGES="wsgi_cors_middleware" +# expose shared ARGs +ARG WORKDIR +ARG VENV -FROM python:3.12 as builder -MAINTAINER "Shahar Evron " +# Set WORKDIR (also creates the dir) +WORKDIR $WORKDIR # Build wheels for uWSGI and all requirements -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ - && apt-get install -y build-essential libpcre3 libpcre3-dev git -RUN pip install -U pip -RUN mkdir /wheels - -ARG UWSGI_VERSION=2.0.23 -RUN pip wheel -w /wheels uwsgi==$UWSGI_VERSION +RUN set -eux ;\ + export DEBIAN_FRONTEND=noninteractive ;\ + apt-get update ;\ + apt-get install -y --no-install-recommends build-essential libpcre3 libpcre3-dev git ;\ + rm -rf /var/lib/apt/lists/* + +# Create virtual env to store dependencies, "activate" it +RUN python -m venv --upgrade-deps "$VENV" +ENV VIRTUAL_ENV="$VENV" PATH="$VENV/bin:$PATH" + +# Set a couple pip-related settings +# Wait a bit longer for slow connections +ENV PIP_TIMEOUT=100 +# Don't nag about newer pip +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 +# Don't cache pip packages +ENV PIP_NO_CACHE_DIR=1 +# Require activated virtual environment +ENV PIP_REQUIRE_VIRTUALENV=1 +# Eventual python cache files go here (not to be copied) +ENV PYTHONPYCACHEPREFIX=/tmp/__pycache__ + +# Install runtime dependencies +RUN --mount=source=requirements/main.txt,target=requirements/main.txt \ + pip install -r requirements/main.txt +RUN pip install uwsgi==$UWSGI_VERSION +# Install extra packages into the virtual env +RUN pip install ${EXTRA_PACKAGES} -COPY requirements/main.txt /requirements.txt -RUN pip wheel -w /wheels -r /requirements.txt +# Copy project contents necessary for an editable install +COPY .git .git/ +COPY giftless giftless/ +COPY pyproject.toml . +# Editable-install the giftless package (add a kind of a project path reference in site-packages) +# To detect the package version dynamically, setuptools-scm needs the git binary +RUN pip install -e . ### --- Build Final Image --- - -FROM python:3.12-slim - -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ - && apt-get install -y libpcre3 libxml2 tini git \ - && apt-get clean \ - && apt -y autoremove - -RUN mkdir /app - -# Install dependencies -COPY --from=builder /wheels /wheels -RUN pip install /wheels/*.whl - -# Copy project code -COPY . /app -RUN pip install -e /app +FROM python:3.12-slim AS final +LABEL org.opencontainers.image.authors="Shahar Evron " ARG USER_NAME=giftless +# Writable path for local LFS storage ARG STORAGE_DIR=/lfs-storage -ENV GIFTLESS_TRANSFER_ADAPTERS_basic_options_storage_options_path $STORAGE_DIR - -RUN useradd -d /app $USER_NAME -RUN mkdir $STORAGE_DIR -RUN chown $USER_NAME $STORAGE_DIR - -# Pip-install some common WSGI middleware modules -# These are not required in every Giftless installation but are common enough -ARG EXTRA_PACKAGES="wsgi_cors_middleware" -RUN pip install ${EXTRA_PACKAGES} - -WORKDIR /app - -ENV UWSGI_MODULE "giftless.wsgi_entrypoint" - +# Set to true to add a runtime dockerhub deprecation warning ARG IS_DOCKERHUB -# Override default docker entrypoint for dockerhub -RUN --mount=target=/build-ctx set -e ;\ - if [ "$IS_DOCKERHUB" = true ]; then \ - cp /build-ctx/scripts/docker-entrypoint-dockerhub.sh scripts/docker-entrypoint.sh ;\ +# expose shared ARGs +ARG WORKDIR +ARG VENV + +# Set WORKDIR (also creates the dir) +WORKDIR $WORKDIR + +# Create a user and set local storage write permissions +RUN set -eux ;\ + useradd -d "$WORKDIR" "$USER_NAME" ;\ + mkdir "$STORAGE_DIR" ;\ + chown "$USER_NAME" "$STORAGE_DIR" + +# Install runtime dependencies +RUN set -eux ;\ + export DEBIAN_FRONTEND=noninteractive ;\ + apt-get update ;\ + apt-get install -y libpcre3 libxml2 tini ;\ + rm -rf /var/lib/apt/lists/* + +# Use the virtual env with dependencies from builder stage +COPY --from=builder "$VENV" "$VENV" +ENV VIRTUAL_ENV="$VENV" PATH="$VENV/bin:$PATH" +# Copy project source back into the same path referenced by the editable install +COPY --from=builder "$WORKDIR/giftless" "giftless" + +# Copy desired docker-entrypoint +RUN --mount=target=/build-ctx set -eux ;\ + target_de=scripts/docker-entrypoint.sh ;\ + mkdir -p "$(dirname "$target_de")" ;\ + if [ "${IS_DOCKERHUB:-}" = true ]; then \ + cp /build-ctx/scripts/docker-entrypoint-dockerhub.sh "$target_de" ;\ + else \ + cp /build-ctx/scripts/docker-entrypoint.sh "$target_de" ;\ fi +# Set runtime properties USER $USER_NAME +ENV GIFTLESS_TRANSFER_ADAPTERS_basic_options_storage_options_path="$STORAGE_DIR" +ENV UWSGI_MODULE="giftless.wsgi_entrypoint" ENTRYPOINT ["tini", "--", "scripts/docker-entrypoint.sh"] CMD ["uwsgi", "-s", "127.0.0.1:5000", "-M", "-T", "--threads", "2", "-p", "2", \ From 31eda4374ac07c8cd6e3a7f02628e690ad45684b Mon Sep 17 00:00:00 2001 From: Vit Zikmund Date: Thu, 2 Jan 2025 12:18:38 +0100 Subject: [PATCH 3/3] cleanup(docker): fix a little code-bloat --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d89a530..80a4ab7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ ARG VENV # Set WORKDIR (also creates the dir) WORKDIR $WORKDIR -# Build wheels for uWSGI and all requirements +# Install packages to build wheels for uWSGI and other requirements RUN set -eux ;\ export DEBIAN_FRONTEND=noninteractive ;\ apt-get update ;\ @@ -40,8 +40,8 @@ ENV PIP_REQUIRE_VIRTUALENV=1 ENV PYTHONPYCACHEPREFIX=/tmp/__pycache__ # Install runtime dependencies -RUN --mount=source=requirements/main.txt,target=requirements/main.txt \ - pip install -r requirements/main.txt +RUN --mount=target=/build-ctx \ + pip install -r /build-ctx/requirements/main.txt RUN pip install uwsgi==$UWSGI_VERSION # Install extra packages into the virtual env RUN pip install ${EXTRA_PACKAGES}