feat(Containerfile): Very nearly working efficient containerfile

Containerfile

@@ -16,49 +16,78 @@
 #       mkdir -p /venv/include/aarch64-linux-gnu/ && \
 #       cp -r /usr/include/aarch64-linux-gnu/* /venv/include/aarch64-linux-gnu/ && \
 
-FROM quay.io/condaforge/miniforge3:latest AS build-venv
+# --- Base Python image -----------------------------------------------------------------
+FROM python:3.12-bookworm AS python-base
 
+# --- Distroless Container creation -----------------------------------------------------
+FROM gcr.io/distroless/cc-debian12 AS python-distroless
+
+ARG CHIPSET_ARCH=aarch64-linux-gnu
+
+# Copy the python installation from the base image
+COPY --from=python-base /usr/local/lib/ /usr/local/lib/
+COPY --from=python-base /usr/local/bin/python /usr/local/bin/python
+COPY --from=python-base /etc/ld.so.cache /etc/ld.so.cache
+
+# Add common compiled libraries
+COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libz.so.1 /lib/${CHIPSET_ARCH}/
+COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libffi.so.8 /lib/${CHIPSET_ARCH}/
+COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libbz2.so.1.0 /lib/${CHIPSET_ARCH}/
+COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libm.so.6 /lib/${CHIPSET_ARCH}/
+COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libc.so.6 /lib/${CHIPSET_ARCH}/
+
+# Don't generate .pyc, enable tracebacks
+ENV LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONFAULTHANDLER=1
+
+# Check python installation works
+COPY --from=python-base /bin/rm /bin/rm
+COPY --from=python-base /bin/sh /bin/sh
+RUN python --version
+RUN rm /bin/sh /bin/rm
+
+# --- Virtualenv builder image ----------------------------------------------------------
+FROM python-base AS build-venv
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
 
 ENV UV_LINK_MODE=copy \
     UV_COMPILE_BYTECODE=1 \
     UV_PYTHON_DOWNLOADS=never \
     UV_PYTHON=python3.12 \
-    UV_PROJECT_ENVIRONMENT=/venv
-COPY pyproject.toml /_lock/
+    UV_NO_CACHE=1 \
+    CFLAGS="-g0 -Wl,--strip-all" \
+    VENV=/.venv
+
+COPY pyproject.toml ./
 
 # Synchronize DEPENDENCIES without the application itself.
 # This layer is cached until uv.lock or pyproject.toml change.
 # Delete any unwanted parts of the installed packages to reduce size
-RUN --mount=type=cache,target=/root/.cache \
-    apt-get update && apt-get install gcc -y && \
-    echo "Creating virtualenv at /venv" && \
-    conda create -qy -p /venv python=3.12 numcodecs
-RUN which gcc
-RUN echo "Installing dependencies into /venv" && \
-    cd /_lock && \
+RUN uv venv ${VENV} && \
+    echo "Installing dependencies into ${VENV}" && \
     mkdir src && \
+    du -h ${VENV}/lib/python3.12/site-packages && \
     uv sync --no-dev --no-install-project && \
-    echo "Optimizing /venv site-packages" && \
-    rm -r /venv/lib/python3.12/site-packages/**/tests && \
-    rm -r /venv/lib/python3.12/site-packages/**/_*cache*
+    echo "Copying libpython package into ${VENV}" && \
+    cp --remove-destination /usr/local/bin/python3.12 ${VENV}/bin/python && \
+    cp /usr/local/lib/libpython3.12.so.1.0 ${VENV}/lib/ && \
+    echo "Optimizing site-packages" && \
+    rm -r ${VENV}/lib/python3.12/site-packages/**/tests && \
+    du -h ${VENV}/lib/python3.12/site-packages | sort -h | tail -n 4
 
-
-# Then install the application itself
-# * Delete the test and cache folders from installed packages to reduce size
 COPY . /src
-RUN --mount=type=cache,target=/root/.cache \
-    uv pip install --no-deps --python=$UV_PROJECT_ENVIRONMENT /src
+RUN uv pip install --no-deps /src && ls /.venv/bin
+
+# --- Distroless App image --------------------------------------------------------------
+FROM python-distroless
 
-# Copy the virtualenv into a distroless image
-# * These are small images that only contain the runtime dependencies
-FROM gcr.io/distroless/python3-debian11
-WORKDIR /app
-COPY --from=build-venv /venv /venv
+COPY --from=build-venv /.venv /venv
 
 ENV RAWDIR=/work/raw \
     ZARRDIR=/work/data \
-    ECCODES_DEFINITION_PATH=/venv/share/eccodes/definitions
+    ECCODES_DEFINITION_PATH=.venv/share/eccodes/definitions
 
 ENTRYPOINT ["/venv/bin/nwp-consumer-cli"]
 VOLUME /work

pyproject.toml

@@ -19,21 +19,19 @@ classifiers = ["Programming Language :: Python :: 3"]
 dependencies = [
     "dask == 2024.8.1",
     "eccodes == 2.38.3",
-    "ecmwf-api-client == 1.6.3",
-    "cfgrib == 0.9.14.0",
+    "cfgrib == 0.9.14.1",
     "dagster-pipes == 1.8.5",
     "joblib == 1.4.2",
     "numpy == 2.1.0",
     "ocf-blosc2 == 0.0.11",
     "psutil == 6.0.0",
-    "requests == 2.32.3",
     "returns == 0.23.0",
     "s3fs == 2024.9.0",
     "xarray == 2024.9.0",
-    "zarr == 2.18.2"
+    "zarr == 2.18.3"
 ]
 
-[project.optional-dependencies]
+[dependency-groups]
 test = [
     "flask == 3.0.0",
     "flask-cors == 4.0.0",
@@ -53,7 +51,9 @@ docs = [
     "pydoctor >= 24.3.0",
 ]
 dev = [
-    "nwp-consumer[test,lint,docs]",
+    { include = "test" },
+    { include = "lint" },
+    { include = "docs" },
 ]
 lsp = [
     "python-lsp-server",