fix(Containerfile): Revert to previous solution

openclimatefix · Nov 5, 2024 · c705646 · c705646
1 parent 0d54261
commit c705646
Show file tree

Hide file tree

Showing 2 changed files with 106 additions and 72 deletions.
diff --git a/Containerfile b/Containerfile
@@ -1,93 +1,126 @@
-# Build a virtualenv using miniconda
-# * Conda creates a completely isolated environment,
-#   including all required shared libraries, enabling
-#   just putting the virtualenv into a distroless image
-#   without having to faff around with linking all
-#   the filelist (including for each dependency) of
-#   https://packages.debian.org/trixie/libpython3.12-dev, e.g.
-#
-#       echo "Copying symlinked python binary into venv" && \
-#       cp --remove-destination /usr/local/bin/python3.12 /venv/bin/python && \
-#       echo "Copying libpython package into venv" && \
-#       cp -r /usr/local/lib/* /venv/lib/ && \
-#       cp -r /usr/local/include/python3.12/* /venv/include/ && \
-#       mkdir -p /venv/lib/aarch64-linux-gnu/ && \
-#       cp -r /usr/lib/aarch64-linux-gnu/* /venv/lib/aarch64-linux-gnu/ && \
-#       mkdir -p /venv/include/aarch64-linux-gnu/ && \
-#       cp -r /usr/include/aarch64-linux-gnu/* /venv/include/aarch64-linux-gnu/ && \
+# POTENTIAL FOR SMALLER CONTAINERFILE IF THIS CAN BE GOT WORKING
+
+
+# # --- Base Python image -----------------------------------------------------------------
+# FROM python:3.12-bookworm AS python-base
+# 
+# # --- Distroless Container creation -----------------------------------------------------
+# FROM gcr.io/distroless/cc-debian12 AS python-distroless
+# 
+# ARG CHIPSET_ARCH=aarch64-linux-gnu
+# 
+# # Copy the python installation from the base image
+# COPY --from=python-base /usr/local/lib/ /usr/local/lib/
+# COPY --from=python-base /usr/local/bin/python /usr/local/bin/python
+# COPY --from=python-base /etc/ld.so.cache /etc/ld.so.cache
+# 
+# # Add common compiled libraries
+# COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libz.so.1 /lib/${CHIPSET_ARCH}/
+# COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libffi.so.8 /lib/${CHIPSET_ARCH}/
+# COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libbz2.so.1.0 /lib/${CHIPSET_ARCH}/
+# COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libm.so.6 /lib/${CHIPSET_ARCH}/
+# COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libc.so.6 /lib/${CHIPSET_ARCH}/
+# 
+# # Don't generate .pyc, enable tracebacks
+# ENV LANG=C.UTF-8 \
+#     LC_ALL=C.UTF-8 \
+#     PYTHONDONTWRITEBYTECODE=1 \
+#     PYTHONFAULTHANDLER=1
+# 
+# # Check python installation works
+# COPY --from=python-base /bin/rm /bin/rm
+# COPY --from=python-base /bin/sh /bin/sh
+# RUN python --version
+# RUN rm /bin/sh /bin/rm
+# 
+# # --- Virtualenv builder image ----------------------------------------------------------
+# FROM python-base AS build-venv
+# COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+# 
+# ENV UV_LINK_MODE=copy \
+#     UV_COMPILE_BYTECODE=1 \
+#     UV_PYTHON_DOWNLOADS=never \
+#     UV_PYTHON=python3.12 \
+#     UV_NO_CACHE=1 \
+#     CFLAGS="-g0 -Wl,--strip-all" \
+#     VENV=/.venv
+# 
+# COPY pyproject.toml ./
+# 
+# # Synchronize DEPENDENCIES without the application itself.
+# # This layer is cached until uv.lock or pyproject.toml change.
+# # Delete any unwanted parts of the installed packages to reduce size
+# RUN uv venv ${VENV} && \
+#     echo "Installing dependencies into ${VENV}" && \
+#     mkdir src && \
+#     du -h ${VENV}/lib/python3.12/site-packages && \
+#     uv sync --no-dev --no-install-project && \
+#     echo "Copying libpython package into ${VENV}" && \
+#     cp --remove-destination /usr/local/bin/python3.12 ${VENV}/bin/python && \
+#     cp /usr/local/lib/libpython3.12.so.1.0 ${VENV}/lib/ && \
+#     echo "Optimizing site-packages" && \
+#     rm -r ${VENV}/lib/python3.12/site-packages/**/tests && \
+#     du -h ${VENV}/lib/python3.12/site-packages | sort -h | tail -n 4
+# 
+# COPY . /src
+# RUN uv pip install --no-deps /src && ls /.venv/bin
+# 
+# # --- Distroless App image --------------------------------------------------------------
+# FROM python-distroless
+# 
+# COPY --from=build-venv /.venv /venv
+# 
+# ENV RAWDIR=/work/raw \
+#     ZARRDIR=/work/data \
+#     ECCODES_DEFINITION_PATH=.venv/share/eccodes/definitions
+# 
+# ENTRYPOINT ["/venv/bin/nwp-consumer-cli"]
+# VOLUME /work
+# STOPSIGNAL SIGINT
+
+
+# WORKING CONTAINERFILE
+
+FROM quay.io/condaforge/miniforge3:latest AS build-venv
 
-# --- Base Python image -----------------------------------------------------------------
-FROM python:3.12-bookworm AS python-base
-
-# --- Distroless Container creation -----------------------------------------------------
-FROM gcr.io/distroless/cc-debian12 AS python-distroless
-
-ARG CHIPSET_ARCH=aarch64-linux-gnu
-
-# Copy the python installation from the base image
-COPY --from=python-base /usr/local/lib/ /usr/local/lib/
-COPY --from=python-base /usr/local/bin/python /usr/local/bin/python
-COPY --from=python-base /etc/ld.so.cache /etc/ld.so.cache
-
-# Add common compiled libraries
-COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libz.so.1 /lib/${CHIPSET_ARCH}/
-COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libffi.so.8 /lib/${CHIPSET_ARCH}/
-COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libbz2.so.1.0 /lib/${CHIPSET_ARCH}/
-COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libm.so.6 /lib/${CHIPSET_ARCH}/
-COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libc.so.6 /lib/${CHIPSET_ARCH}/
-
-# Don't generate .pyc, enable tracebacks
-ENV LANG=C.UTF-8 \
-    LC_ALL=C.UTF-8 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONFAULTHANDLER=1
-
-# Check python installation works
-COPY --from=python-base /bin/rm /bin/rm
-COPY --from=python-base /bin/sh /bin/sh
-RUN python --version
-RUN rm /bin/sh /bin/rm
-
-# --- Virtualenv builder image ----------------------------------------------------------
-FROM python-base AS build-venv
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
 
 ENV UV_LINK_MODE=copy \
     UV_COMPILE_BYTECODE=1 \
     UV_PYTHON_DOWNLOADS=never \
     UV_PYTHON=python3.12 \
-    UV_NO_CACHE=1 \
-    CFLAGS="-g0 -Wl,--strip-all" \
-    VENV=/.venv
-
-COPY pyproject.toml ./
+    UV_PROJECT_ENVIRONMENT=/venv
+COPY pyproject.toml /_lock/
 
 # Synchronize DEPENDENCIES without the application itself.
 # This layer is cached until uv.lock or pyproject.toml change.
 # Delete any unwanted parts of the installed packages to reduce size
-RUN uv venv ${VENV} && \
-    echo "Installing dependencies into ${VENV}" && \
+RUN apt-get update && apt-get install gcc -y && \
+    echo "Creating virtualenv at /venv" && \
+    conda create -qy -p /venv python=3.12 numcodecs eccodes
+RUN echo "Installing dependencies into /venv" && \
+    cd /_lock && \
     mkdir src && \
-    du -h ${VENV}/lib/python3.12/site-packages && \
     uv sync --no-dev --no-install-project && \
-    echo "Copying libpython package into ${VENV}" && \
-    cp --remove-destination /usr/local/bin/python3.12 ${VENV}/bin/python && \
-    cp /usr/local/lib/libpython3.12.so.1.0 ${VENV}/lib/ && \
-    echo "Optimizing site-packages" && \
-    rm -r ${VENV}/lib/python3.12/site-packages/**/tests && \
-    du -h ${VENV}/lib/python3.12/site-packages | sort -h | tail -n 4
+    echo "Optimizing /venv site-packages" && \
+    rm -r /venv/lib/python3.12/site-packages/**/tests && \
+    rm -r /venv/lib/python3.12/site-packages/**/_*cache* && \
+    rm -r /venv/share/eccodes/definitions/bufr
 
+# Then install the application itself
+# * Delete the test and cache folders from installed packages to reduce size
 COPY . /src
-RUN uv pip install --no-deps /src && ls /.venv/bin
-
-# --- Distroless App image --------------------------------------------------------------
-FROM python-distroless
+RUN uv pip install --no-deps --python=$UV_PROJECT_ENVIRONMENT /src
 
-COPY --from=build-venv /.venv /venv
+# Copy the virtualenv into a distroless image
+# * These are small images that only contain the runtime dependencies
+FROM gcr.io/distroless/python3-debian11
+WORKDIR /app
+COPY --from=build-venv /venv /venv
 
 ENV RAWDIR=/work/raw \
     ZARRDIR=/work/data \
-    ECCODES_DEFINITION_PATH=.venv/share/eccodes/definitions
+    ECCODES_DEFINITION_PATH=/venv/share/eccodes/definitions
 
 ENTRYPOINT ["/venv/bin/nwp-consumer-cli"]
 VOLUME /work

diff --git a/pyproject.toml b/pyproject.toml
@@ -19,6 +19,7 @@ classifiers = ["Programming Language :: Python :: 3"]
 dependencies = [
     "dask == 2024.8.1",
     "eccodes == 2.38.3",
+    "ecmwf-api-client == 1.6.3",
     "cfgrib == 0.9.14.1",
     "dagster-pipes == 1.8.5",
     "joblib == 1.4.2",
@@ -28,7 +29,7 @@ dependencies = [
     "returns == 0.23.0",
     "s3fs == 2024.9.0",
     "xarray == 2024.9.0",
-    "zarr == 2.18.3"
+    "zarr == 2.18.2"
 ]
 
 [dependency-groups]