From ea88f43df042cd1445b371ce8f6f71f891ca4eac Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Tue, 22 Oct 2024 22:25:23 +0200 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8FMaintenance:=20Improve=20UV?= =?UTF-8?q?=20usage=20(#6566)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 2 +- packages/postgres-database/docker/Dockerfile | 16 ++- .../postgres-database/scripts/erd/Dockerfile | 16 ++- packages/service-integration/Dockerfile | 19 +-- requirements/tools/Dockerfile | 14 +-- scripts/erd/Dockerfile | 14 ++- .../maintenance/migrate_project/Dockerfile | 18 ++- scripts/openapi/oas_resolver/Dockerfile | 17 +-- scripts/pydeps-docker/Dockerfile | 15 ++- services/agent/Dockerfile | 28 ++--- services/agent/docker/boot.sh | 14 ++- services/agent/docker/entrypoint.sh | 29 ++--- services/api-server/Dockerfile | 28 ++--- services/api-server/docker/boot.sh | 14 ++- services/api-server/docker/entrypoint.sh | 5 - services/autoscaling/Dockerfile | 33 +++-- services/autoscaling/docker/boot.sh | 15 ++- services/autoscaling/docker/entrypoint.sh | 5 - services/catalog/Dockerfile | 28 ++--- services/catalog/docker/boot.sh | 14 ++- services/catalog/docker/entrypoint.sh | 5 - services/clusters-keeper/Dockerfile | 28 ++--- services/clusters-keeper/docker/boot.sh | 14 ++- services/dask-sidecar/Dockerfile | 34 +++--- services/dask-sidecar/docker/boot.sh | 13 +- services/dask-sidecar/docker/entrypoint.sh | 114 ++++++++---------- services/datcore-adapter/Dockerfile | 28 ++--- services/datcore-adapter/docker/boot.sh | 14 ++- services/datcore-adapter/docker/entrypoint.sh | 6 - services/director-v2/Dockerfile | 28 ++--- services/director-v2/docker/boot.sh | 14 ++- services/director-v2/docker/entrypoint.sh | 30 ++--- services/director/requirements/_tools.txt | 29 ++++- services/dynamic-scheduler/Dockerfile | 28 ++--- services/dynamic-scheduler/docker/boot.sh | 14 ++- .../dynamic-scheduler/docker/entrypoint.sh | 5 - services/dynamic-sidecar/Dockerfile | 44 ++++--- services/dynamic-sidecar/docker/boot.sh | 9 +- services/dynamic-sidecar/docker/entrypoint.sh | 5 - services/efs-guardian/Dockerfile | 28 ++--- services/efs-guardian/docker/boot.sh | 14 ++- services/invitations/Dockerfile | 28 ++--- services/invitations/docker/boot.sh | 14 ++- services/invitations/docker/entrypoint.sh | 5 - services/migration/Dockerfile | 17 ++- services/osparc-gateway-server/Dockerfile | 24 ++-- services/osparc-gateway-server/docker/boot.sh | 25 ++-- .../docker/entrypoint.sh | 100 +++++++-------- services/payments/Dockerfile | 28 ++--- services/payments/docker/boot.sh | 14 ++- services/payments/docker/entrypoint.sh | 5 - services/resource-usage-tracker/Dockerfile | 28 ++--- .../resource-usage-tracker/docker/boot.sh | 14 ++- .../docker/entrypoint.sh | 5 - services/storage/Dockerfile | 28 ++--- services/storage/docker/boot.sh | 14 ++- services/storage/docker/entrypoint.sh | 7 -- services/web/Dockerfile | 28 ++--- services/web/server/docker/boot.sh | 14 ++- services/web/server/docker/entrypoint.sh | 5 - 60 files changed, 636 insertions(+), 615 deletions(-) diff --git a/.dockerignore b/.dockerignore index 34ae3126ea5..a00f0b10508 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,7 +8,7 @@ ops/ *.py[cod] # virtualenv -.venv +**/.venv #python eggs **/*.egg-info diff --git a/packages/postgres-database/docker/Dockerfile b/packages/postgres-database/docker/Dockerfile index 7c3e74870b2..ba5a04156af 100644 --- a/packages/postgres-database/docker/Dockerfile +++ b/packages/postgres-database/docker/Dockerfile @@ -1,5 +1,9 @@ # syntax=docker/dockerfile:1 -FROM python:3.6-slim AS base +ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian +FROM python:${PYTHON_VERSION}-slim-bookworm AS base LABEL maintainer=sanderegg @@ -22,16 +26,14 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools @@ -44,6 +46,8 @@ RUN git clone --single-branch --branch ${GIT_BRANCH} ${GIT_REPOSITORY} osparc-si FROM base AS production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu # ensure home folder is read/writable for user scu diff --git a/packages/postgres-database/scripts/erd/Dockerfile b/packages/postgres-database/scripts/erd/Dockerfile index e18bec73e16..168c5522290 100644 --- a/packages/postgres-database/scripts/erd/Dockerfile +++ b/packages/postgres-database/scripts/erd/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base RUN apt-get update \ @@ -15,15 +20,14 @@ RUN apt-get update \ && apt-get clean -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --upgrade \ - pip~=24.0 \ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --upgrade \ wheel \ setuptools # devenv COPY requirements.txt requirements.txt -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install \ - -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements.txt diff --git a/packages/service-integration/Dockerfile b/packages/service-integration/Dockerfile index 15c1de6cc24..b41f4d07b0b 100644 --- a/packages/service-integration/Dockerfile +++ b/packages/service-integration/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base LABEL maintainer=pcrespov @@ -38,12 +43,13 @@ ENV LANG=C.UTF-8 # Turns off writing .pyc files; superfluous on an ephemeral container. ENV PYTHONDONTWRITEBYTECODE=1 \ VIRTUAL_ENV=/home/scu/.venv +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 # Ensures that the python and pip executables used # in the image will be those from our virtualenv. ENV PATH="${VIRTUAL_ENV}/bin:$PATH" - # -------------------------- Build stage ------------------- FROM base AS build @@ -55,15 +61,14 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ pip~=24.0 \ wheel \ @@ -74,7 +79,7 @@ WORKDIR /build/packages/service-integration RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=packages/service-integration,target=/build/packages/service-integration,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ + --mount=type=cache,target=/root/.cache/uv \ uv pip install \ --requirement requirements/prod.txt \ && uv pip list @@ -86,8 +91,6 @@ FROM base AS development # NOTE: this is necessary to allow to build development images but is the same as production here FROM base AS production -ENV PYTHONOPTIMIZE=TRUE - WORKDIR /home/scu # ensure home folder is read/writable for user scu RUN chown -R scu /home/scu diff --git a/requirements/tools/Dockerfile b/requirements/tools/Dockerfile index e6af4395ed6..0901b5e8302 100644 --- a/requirements/tools/Dockerfile +++ b/requirements/tools/Dockerfile @@ -9,6 +9,9 @@ # # ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base ENV VIRTUAL_ENV=/home/scu/.venv @@ -24,24 +27,21 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get clean -y -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools # devenv -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install \ - pip-tools \ pipreqs \ pipdeptree && \ uv pip list -vv diff --git a/scripts/erd/Dockerfile b/scripts/erd/Dockerfile index 9294fa23e1d..8289d6ab16c 100644 --- a/scripts/erd/Dockerfile +++ b/scripts/erd/Dockerfile @@ -8,6 +8,9 @@ # ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base RUN apt-get update \ @@ -22,13 +25,14 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --upgrade \ - pip~=24.0 \ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --upgrade \ wheel \ setuptools COPY requirements.txt . -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install -r requirements.txt diff --git a/scripts/maintenance/migrate_project/Dockerfile b/scripts/maintenance/migrate_project/Dockerfile index d4a17f9ba40..b215ed0886d 100644 --- a/scripts/maintenance/migrate_project/Dockerfile +++ b/scripts/maintenance/migrate_project/Dockerfile @@ -1,21 +1,27 @@ # syntax=docker/dockerfile:1 +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:3.11.9-buster RUN curl https://rclone.org/install.sh | bash && \ rclone --version +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ + WORKDIR /scripts COPY packages/postgres-database postgres-database -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - cd postgres-database && pip install . +RUN --mount=type=cache,target=/root/.cache/uv \ + cd postgres-database && uv pip install . COPY packages/settings-library settings-library -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - cd settings-library && pip install . +RUN --mount=type=cache,target=/root/.cache/uv \ + cd settings-library && uv pip install . COPY scripts/maintenance/migrate_project/requirements.txt /scripts/requirements.txt -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install -r /scripts/requirements.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install -r /scripts/requirements.txt COPY scripts/maintenance/migrate_project/src/*.py /scripts/ diff --git a/scripts/openapi/oas_resolver/Dockerfile b/scripts/openapi/oas_resolver/Dockerfile index 5b0d51570ae..19ffedfdc16 100644 --- a/scripts/openapi/oas_resolver/Dockerfile +++ b/scripts/openapi/oas_resolver/Dockerfile @@ -2,6 +2,9 @@ # Usage: # docker build . -t oas_resolver # docker run -v /path/to/api:/input -v /path/to/compiled/file:/output oas_resolver /input/path/to/openapi.yaml /output/output_file.yaml +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:3.6-alpine LABEL maintainer=sanderegg @@ -11,16 +14,16 @@ VOLUME [ "/output" ] WORKDIR /src +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ + # update pip -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --upgrade \ - pip~=24.0 \ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --upgrade \ wheel \ setuptools -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install prance && \ - pip install click &&\ - pip install openapi_spec_validator +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install prance click openapi_spec_validator ENTRYPOINT [ "prance", "compile" , "--backend=openapi-spec-validator"] diff --git a/scripts/pydeps-docker/Dockerfile b/scripts/pydeps-docker/Dockerfile index 610fe9aa6f0..51d24208672 100644 --- a/scripts/pydeps-docker/Dockerfile +++ b/scripts/pydeps-docker/Dockerfile @@ -9,6 +9,9 @@ # # ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base @@ -23,18 +26,20 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ + ARG HOME_DIR RUN mkdir -p ${HOME_DIR} COPY .pydeps ${HOME_DIR}/.pydeps -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --upgrade \ - pip~=24.0 \ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --upgrade \ wheel \ setuptools # devenv -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install \ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install \ pydeps diff --git a/services/agent/Dockerfile b/services/agent/Dockerfile index fe6fef17c35..9b5d031affd 100644 --- a/services/agent/Dockerfile +++ b/services/agent/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -73,29 +78,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/agent/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -113,9 +111,9 @@ WORKDIR /build/services/agent RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/agent,target=/build/services/agent,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -132,6 +130,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/agent/docker/boot.sh b/services/agent/docker/boot.sh index 333a4af7469..259e1f54e71 100755 --- a/services/agent/docker/boot.sh +++ b/services/agent/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/agent || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/agent + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/agent/docker/entrypoint.sh b/services/agent/docker/entrypoint.sh index 87f1cc84da5..e89ad5408a3 100755 --- a/services/agent/docker/entrypoint.sh +++ b/services/agent/docker/entrypoint.sh @@ -63,27 +63,20 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock -if stat $DOCKER_MOUNT > /dev/null 2>&1 -then - echo "$INFO detected docker socket is mounted, adding user to group..." - GROUPID=$(stat --format=%g $DOCKER_MOUNT) - GROUPNAME=scdocker +if stat $DOCKER_MOUNT >/dev/null 2>&1; then + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) + GROUPNAME=scdocker - if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1 - then - echo "$WARNING docker group with $GROUPID already exists, getting group name..." - # if group already exists in container, then reuse name - GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) - echo "$WARNING docker group with $GROUPID has name $GROUPNAME" - fi - adduser "$SC_USER_NAME" "$GROUPNAME" + if ! addgroup --gid "$GROUPID" $GROUPNAME >/dev/null 2>&1; then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." + # if group already exists in container, then reuse name + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" + fi + adduser "$SC_USER_NAME" "$GROUPNAME" fi echo "$INFO Starting $* ..." diff --git a/services/api-server/Dockerfile b/services/api-server/Dockerfile index ced7167fe5c..933df727fc6 100644 --- a/services/api-server/Dockerfile +++ b/services/api-server/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # USAGE: @@ -69,29 +74,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/api-server/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -109,9 +107,9 @@ WORKDIR /build/services/api-server RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/api-server,target=/build/services/api-server,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/api-server/docker/boot.sh b/services/api-server/docker/boot.sh index 39f5867806a..38322883120 100755 --- a/services/api-server/docker/boot.sh +++ b/services/api-server/docker/boot.sh @@ -18,12 +18,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/api-server || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/api-server + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # RUNNING application ---------------------------------------- diff --git a/services/api-server/docker/entrypoint.sh b/services/api-server/docker/entrypoint.sh index 68791d962fe..b579236b562 100755 --- a/services/api-server/docker/entrypoint.sh +++ b/services/api-server/docker/entrypoint.sh @@ -64,11 +64,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs ptvsd - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/autoscaling/Dockerfile b/services/autoscaling/Dockerfile index 1821d873057..4ace322f5f9 100644 --- a/services/autoscaling/Dockerfile +++ b/services/autoscaling/Dockerfile @@ -1,10 +1,15 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # USAGE: -# cd sercices/autoscaling +# cd services/autoscaling # docker build -f Dockerfile -t autoscaling:prod --target production ../../ # docker run autoscaling:prod # @@ -89,31 +94,21 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ + # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build -# install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/autoscaling/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt - - # --------------------------Prod-depends-only stage ------------------- # This stage is for production only dependencies that get partially wiped out afterwards (final docker image concerns) # @@ -129,9 +124,9 @@ WORKDIR /build/services/autoscaling RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/autoscaling,target=/build/services/autoscaling,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -148,6 +143,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu # ensure home folder is read/writable for user scu diff --git a/services/autoscaling/docker/boot.sh b/services/autoscaling/docker/boot.sh index 0e876652386..8fe0488daed 100755 --- a/services/autoscaling/docker/boot.sh +++ b/services/autoscaling/docker/boot.sh @@ -23,12 +23,15 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/autoscaling || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + cd services/autoscaling + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/autoscaling/docker/entrypoint.sh b/services/autoscaling/docker/entrypoint.sh index f928edbb4f5..ad982fd8d5c 100755 --- a/services/autoscaling/docker/entrypoint.sh +++ b/services/autoscaling/docker/entrypoint.sh @@ -70,11 +70,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock if stat $DOCKER_MOUNT >/dev/null 2>&1; then diff --git a/services/catalog/Dockerfile b/services/catalog/Dockerfile index c4f48d2ec08..8fb6418b3ce 100644 --- a/services/catalog/Dockerfile +++ b/services/catalog/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # @@ -70,29 +75,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/catalog/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -110,9 +108,9 @@ WORKDIR /build/services/catalog RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/catalog,target=/build/services/catalog,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/catalog/docker/boot.sh b/services/catalog/docker/boot.sh index 69ce7ebb4ec..f7f01aec46e 100755 --- a/services/catalog/docker/boot.sh +++ b/services/catalog/docker/boot.sh @@ -18,12 +18,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/catalog || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/catalog + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # RUNNING application ---------------------------------------- diff --git a/services/catalog/docker/entrypoint.sh b/services/catalog/docker/entrypoint.sh index 61529af46c6..63e9249be31 100755 --- a/services/catalog/docker/entrypoint.sh +++ b/services/catalog/docker/entrypoint.sh @@ -63,11 +63,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs ptvsd - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/clusters-keeper/Dockerfile b/services/clusters-keeper/Dockerfile index 2dd08b4b44b..e75ba032ac0 100644 --- a/services/clusters-keeper/Dockerfile +++ b/services/clusters-keeper/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -89,29 +94,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/clusters-keeper/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -129,9 +127,9 @@ WORKDIR /build/services/clusters-keeper RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/clusters-keeper,target=/build/services/clusters-keeper,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -148,6 +146,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu # ensure home folder is read/writable for user scu diff --git a/services/clusters-keeper/docker/boot.sh b/services/clusters-keeper/docker/boot.sh index f0239f42e72..5aa4b0e8e1c 100755 --- a/services/clusters-keeper/docker/boot.sh +++ b/services/clusters-keeper/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/clusters-keeper || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/clusters-keeper + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/dask-sidecar/Dockerfile b/services/dask-sidecar/Dockerfile index a85f653f5cc..5ea8e90d545 100644 --- a/services/dask-sidecar/Dockerfile +++ b/services/dask-sidecar/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM --platform=${TARGETPLATFORM} python:${PYTHON_VERSION}-slim-bookworm AS base ARG TARGETPLATFORM ARG BUILDPLATFORM @@ -80,27 +85,20 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build -# install base 3rd party dependencies (NOTE: this speeds up devel mode) -RUN \ - --mount=type=bind,source=services/dask-sidecar/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- # This stage is for production only dependencies that get partially wiped out afterwards (final docker image concerns) @@ -117,9 +115,9 @@ WORKDIR /build/services/dask-sidecar RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/dask-sidecar,target=/build/services/dask-sidecar,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list # --------------------------Production stage ------------------- @@ -135,13 +133,17 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu - # ensure home folder is read/writable for user scu RUN chown -R scu /home/scu -# bring installed package without build tools + +# Starting from clean base image, copies pre-installed virtualenv from prod-only-deps COPY --from=prod-only-deps --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +# Copies booting scripts COPY --chown=scu:scu services/dask-sidecar/docker services/dask-sidecar/docker RUN chmod +x services/dask-sidecar/docker/*.sh diff --git a/services/dask-sidecar/docker/boot.sh b/services/dask-sidecar/docker/boot.sh index 213377cbf0e..2cfdbf349d7 100755 --- a/services/dask-sidecar/docker/boot.sh +++ b/services/dask-sidecar/docker/boot.sh @@ -24,11 +24,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then print_info "Python :" python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/dask-sidecar || exit 1 - pip install --no-cache-dir -r requirements/dev.txt - cd - || exit 1 + cd services/dask-sidecar + uv pip sync --quiet --no-cache-dir requirements/dev.txt + cd - print_info "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # RUNNING application ---------------------------------------- diff --git a/services/dask-sidecar/docker/entrypoint.sh b/services/dask-sidecar/docker/entrypoint.sh index 5c00a365b3a..f69fd1a71d8 100755 --- a/services/dask-sidecar/docker/entrypoint.sh +++ b/services/dask-sidecar/docker/entrypoint.sh @@ -22,93 +22,77 @@ update-ca-certificates # *runs* as non-root user [scu] # echo "$INFO" "Entrypoint for stage ${SC_BUILD_TARGET} ..." -echo User :"$(id "$(whoami)")" -echo Workdir :"$(pwd)" -echo scuUser :"$(id scu)" - - -if [ "${SC_BUILD_TARGET}" = "development" ] -then - echo "$INFO" "development mode detected..." - # NOTE: expects docker run ... -v $(pwd):/devel/services/dask-sidecar - DEVEL_MOUNT="/devel/services/dask-sidecar" - - stat $DEVEL_MOUNT > /dev/null 2>&1 || \ - (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) - - echo "setting correct user id/group id..." - HOST_USERID=$(stat --format=%u "${DEVEL_MOUNT}") - HOST_GROUPID=$(stat --format=%g "${DEVEL_MOUNT}") - CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut --delimiter=: --fields=1) - if [ "$HOST_USERID" -eq 0 ] - then - echo "Warning: Folder mounted owned by root user... adding $SC_USER_NAME to root..." - adduser "$SC_USER_NAME" root +echo User :"$(id "$(whoami)")" +echo Workdir :"$(pwd)" +echo scuUser :"$(id scu)" + +if [ "${SC_BUILD_TARGET}" = "development" ]; then + echo "$INFO" "development mode detected..." + # NOTE: expects docker run ... -v $(pwd):/devel/services/dask-sidecar + DEVEL_MOUNT="/devel/services/dask-sidecar" + + stat $DEVEL_MOUNT >/dev/null 2>&1 || + (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) + + echo "setting correct user id/group id..." + HOST_USERID=$(stat --format=%u "${DEVEL_MOUNT}") + HOST_GROUPID=$(stat --format=%g "${DEVEL_MOUNT}") + CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut --delimiter=: --fields=1) + if [ "$HOST_USERID" -eq 0 ]; then + echo "Warning: Folder mounted owned by root user... adding $SC_USER_NAME to root..." + adduser "$SC_USER_NAME" root + else + echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." + # take host's credentials in $SC_USER_NAME + if [ -z "$CONT_GROUPNAME" ]; then + echo "Creating new group my$SC_USER_NAME" + CONT_GROUPNAME=my$SC_USER_NAME + addgroup --gid "$HOST_GROUPID" "$CONT_GROUPNAME" else - echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." - # take host's credentials in $SC_USER_NAME - if [ -z "$CONT_GROUPNAME" ] - then - echo "Creating new group my$SC_USER_NAME" - CONT_GROUPNAME=my$SC_USER_NAME - addgroup --gid "$HOST_GROUPID" "$CONT_GROUPNAME" - else - echo "group already exists" - fi - echo "adding $SC_USER_NAME to group $CONT_GROUPNAME..." - adduser "$SC_USER_NAME" "$CONT_GROUPNAME" - - echo "changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)" - usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME" - - echo "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" - find / -path /proc -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \; - # change user property of files already around - echo "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" - find / -path /proc -prune -o -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; + echo "group already exists" fi -fi + echo "adding $SC_USER_NAME to group $CONT_GROUPNAME..." + adduser "$SC_USER_NAME" "$CONT_GROUPNAME" + echo "changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)" + usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME" -if [ "${SC_BOOT_MODE}" = "debug" ] -then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy + echo "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \; + # change user property of files already around + echo "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; + fi fi - if [ ${DASK_START_AS_SCHEDULER+x} ]; then echo "$INFO Starting $* as SCHEDULER ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" - else # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock - if stat $DOCKER_MOUNT > /dev/null 2>&1 - then - echo "$INFO detected docker socket is mounted, adding user to group..." - GROUPID=$(stat --format=%g $DOCKER_MOUNT) - GROUPNAME=scdocker - - if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1 - then - echo "$WARNING docker group with $GROUPID already exists, getting group name..." - # if group already exists in container, then reuse name - GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) - echo "$WARNING docker group with $GROUPID has name $GROUPNAME" - fi - adduser "$SC_USER_NAME" "$GROUPNAME" + if stat $DOCKER_MOUNT >/dev/null 2>&1; then + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) + GROUPNAME=scdocker + + if ! addgroup --gid "$GROUPID" $GROUPNAME >/dev/null 2>&1; then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." + # if group already exists in container, then reuse name + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" + fi + adduser "$SC_USER_NAME" "$GROUPNAME" fi echo "$INFO ensuring write rights on computational shared folder ..." mkdir --parents "${SIDECAR_COMP_SERVICES_SHARED_FOLDER}" chown --recursive "$SC_USER_NAME":"$GROUPNAME" "${SIDECAR_COMP_SERVICES_SHARED_FOLDER}" - echo "$INFO Starting $* as WORKER ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/datcore-adapter/Dockerfile b/services/datcore-adapter/Dockerfile index bdd52d81841..e83bf01c37e 100644 --- a/services/datcore-adapter/Dockerfile +++ b/services/datcore-adapter/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -70,29 +75,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/datcore-adapter/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -110,9 +108,9 @@ WORKDIR /build/services/datcore-adapter RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/datcore-adapter,target=/build/services/datcore-adapter,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/datcore-adapter/docker/boot.sh b/services/datcore-adapter/docker/boot.sh index 1002a32de8a..74f8d84a7aa 100755 --- a/services/datcore-adapter/docker/boot.sh +++ b/services/datcore-adapter/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/datcore-adapter || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/datcore-adapter + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # RUNNING application ---------------------------------------- diff --git a/services/datcore-adapter/docker/entrypoint.sh b/services/datcore-adapter/docker/entrypoint.sh index 3ac31b9b815..25153a6b2a2 100755 --- a/services/datcore-adapter/docker/entrypoint.sh +++ b/services/datcore-adapter/docker/entrypoint.sh @@ -20,7 +20,6 @@ echo "$INFO" "User : $(id scu)" echo "$INFO" "python : $(command -v python)" echo "$INFO" "pip : $(command -v pip)" - # # DEVELOPMENT MODE # - expects docker run ... -v $(pwd):$SC_DEVEL_MOUNT @@ -64,11 +63,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/director-v2/Dockerfile b/services/director-v2/Dockerfile index 4780eb62e33..5388dbb6f73 100644 --- a/services/director-v2/Dockerfile +++ b/services/director-v2/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -70,29 +75,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/director-v2/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -110,9 +108,9 @@ WORKDIR /build/services/director-v2 RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/director-v2,target=/build/services/director-v2,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/director-v2/docker/boot.sh b/services/director-v2/docker/boot.sh index 348b976149c..a96161d9660 100755 --- a/services/director-v2/docker/boot.sh +++ b/services/director-v2/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/director-v2 || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/director-v2 + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/director-v2/docker/entrypoint.sh b/services/director-v2/docker/entrypoint.sh index 57b0d0ac2ff..42995c6ae78 100755 --- a/services/director-v2/docker/entrypoint.sh +++ b/services/director-v2/docker/entrypoint.sh @@ -27,7 +27,6 @@ echo "$INFO" "User : $(id scu)" echo "$INFO" "python : $(command -v python)" echo "$INFO" "pip : $(command -v pip)" - # # DEVELOPMENT MODE # - expects docker run ... -v $(pwd):$SC_DEVEL_MOUNT @@ -71,27 +70,20 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock -if stat $DOCKER_MOUNT > /dev/null 2>&1 -then - echo "$INFO detected docker socket is mounted, adding user to group..." - GROUPID=$(stat --format=%g $DOCKER_MOUNT) - GROUPNAME=scdocker +if stat $DOCKER_MOUNT >/dev/null 2>&1; then + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) + GROUPNAME=scdocker - if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1 - then - echo "$WARNING docker group with $GROUPID already exists, getting group name..." - # if group already exists in container, then reuse name - GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) - echo "$WARNING docker group with $GROUPID has name $GROUPNAME" - fi - adduser "$SC_USER_NAME" "$GROUPNAME" + if ! addgroup --gid "$GROUPID" $GROUPNAME >/dev/null 2>&1; then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." + # if group already exists in container, then reuse name + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" + fi + adduser "$SC_USER_NAME" "$GROUPNAME" fi echo "$INFO Starting $* ..." diff --git a/services/director/requirements/_tools.txt b/services/director/requirements/_tools.txt index 4c0c50f666d..38ed7220aed 100644 --- a/services/director/requirements/_tools.txt +++ b/services/director/requirements/_tools.txt @@ -1,3 +1,9 @@ +# +# This file is autogenerated by pip-compile with python 3.6 +# To update, run: +# +# pip-compile --output-file=requirements/_tools.txt --strip-extras requirements/_tools.in +# appdirs==1.4.4 # via black black==20.8b1 @@ -8,14 +14,22 @@ click==8.0.3 # via # black # pip-tools +dataclasses==0.7 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # black +importlib-metadata==2.0.0 + # via + # -c requirements/_test.txt + # click + # pep517 mypy-extensions==0.4.3 # via black pathspec==0.9.0 # via black pep517==0.12.0 # via pip-tools -pip==24.2 - # via pip-tools pip-tools==6.4.0 # via -r requirements/_tools.in pyyaml==5.4 @@ -25,8 +39,6 @@ pyyaml==5.4 # watchdog regex==2022.1.18 # via black -setuptools==75.1.0 - # via pip-tools toml==0.10.2 # via # -c requirements/_test.txt @@ -43,3 +55,12 @@ watchdog==2.1.6 # via -r requirements/_tools.in wheel==0.37.1 # via pip-tools +zipp==3.4.0 + # via + # -c requirements/_test.txt + # importlib-metadata + # pep517 + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/services/dynamic-scheduler/Dockerfile b/services/dynamic-scheduler/Dockerfile index bc94b83125a..b3e9119c898 100644 --- a/services/dynamic-scheduler/Dockerfile +++ b/services/dynamic-scheduler/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -69,29 +74,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/dynamic-scheduler/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + @@ -110,9 +108,9 @@ WORKDIR /build/services/dynamic-scheduler RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/dynamic-scheduler,target=/build/services/dynamic-scheduler,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/dynamic-scheduler/docker/boot.sh b/services/dynamic-scheduler/docker/boot.sh index 7164715d7dc..0660b8c8cc0 100755 --- a/services/dynamic-scheduler/docker/boot.sh +++ b/services/dynamic-scheduler/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/dynamic-scheduler || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/dynamic-scheduler + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/dynamic-scheduler/docker/entrypoint.sh b/services/dynamic-scheduler/docker/entrypoint.sh index 908e8f9e64b..25153a6b2a2 100755 --- a/services/dynamic-scheduler/docker/entrypoint.sh +++ b/services/dynamic-scheduler/docker/entrypoint.sh @@ -63,11 +63,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/dynamic-sidecar/Dockerfile b/services/dynamic-sidecar/Dockerfile index 3be8bae74d0..5eb6662593f 100644 --- a/services/dynamic-sidecar/Dockerfile +++ b/services/dynamic-sidecar/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # USAGE: @@ -53,14 +58,14 @@ RUN \ ./install_rclone.bash RUN AWS_CLI_VERSION="2.11.11" \ - && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip" -o "awscliv2.zip" \ - && apt-get update && apt-get install -y unzip \ - && unzip awscliv2.zip \ - && ./aws/install \ - && apt-get remove --purge -y unzip \ - && rm awscliv2.zip \ - && rm -rf awscliv2 \ - && aws --version + && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip" -o "awscliv2.zip" \ + && apt-get update && apt-get install -y unzip \ + && unzip awscliv2.zip \ + && ./aws/install \ + && apt-get remove --purge -y unzip \ + && rm awscliv2.zip \ + && rm -rf awscliv2 \ + && aws --version # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) ENV SC_USER_ID=8004 \ @@ -110,31 +115,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" \ && mkdir -p "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build -# install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/dynamic-sidecar/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/pip \ - uv pip install \ - --requirement _base.txt # copy utility devops scripts COPY --chown=scu:scu services/dynamic-sidecar/scripts/Makefile /home/scu @@ -155,9 +151,9 @@ WORKDIR /build/services/dynamic-sidecar RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/dynamic-sidecar,target=/build/services/dynamic-sidecar,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list # --------------------------Production stage ------------------- @@ -173,6 +169,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/dynamic-sidecar/docker/boot.sh b/services/dynamic-sidecar/docker/boot.sh index ff63ce74cfb..d21e87cb070 100755 --- a/services/dynamic-sidecar/docker/boot.sh +++ b/services/dynamic-sidecar/docker/boot.sh @@ -25,12 +25,17 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then # NOTE: uv does not like this requirement file... cd /devel/services/dynamic-sidecar - pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" pip list | sed 's/^/ /' fi +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy +fi + # # RUNNING application # diff --git a/services/dynamic-sidecar/docker/entrypoint.sh b/services/dynamic-sidecar/docker/entrypoint.sh index e326c25825b..9ccf61d8f67 100755 --- a/services/dynamic-sidecar/docker/entrypoint.sh +++ b/services/dynamic-sidecar/docker/entrypoint.sh @@ -74,11 +74,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock if stat $DOCKER_MOUNT >/dev/null 2>&1; then diff --git a/services/efs-guardian/Dockerfile b/services/efs-guardian/Dockerfile index 229e43563c4..82174799ca5 100644 --- a/services/efs-guardian/Dockerfile +++ b/services/efs-guardian/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -107,29 +112,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/efs-guardian/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -147,9 +145,9 @@ WORKDIR /build/services/efs-guardian RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/efs-guardian,target=/build/services/efs-guardian,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -166,6 +164,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/efs diff --git a/services/efs-guardian/docker/boot.sh b/services/efs-guardian/docker/boot.sh index d4d046d5a14..0575c7679fb 100755 --- a/services/efs-guardian/docker/boot.sh +++ b/services/efs-guardian/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/efs-guardian || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/efs-guardian + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/invitations/Dockerfile b/services/invitations/Dockerfile index 5d935426c48..0422ca73cbd 100644 --- a/services/invitations/Dockerfile +++ b/services/invitations/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -69,29 +74,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/invitations/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + @@ -110,9 +108,9 @@ WORKDIR /build/services/invitations RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/invitations,target=/build/services/invitations,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/invitations/docker/boot.sh b/services/invitations/docker/boot.sh index b6ed691759f..7e771439871 100755 --- a/services/invitations/docker/boot.sh +++ b/services/invitations/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/invitations || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/invitations + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/invitations/docker/entrypoint.sh b/services/invitations/docker/entrypoint.sh index 908e8f9e64b..25153a6b2a2 100755 --- a/services/invitations/docker/entrypoint.sh +++ b/services/invitations/docker/entrypoint.sh @@ -63,11 +63,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/migration/Dockerfile b/services/migration/Dockerfile index d70eec97f93..30a20c6df93 100644 --- a/services/migration/Dockerfile +++ b/services/migration/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base LABEL maintainer=sanderegg @@ -43,16 +48,14 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ build-essential \ git -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools @@ -61,7 +64,7 @@ WORKDIR /build/packages/postgres-database # install only base 3rd party dependencies RUN \ --mount=type=bind,source=packages/postgres-database,target=/build/packages/postgres-database,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ + --mount=type=cache,target=/root/.cache/uv \ uv pip install \ --requirement requirements/prod.txt \ && uv pip list @@ -71,6 +74,8 @@ RUN \ FROM base AS production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 # testing defaults ENV POSTGRES_USER=scu \ diff --git a/services/osparc-gateway-server/Dockerfile b/services/osparc-gateway-server/Dockerfile index 59f55c8b3b6..899ef0fb434 100644 --- a/services/osparc-gateway-server/Dockerfile +++ b/services/osparc-gateway-server/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bullseye AS base ARG TARGETPLATFORM ARG BUILDPLATFORM @@ -72,26 +77,19 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ git \ golang-go -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build -# install base 3rd party dependencies (NOTE: this speeds up devel mode) -RUN \ - --mount=type=bind,source=services/osparc-gateway-server/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # in ARM64 mode there is a catch: the piwheels package does not contain the dask-gateway-proxy executable in 64-bit RUN dpkgArch="$(dpkg --print-architecture)";\ @@ -117,7 +115,7 @@ WORKDIR /build/services/osparc-gateway-server RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/osparc-gateway-server,target=/build/services/osparc-gateway-server,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ + --mount=type=cache,target=/root/.cache/uv \ uv pip install \ --requirement requirements/prod.txt @@ -134,6 +132,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu # ensure home folder is read/writable for user scu diff --git a/services/osparc-gateway-server/docker/boot.sh b/services/osparc-gateway-server/docker/boot.sh index d1daccc6195..d2b912eb3ba 100755 --- a/services/osparc-gateway-server/docker/boot.sh +++ b/services/osparc-gateway-server/docker/boot.sh @@ -18,22 +18,27 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then echo "$INFO" "Python :" python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/osparc-gateway-server || exit 1 - pip install --no-cache-dir -r requirements/dev.txt - cd - || exit 1 + cd services/osparc-gateway-server + uv pip install --no-cache-dir -r requirements/dev.txt + cd - echo "$INFO" "PIP :" pip list | sed 's/^/ /' fi +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy +fi + if [ "${SC_BOOT_MODE}" = "debug" ]; then exec python -m debugpy --listen 0.0.0.0:"${OSPARC_GATEWAY_SERVER_DEBUGGING_PORT}" -m watchmedo auto-restart \ - --recursive \ - --pattern="*.py;*/src/*" \ - --ignore-patterns="*test*;pytest_simcore/*;setup.py;*ignore*" \ - --ignore-directories -- \ - osparc-gateway-server \ - --config "${GATEWAY_SERVER_CONFIG_FILE_CONTAINER}" \ - --debug + --recursive \ + --pattern="*.py;*/src/*" \ + --ignore-patterns="*test*;pytest_simcore/*;setup.py;*ignore*" \ + --ignore-directories -- \ + osparc-gateway-server \ + --config "${GATEWAY_SERVER_CONFIG_FILE_CONTAINER}" \ + --debug else exec osparc-gateway-server \ --config "${GATEWAY_SERVER_CONFIG_FILE_CONTAINER}" diff --git a/services/osparc-gateway-server/docker/entrypoint.sh b/services/osparc-gateway-server/docker/entrypoint.sh index 91204dd9b60..cd8eb9a01ef 100755 --- a/services/osparc-gateway-server/docker/entrypoint.sh +++ b/services/osparc-gateway-server/docker/entrypoint.sh @@ -15,74 +15,62 @@ ERROR="ERROR: [$(basename "$0")] " # *runs* as non-root user [scu] # echo "$INFO" "Entrypoint for stage ${SC_BUILD_TARGET} ..." -echo User :"$(id "$(whoami)")" -echo Workdir :"$(pwd)" -echo scuUser :"$(id scu)" +echo User :"$(id "$(whoami)")" +echo Workdir :"$(pwd)" +echo scuUser :"$(id scu)" +if [ "${SC_BUILD_TARGET}" = "development" ]; then + echo "$INFO" "development mode detected..." + # NOTE: expects docker run ... -v $(pwd):/devel/services/osparc-gateway-server + DEVEL_MOUNT=${DEVEL_MOUNT:="/devel/services/osparc-gateway-server"} -if [ "${SC_BUILD_TARGET}" = "development" ] -then - echo "$INFO" "development mode detected..." - # NOTE: expects docker run ... -v $(pwd):/devel/services/osparc-gateway-server - DEVEL_MOUNT=${DEVEL_MOUNT:="/devel/services/osparc-gateway-server"} + stat $DEVEL_MOUNT >/dev/null 2>&1 || + (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) - stat $DEVEL_MOUNT > /dev/null 2>&1 || \ - (echo "$ERROR" "You must mount '$DEVEL_MOUNT' to deduce user and group ids" && exit 1) - - echo "setting correct user id/group id..." - HOST_USERID=$(stat --format=%u "${DEVEL_MOUNT}") - HOST_GROUPID=$(stat --format=%g "${DEVEL_MOUNT}") - CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut --delimiter=: --fields=1) - if [ "$HOST_USERID" -eq 0 ] - then - echo "Warning: Folder mounted owned by root user... adding $SC_USER_NAME to root..." - adduser "$SC_USER_NAME" root + echo "setting correct user id/group id..." + HOST_USERID=$(stat --format=%u "${DEVEL_MOUNT}") + HOST_GROUPID=$(stat --format=%g "${DEVEL_MOUNT}") + CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut --delimiter=: --fields=1) + if [ "$HOST_USERID" -eq 0 ]; then + echo "Warning: Folder mounted owned by root user... adding $SC_USER_NAME to root..." + adduser "$SC_USER_NAME" root + else + echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." + # take host's credentials in $SC_USER_NAME + if [ -z "$CONT_GROUPNAME" ]; then + echo "Creating new group my$SC_USER_NAME" + CONT_GROUPNAME=my$SC_USER_NAME + addgroup --gid "$HOST_GROUPID" "$CONT_GROUPNAME" else - echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." - # take host's credentials in $SC_USER_NAME - if [ -z "$CONT_GROUPNAME" ] - then - echo "Creating new group my$SC_USER_NAME" - CONT_GROUPNAME=my$SC_USER_NAME - addgroup --gid "$HOST_GROUPID" "$CONT_GROUPNAME" - else - echo "group already exists" - fi - echo "adding $SC_USER_NAME to group $CONT_GROUPNAME..." - adduser "$SC_USER_NAME" "$CONT_GROUPNAME" - - echo "changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)" - usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME" - - echo "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" - find / -path /proc -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \; - # change user property of files already around - echo "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" - find / -path /proc -prune -o -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; + echo "group already exists" fi -fi + echo "adding $SC_USER_NAME to group $CONT_GROUPNAME..." + adduser "$SC_USER_NAME" "$CONT_GROUPNAME" + echo "changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)" + usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME" -if [ "${SC_BOOT_MODE}" = "debug" ] -then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy + echo "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \; + # change user property of files already around + echo "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; + fi fi DOCKER_MOUNT=/var/run/docker.sock if stat $DOCKER_MOUNT >/dev/null 2>&1; then - echo "$INFO detected docker socket is mounted, adding user to group..." - GROUPID=$(stat --format=%g $DOCKER_MOUNT) - GROUPNAME=scdocker + echo "$INFO detected docker socket is mounted, adding user to group..." + GROUPID=$(stat --format=%g $DOCKER_MOUNT) + GROUPNAME=scdocker - if ! addgroup --gid "$GROUPID" $GROUPNAME > /dev/null 2>&1 - then - echo "$WARNING docker group with $GROUPID already exists, getting group name..." - # if group already exists in container, then reuse name - GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) - echo "$WARNING docker group with $GROUPID has name $GROUPNAME" - fi - adduser "$SC_USER_NAME" "$GROUPNAME" + if ! addgroup --gid "$GROUPID" $GROUPNAME >/dev/null 2>&1; then + echo "$WARNING docker group with $GROUPID already exists, getting group name..." + # if group already exists in container, then reuse name + GROUPNAME=$(getent group "${GROUPID}" | cut --delimiter=: --fields=1) + echo "$WARNING docker group with $GROUPID has name $GROUPNAME" + fi + adduser "$SC_USER_NAME" "$GROUPNAME" fi echo "$INFO Starting osparc-gateway-server ..." diff --git a/services/payments/Dockerfile b/services/payments/Dockerfile index 90cbb1c908e..32fada11361 100644 --- a/services/payments/Dockerfile +++ b/services/payments/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -69,29 +74,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/payments/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + @@ -110,9 +108,9 @@ WORKDIR /build/services/payments RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/payments,target=/build/services/payments,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/payments/docker/boot.sh b/services/payments/docker/boot.sh index bf5b6881fa1..be92e7ac4fc 100755 --- a/services/payments/docker/boot.sh +++ b/services/payments/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/payments || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/payments + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/payments/docker/entrypoint.sh b/services/payments/docker/entrypoint.sh index 908e8f9e64b..25153a6b2a2 100755 --- a/services/payments/docker/entrypoint.sh +++ b/services/payments/docker/entrypoint.sh @@ -63,11 +63,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/resource-usage-tracker/Dockerfile b/services/resource-usage-tracker/Dockerfile index 55976c00c53..528ecabfd3e 100644 --- a/services/resource-usage-tracker/Dockerfile +++ b/services/resource-usage-tracker/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # @@ -70,29 +75,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ && apt-get install -y --no-install-recommends \ build-essential -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build # install base 3rd party dependencies -# NOTE: copies to /build to avoid overwriting later which would invalidate this layer -RUN \ - --mount=type=bind,source=services/resource-usage-tracker/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- @@ -110,9 +108,9 @@ WORKDIR /build/services/resource-usage-tracker RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/resource-usage-tracker,target=/build/services/resource-usage-tracker,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list @@ -129,6 +127,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/resource-usage-tracker/docker/boot.sh b/services/resource-usage-tracker/docker/boot.sh index 878b164a05e..d6e54f8f347 100755 --- a/services/resource-usage-tracker/docker/boot.sh +++ b/services/resource-usage-tracker/docker/boot.sh @@ -23,12 +23,16 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/resource-usage-tracker || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/resource-usage-tracker + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy fi # diff --git a/services/resource-usage-tracker/docker/entrypoint.sh b/services/resource-usage-tracker/docker/entrypoint.sh index a1a8c160032..e89ad5408a3 100755 --- a/services/resource-usage-tracker/docker/entrypoint.sh +++ b/services/resource-usage-tracker/docker/entrypoint.sh @@ -63,11 +63,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - # Appends docker group if socket is mounted DOCKER_MOUNT=/var/run/docker.sock if stat $DOCKER_MOUNT >/dev/null 2>&1; then diff --git a/services/storage/Dockerfile b/services/storage/Dockerfile index 12e60feb78d..8c9f0e381c0 100644 --- a/services/storage/Dockerfile +++ b/services/storage/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # USAGE: @@ -78,28 +83,21 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ build-essential \ git -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build -# install only base 3rd party dependencies -RUN \ - --mount=type=bind,source=services/storage/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # -------------------------- ------------------------------- @@ -122,9 +120,9 @@ WORKDIR /build/services/storage RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/storage,target=/build/services/storage,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list # -------------------------- ------------------------------- @@ -141,6 +139,8 @@ ENV SC_BUILD_TARGET=production \ SC_HEALTHCHECK_RETRY=3 ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index 8f4c4f64346..ea9bf8bd877 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -18,12 +18,11 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/storage || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/storage + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list echo "$INFO" "Setting entrypoint to use watchmedo autorestart..." entrypoint='watchmedo auto-restart --recursive --pattern="*.py;*/src/*" --ignore-patterns="*test*;pytest_simcore/*;setup.py;*ignore*" --ignore-directories --' @@ -32,6 +31,11 @@ elif [ "${SC_BUILD_TARGET}" = "production" ]; then entrypoint="" fi +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy +fi + APP_LOG_LEVEL=${STORAGE_LOGLEVEL:-${LOG_LEVEL:-${LOGLEVEL:-INFO}}} SERVER_LOG_LEVEL=$(echo "${APP_LOG_LEVEL}" | tr '[:upper:]' '[:lower:]') diff --git a/services/storage/docker/entrypoint.sh b/services/storage/docker/entrypoint.sh index 076dd425e67..11684a3a591 100755 --- a/services/storage/docker/entrypoint.sh +++ b/services/storage/docker/entrypoint.sh @@ -15,7 +15,6 @@ ERROR="ERROR: [$(basename "$0")] " # It needs to be executed as root. update-ca-certificates - # This entrypoint script: # # - Executes *inside* of the container upon start as --user [default root] @@ -29,7 +28,6 @@ echo "$INFO" "User : $(id scu)" echo "$INFO" "python : $(command -v python)" echo "$INFO" "pip : $(command -v pip)" - if [ "${SC_BUILD_TARGET}" = "development" ]; then echo "$INFO" "development mode detected..." # NOTE: expects docker run ... -v $(pwd):$DEVEL_MOUNT @@ -70,11 +68,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)" diff --git a/services/web/Dockerfile b/services/web/Dockerfile index 1a1a443333a..e7ff6b3fe41 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 + +# Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.4" +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # USAGE: @@ -81,29 +86,22 @@ RUN --mount=type=cache,target=/var/cache/apt,mode=0755,sharing=private \ build-essential \ libffi-dev -# NOTE: install https://github.com/astral-sh/uv ultra-fast rust-based pip replacement -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install uv~=0.2 +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ # NOTE: python virtualenv is used here such that installed # packages may be moved to production image easily by copying the venv RUN uv venv "${VIRTUAL_ENV}" -RUN --mount=type=cache,mode=0755,target=/root/.cache/uv \ +RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade \ - pip~=24.0 \ wheel \ setuptools WORKDIR /build -# install only base 3rd party dependencies -RUN \ - --mount=type=bind,source=services/web/server/requirements/_base.txt,target=_base.txt \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement _base.txt + # --------------------------Prod-depends-only stage ------------------- # This stage is for production only dependencies that get partially wiped out afterwards (final docker image concerns) @@ -121,9 +119,9 @@ WORKDIR /build/services/web/server RUN \ --mount=type=bind,source=packages,target=/build/packages,rw \ --mount=type=bind,source=services/web/server,target=/build/services/web/server,rw \ - --mount=type=cache,mode=0755,target=/root/.cache/uv \ - uv pip install \ - --requirement requirements/prod.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ && uv pip list # --------------------------Production stage ------------------- @@ -139,6 +137,8 @@ ENV SC_BUILD_TARGET=production \ SC_BOOT_MODE=production ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 WORKDIR /home/scu diff --git a/services/web/server/docker/boot.sh b/services/web/server/docker/boot.sh index e6b36ce5788..08a812f7bb0 100755 --- a/services/web/server/docker/boot.sh +++ b/services/web/server/docker/boot.sh @@ -18,18 +18,22 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then python --version | sed 's/^/ /' command -v python | sed 's/^/ /' - cd services/web/server || exit 1 - pip install uv - uv pip --quiet --no-cache-dir install -r requirements/dev.txt - cd - || exit 1 + cd services/web/server + uv pip --quiet --no-cache-dir sync requirements/dev.txt + cd - echo "$INFO" "PIP :" - uv pip list | sed 's/^/ /' + uv pip list APP_CONFIG=server-docker-dev.yaml elif [ "${SC_BUILD_TARGET}" = "production" ]; then APP_CONFIG=server-docker-prod.yaml fi +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install --no-cache-dir debugpy +fi + APP_LOG_LEVEL=${WEBSERVER_LOGLEVEL:-${LOG_LEVEL:-${LOGLEVEL:-INFO}}} SERVER_LOG_LEVEL=$(echo "${APP_LOG_LEVEL}" | tr '[:upper:]' '[:lower:]') diff --git a/services/web/server/docker/entrypoint.sh b/services/web/server/docker/entrypoint.sh index d488a0ef2c7..d7e4f13bb7a 100755 --- a/services/web/server/docker/entrypoint.sh +++ b/services/web/server/docker/entrypoint.sh @@ -71,11 +71,6 @@ if [ "${SC_BUILD_TARGET}" = "development" ]; then fi fi -if [ "${SC_BOOT_MODE}" = "debug" ]; then - # NOTE: production does NOT pre-installs debugpy - pip install --no-cache-dir debugpy -fi - echo "$INFO Starting $* ..." echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" echo " local dir : $(ls -al)"