From add1063106b58fbc05465d411b49336af94d9718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 3 Apr 2023 20:44:15 +0200 Subject: [PATCH] feat: leverage `RUN --mount` for faster image building We make use of the Docker build cache to install python and nodejs requirements faster in the case of repeated builds. This feature is only possible for users of BuildKit, so we detect whether `docker buildx` is available at runtime. We do not make use of `COPY --link` because the `--link` option is incompatible with `--chown=app:app`: https://github.com/docker/buildx/issues/1408 For reference, see: https://www.docker.com/blog/dockerfiles-now-support-multiple-build-contexts/ https://docs.docker.com/engine/reference/commandline/buildx_build/#build-context --- .../20230427_154822_regis_build_mount.md | 1 + tests/commands/test_images.py | 27 ++++-- tutor/commands/images.py | 5 +- tutor/env.py | 1 + tutor/images.py | 7 +- tutor/templates/build/openedx/Dockerfile | 87 ++++++++++++------- tutor/utils.py | 19 ++++ 7 files changed, 102 insertions(+), 45 deletions(-) create mode 100644 changelog.d/20230427_154822_regis_build_mount.md diff --git a/changelog.d/20230427_154822_regis_build_mount.md b/changelog.d/20230427_154822_regis_build_mount.md new file mode 100644 index 00000000000..dbc5e908de1 --- /dev/null +++ b/changelog.d/20230427_154822_regis_build_mount.md @@ -0,0 +1 @@ +- [Improvement] Considerably accelerate building the "openedx" Docker image with `RUN --mount=type=cache`. This feature is only for Docker with BuildKit, so detection is performed at build-time. (by @regisb) diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index 99e74e941b0..5c1c2933147 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -1,7 +1,7 @@ from unittest.mock import Mock, patch from tests.helpers import PluginsTestCase, temporary_root -from tutor import images, plugins +from tutor import images, plugins, utils from tutor.__about__ import __version__ from tutor.commands.images import ImageNotFoundError @@ -128,16 +128,29 @@ def test_images_build_plugin_with_args(self, image_build: Mock) -> None: "service1", ] with temporary_root() as root: - self.invoke_in_root(root, ["config", "save"]) - result = self.invoke_in_root(root, build_args) + utils.is_buildkit_enabled.cache_clear() + with patch.object(utils, "is_buildkit_enabled", return_value=False): + self.invoke_in_root(root, ["config", "save"]) + result = self.invoke_in_root(root, build_args) self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) image_build.assert_called() self.assertIn("service1:1.0.0", image_build.call_args[0]) - for arg in image_build.call_args[0][2:]: - # The only extra args are `--build-arg` - if arg != "--build-arg": - self.assertIn(arg, build_args) + self.assertEqual( + [ + "service1:1.0.0", + "--no-cache", + "--build-arg", + "myarg=value", + "--add-host", + "host", + "--target", + "target", + "docker_args", + "--cache-from=type=registry,ref=service1:1.0.0-cache", + ], + list(image_build.call_args[0][1:]) + ) def test_images_push(self) -> None: result = self.invoke(["images", "push"]) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index c300ff80616..a162bee086c 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -136,8 +136,9 @@ def build( if target: command_args += ["--target", target] if utils.is_buildkit_enabled(): - # Export image to docker. - command_args.append("--output=type=image") + # Export image to docker. This is necessary to make the image available to docker-compose. + # The `--load` option is a shorthand for `--output=type=docker`. + command_args.append("--load") if docker_args: command_args += docker_args for image in image_names: diff --git a/tutor/env.py b/tutor/env.py index 1c35fdba963..532a9fd87e0 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -54,6 +54,7 @@ def _prepare_environment() -> None: ("HOST_USER_ID", utils.get_user_id()), ("TUTOR_APP", __app__.replace("-", "_")), ("TUTOR_VERSION", __version__), + ("is_buildkit_enabled", utils.is_buildkit_enabled), ], ) diff --git a/tutor/images.py b/tutor/images.py index d0637fe4903..0d1e80bd578 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -9,9 +9,10 @@ def get_tag(config: Config, name: str) -> str: def build(path: str, tag: str, *args: str) -> None: fmt.echo_info(f"Building image {tag}") - command = hooks.Filters.DOCKER_BUILD_COMMAND.apply( - ["build", "-t", tag, *args, path] - ) + build_command = ["build", "-t", tag, *args, path] + if utils.is_buildkit_enabled(): + build_command.insert(0, "buildx") + command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(build_command) utils.docker(*command) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 3bbc842c210..14c6ead7e95 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -3,7 +3,9 @@ FROM docker.io/ubuntu:20.04 as minimal LABEL maintainer="Overhang.io " ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && \ +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked{% endif %} \ + apt update && \ apt install -y build-essential curl git language-pack-en ENV LC_ALL en_US.UTF-8 {{ patch("openedx-dockerfile-minimal") }} @@ -11,16 +13,23 @@ ENV LC_ALL en_US.UTF-8 ###### Install python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv FROM minimal as python # https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites -RUN apt update && \ +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \ apt install -y libssl-dev zlib1g-dev libbz2-dev \ libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev python-openssl git -# https://github.com/pyenv/pyenv/releases + +# Install pyenv # https://www.python.org/downloads/ +# https://github.com/pyenv/pyenv/releases ARG PYTHON_VERSION=3.8.15 ENV PYENV_ROOT /opt/pyenv RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.3.17 --depth 1 + +# Install Python RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION + +# Create virtualenv RUN $PYENV_ROOT/versions/$PYTHON_VERSION/bin/python -m venv /openedx/venv ###### Checkout edx-platform code @@ -45,6 +54,12 @@ RUN git config --global user.email "tutor@overhang.io" \ {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/.patch | git am #} {{ patch("openedx-dockerfile-post-git-checkout") }} +##### Empty layer with just the repo at the root. +# This is useful when overriding the build context with a host repo: +# docker build --build-context edx-platform=/path/to/edx-platform +FROM scratch as edx-platform +COPY --from=code /openedx/edx-platform / + ###### Download extra locales to /openedx/locale/contrib/locale FROM minimal as locales ARG OPENEDX_I18N_VERSION={{ OPENEDX_COMMON_VERSION }} @@ -59,36 +74,39 @@ RUN cd /tmp \ FROM python as python-requirements ENV PATH /openedx/venv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ +ENV XDG_CACHE_HOME /openedx/.cache -RUN apt update && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \ + && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev # Install the right version of pip/setuptools -# https://pypi.org/project/setuptools/ -# https://pypi.org/project/pip/ -# https://pypi.org/project/wheel/ -RUN pip install setuptools==67.6.1 pip==23.0.1. wheel==0.40.0 +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \ + # https://pypi.org/project/setuptools/ + # https://pypi.org/project/pip/ + # https://pypi.org/project/wheel/ + setuptools==67.6.1 pip==23.0.1. wheel==0.40.0 # Install base requirements -COPY --from=code /openedx/edx-platform/requirements/edx/base.txt /tmp/base.txt -RUN pip install -r /tmp/base.txt +RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \ + --mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r /openedx/edx-platform/requirements/edx/base.txt -# Install django-redis for using redis as a django cache -# https://pypi.org/project/django-redis/ -RUN pip install django-redis==5.2.0 - -# Install uwsgi -# https://pypi.org/project/uWSGI/ -RUN pip install uwsgi==2.0.21 +# Install extra requirements +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \ + # Use redis as a django cache https://pypi.org/project/django-redis/ + django-redis==5.2.0 \ + # uwsgi server https://pypi.org/project/uWSGI/ + uwsgi==2.0.21 {{ patch("openedx-dockerfile-post-python-requirements") }} # Install private requirements: this is useful for installing custom xblocks. COPY ./requirements/ /openedx/requirements -RUN cd /openedx/requirements/ \ +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}cd /openedx/requirements/ \ && touch ./private.txt \ && pip install -r ./private.txt -{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN pip install '{{ extra_requirements }}' +{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}' {% endfor %} ###### Install nodejs with nodeenv in /openedx/nodeenv @@ -103,18 +121,18 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt # Install nodejs requirements ARG NPM_REGISTRY={{ NPM_REGISTRY }} -COPY --from=code /openedx/edx-platform/package.json /openedx/edx-platform/package.json -COPY --from=code /openedx/edx-platform/package-lock.json /openedx/edx-platform/package-lock.json WORKDIR /openedx/edx-platform -RUN npm clean-install --verbose --registry=$NPM_REGISTRY +RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ + --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \ + --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --verbose --no-audit --registry=$NPM_REGISTRY ###### Production image with system and python requirements FROM minimal as production # Install system requirements -RUN apt update && \ - apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind && \ - rm -rf /var/lib/apt/lists/* +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \ + && apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind # From then on, run as unprivileged "app" user # Note that this must always be different from root (APP_USER_ID=0) @@ -124,14 +142,17 @@ RUN useradd --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER USER ${APP_USER_ID} # https://hub.docker.com/r/powerman/dockerize/tags -COPY --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize -COPY --chown=app:app --from=code /openedx/edx-platform /openedx/edx-platform +COPY {% if is_buildkit_enabled() %}--link {% endif %}--from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize +COPY --chown=app:app --from=edx-platform / /openedx/edx-platform COPY --chown=app:app --from=locales /openedx/locale /openedx/locale COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv COPY --chown=app:app --from=python-requirements /openedx/venv /openedx/venv COPY --chown=app:app --from=python-requirements /openedx/requirements /openedx/requirements COPY --chown=app:app --from=nodejs-requirements /openedx/nodeenv /openedx/nodeenv -COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/edx-platform/node_modules +COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/node_modules + +# Symlink node_modules such that we can bind-mount the edx-platform repository +RUN ln -s /openedx/node_modules /openedx/edx-platform/node_modules ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ @@ -215,16 +236,16 @@ FROM production as development # Install useful system requirements (as root) USER root -RUN apt update && \ - apt install -y vim iputils-ping dnsutils telnet \ - && rm -rf /var/lib/apt/lists/* +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \ + apt install -y vim iputils-ping dnsutils telnet USER app # Install dev python requirements -RUN pip install -r requirements/edx/development.txt +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r requirements/edx/development.txt # https://pypi.org/project/ipdb/ # https://pypi.org/project/ipython -RUN pip install ipdb==0.13.13 ipython==8.12.0 +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install ipdb==0.13.13 ipython==8.12.0 # Add ipdb as default PYTHONBREAKPOINT ENV PYTHONBREAKPOINT=ipdb.set_trace diff --git a/tutor/utils.py b/tutor/utils.py index d355f345b18..705731d3b60 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -173,6 +173,25 @@ def docker(*command: str) -> int: return execute("docker", *command) +@lru_cache(maxsize=None) +def is_buildkit_enabled() -> bool: + """ + A helper function to determine whether we can run `docker buildx` with BuildKit. + """ + # First, we respect the DOCKER_BUILDKIT environment variable + enabled_by_env = { + "1": True, + "0": False, + }.get(os.environ.get("DOCKER_BUILDKIT", "")) + if enabled_by_env is not None: + return enabled_by_env + try: + subprocess.run(["docker", "buildx", "version"], capture_output=True, check=True) + return True + except subprocess.CalledProcessError: + return False + + def docker_compose(*command: str) -> int: return execute("docker", "compose", *command)