diff --git a/changelog.d/20230427_154822_regis_build_mount.md b/changelog.d/20230427_154822_regis_build_mount.md new file mode 100644 index 00000000000..dbc5e908de1 --- /dev/null +++ b/changelog.d/20230427_154822_regis_build_mount.md @@ -0,0 +1 @@ +- [Improvement] Considerably accelerate building the "openedx" Docker image with `RUN --mount=type=cache`. This feature is only for Docker with BuildKit, so detection is performed at build-time. (by @regisb) diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index 20abdd2bd1e..622be9b8aa7 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -1,7 +1,7 @@ from unittest.mock import Mock, patch from tests.helpers import PluginsTestCase, temporary_root -from tutor import images, plugins +from tutor import images, plugins, utils from tutor.__about__ import __version__ from tutor.commands.images import ImageNotFoundError @@ -128,16 +128,29 @@ def test_images_build_plugin_with_args(self, image_build: Mock) -> None: "service1", ] with temporary_root() as root: - self.invoke_in_root(root, ["config", "save"]) - result = self.invoke_in_root(root, build_args) + utils.is_buildkit_enabled.cache_clear() + with patch.object(utils, "is_buildkit_enabled", return_value=False): + self.invoke_in_root(root, ["config", "save"]) + result = self.invoke_in_root(root, build_args) self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) image_build.assert_called() self.assertIn("service1:1.0.0", image_build.call_args[0]) - for arg in image_build.call_args[0][2:]: - # The only extra args are `--build-arg` - if arg != "--build-arg": - self.assertIn(arg, build_args) + self.assertEqual( + [ + "service1:1.0.0", + "--no-cache", + "--build-arg", + "myarg=value", + "--add-host", + "host", + "--target", + "target", + "docker_args", + "--cache-from=type=registry,ref=service1:1.0.0-cache", + ], + list(image_build.call_args[0][1:]) + ) def test_images_push(self) -> None: result = self.invoke(["images", "push"]) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index c300ff80616..1dfce27b8f5 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -83,6 +83,14 @@ def images_command() -> None: is_flag=True, help="Push the build cache to the remote registry. You should only enable this option if you have push rights to the remote registry.", ) +@click.option( + "--output", + "docker_output", + # Export image to docker. This is necessary to make the image available to docker-compose. + # The `--load` option is a shorthand for `--output=type=docker`. + default="type=docker", + help="Same as `docker build --output=...`. This option will only be used when BuildKit is enabled.", +) @click.option( "-a", "--build-arg", @@ -114,6 +122,7 @@ def build( no_cache: bool, no_registry_cache: bool, cache_to_registry: bool, + docker_output: str, build_args: list[str], add_hosts: list[str], target: str, @@ -135,9 +144,8 @@ def build( command_args += ["--add-host", add_host] if target: command_args += ["--target", target] - if utils.is_buildkit_enabled(): - # Export image to docker. - command_args.append("--output=type=image") + if utils.is_buildkit_enabled() and docker_output: + command_args.append(f"--output={docker_output}") if docker_args: command_args += docker_args for image in image_names: diff --git a/tutor/env.py b/tutor/env.py index e24e90e5f88..2460ec8a70e 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -54,6 +54,7 @@ def _prepare_environment() -> None: ("HOST_USER_ID", utils.get_user_id()), ("TUTOR_APP", __app__.replace("-", "_")), ("TUTOR_VERSION", __version__), + ("is_buildkit_enabled", utils.is_buildkit_enabled), ], ) diff --git a/tutor/images.py b/tutor/images.py index d0637fe4903..0d1e80bd578 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -9,9 +9,10 @@ def get_tag(config: Config, name: str) -> str: def build(path: str, tag: str, *args: str) -> None: fmt.echo_info(f"Building image {tag}") - command = hooks.Filters.DOCKER_BUILD_COMMAND.apply( - ["build", "-t", tag, *args, path] - ) + build_command = ["build", "-t", tag, *args, path] + if utils.is_buildkit_enabled(): + build_command.insert(0, "buildx") + command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(build_command) utils.docker(*command) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 3bbc842c210..e9284b49bb5 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -1,9 +1,12 @@ +{% if is_buildkit_enabled() %}# syntax=docker/dockerfile:1.4{% endif %} ###### Minimal image with base system requirements for most stages FROM docker.io/ubuntu:20.04 as minimal LABEL maintainer="Overhang.io " ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && \ +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked{% endif %} \ + apt update && \ apt install -y build-essential curl git language-pack-en ENV LC_ALL en_US.UTF-8 {{ patch("openedx-dockerfile-minimal") }} @@ -11,16 +14,23 @@ ENV LC_ALL en_US.UTF-8 ###### Install python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv FROM minimal as python # https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites -RUN apt update && \ +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \ apt install -y libssl-dev zlib1g-dev libbz2-dev \ libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev python-openssl git -# https://github.com/pyenv/pyenv/releases + +# Install pyenv # https://www.python.org/downloads/ +# https://github.com/pyenv/pyenv/releases ARG PYTHON_VERSION=3.8.15 ENV PYENV_ROOT /opt/pyenv RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.3.17 --depth 1 + +# Install Python RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION + +# Create virtualenv RUN $PYENV_ROOT/versions/$PYTHON_VERSION/bin/python -m venv /openedx/venv ###### Checkout edx-platform code @@ -45,6 +55,12 @@ RUN git config --global user.email "tutor@overhang.io" \ {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/.patch | git am #} {{ patch("openedx-dockerfile-post-git-checkout") }} +##### Empty layer with just the repo at the root. +# This is useful when overriding the build context with a host repo: +# docker build --build-context edx-platform=/path/to/edx-platform +FROM scratch as edx-platform +COPY --from=code /openedx/edx-platform / + ###### Download extra locales to /openedx/locale/contrib/locale FROM minimal as locales ARG OPENEDX_I18N_VERSION={{ OPENEDX_COMMON_VERSION }} @@ -59,36 +75,39 @@ RUN cd /tmp \ FROM python as python-requirements ENV PATH /openedx/venv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ +ENV XDG_CACHE_HOME /openedx/.cache -RUN apt update && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \ + && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev # Install the right version of pip/setuptools -# https://pypi.org/project/setuptools/ -# https://pypi.org/project/pip/ -# https://pypi.org/project/wheel/ -RUN pip install setuptools==67.6.1 pip==23.0.1. wheel==0.40.0 +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \ + # https://pypi.org/project/setuptools/ + # https://pypi.org/project/pip/ + # https://pypi.org/project/wheel/ + setuptools==67.6.1 pip==23.0.1. wheel==0.40.0 # Install base requirements -COPY --from=code /openedx/edx-platform/requirements/edx/base.txt /tmp/base.txt -RUN pip install -r /tmp/base.txt +RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \ + --mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r /openedx/edx-platform/requirements/edx/base.txt -# Install django-redis for using redis as a django cache -# https://pypi.org/project/django-redis/ -RUN pip install django-redis==5.2.0 - -# Install uwsgi -# https://pypi.org/project/uWSGI/ -RUN pip install uwsgi==2.0.21 +# Install extra requirements +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \ + # Use redis as a django cache https://pypi.org/project/django-redis/ + django-redis==5.2.0 \ + # uwsgi server https://pypi.org/project/uWSGI/ + uwsgi==2.0.21 {{ patch("openedx-dockerfile-post-python-requirements") }} # Install private requirements: this is useful for installing custom xblocks. COPY ./requirements/ /openedx/requirements -RUN cd /openedx/requirements/ \ +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}cd /openedx/requirements/ \ && touch ./private.txt \ && pip install -r ./private.txt -{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN pip install '{{ extra_requirements }}' +{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}' {% endfor %} ###### Install nodejs with nodeenv in /openedx/nodeenv @@ -103,18 +122,18 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt # Install nodejs requirements ARG NPM_REGISTRY={{ NPM_REGISTRY }} -COPY --from=code /openedx/edx-platform/package.json /openedx/edx-platform/package.json -COPY --from=code /openedx/edx-platform/package-lock.json /openedx/edx-platform/package-lock.json WORKDIR /openedx/edx-platform -RUN npm clean-install --verbose --registry=$NPM_REGISTRY +RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ + --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \ + --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --verbose --no-audit --registry=$NPM_REGISTRY ###### Production image with system and python requirements FROM minimal as production # Install system requirements -RUN apt update && \ - apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind && \ - rm -rf /var/lib/apt/lists/* +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \ + && apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind # From then on, run as unprivileged "app" user # Note that this must always be different from root (APP_USER_ID=0) @@ -124,14 +143,17 @@ RUN useradd --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER USER ${APP_USER_ID} # https://hub.docker.com/r/powerman/dockerize/tags -COPY --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize -COPY --chown=app:app --from=code /openedx/edx-platform /openedx/edx-platform +COPY {% if is_buildkit_enabled() %}--link {% endif %}--from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize +COPY --chown=app:app --from=edx-platform / /openedx/edx-platform COPY --chown=app:app --from=locales /openedx/locale /openedx/locale COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv COPY --chown=app:app --from=python-requirements /openedx/venv /openedx/venv COPY --chown=app:app --from=python-requirements /openedx/requirements /openedx/requirements COPY --chown=app:app --from=nodejs-requirements /openedx/nodeenv /openedx/nodeenv -COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/edx-platform/node_modules +COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/node_modules + +# Symlink node_modules such that we can bind-mount the edx-platform repository +RUN ln -s /openedx/node_modules /openedx/edx-platform/node_modules ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ @@ -215,16 +237,16 @@ FROM production as development # Install useful system requirements (as root) USER root -RUN apt update && \ - apt install -y vim iputils-ping dnsutils telnet \ - && rm -rf /var/lib/apt/lists/* +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \ + apt install -y vim iputils-ping dnsutils telnet USER app # Install dev python requirements -RUN pip install -r requirements/edx/development.txt +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r requirements/edx/development.txt # https://pypi.org/project/ipdb/ # https://pypi.org/project/ipython -RUN pip install ipdb==0.13.13 ipython==8.12.0 +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install ipdb==0.13.13 ipython==8.12.0 # Add ipdb as default PYTHONBREAKPOINT ENV PYTHONBREAKPOINT=ipdb.set_trace diff --git a/tutor/utils.py b/tutor/utils.py index 545ce38b204..06c0e3d0b98 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -173,6 +173,25 @@ def docker(*command: str) -> int: return execute("docker", *command) +@lru_cache(maxsize=None) +def is_buildkit_enabled() -> bool: + """ + A helper function to determine whether we can run `docker buildx` with BuildKit. + """ + # First, we respect the DOCKER_BUILDKIT environment variable + enabled_by_env = { + "1": True, + "0": False, + }.get(os.environ.get("DOCKER_BUILDKIT", "")) + if enabled_by_env is not None: + return enabled_by_env + try: + subprocess.run(["docker", "buildx", "version"], capture_output=True, check=True) + return True + except subprocess.CalledProcessError: + return False + + def docker_compose(*command: str) -> int: return execute("docker", "compose", *command)