feat: tutor mounts populate

TODOs:
* describe commit
* address TODOs in code
* should it be `tutor dev populate-mounts` instead?
* add to `tutor dev launch`?
* circulate a TEP?

Part of: TODO link ticket

tutor/commands/jobs.py

@@ -42,15 +42,6 @@ def _add_core_init_tasks() -> None:
             ("mysql", env.read_core_template_file("jobs", "init", "mysql.sh"))
         )
     with hooks.Contexts.app("lms").enter():
-        hooks.Filters.CLI_DO_INIT_TASKS.add_item(
-            (
-                "lms",
-                env.read_core_template_file("jobs", "init", "mounted-edx-platform.sh"),
-            ),
-            # If edx-platform is mounted, then we may need to perform some setup
-            # before other initialization scripts can be run.
-            priority=priorities.HIGH,
-        )
         hooks.Filters.CLI_DO_INIT_TASKS.add_item(
             ("lms", env.read_core_template_file("jobs", "init", "lms.sh"))
         )

tutor/commands/mounts.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+from collections import defaultdict
 
 import click
 import yaml
@@ -10,7 +11,13 @@
 from tutor import exceptions, fmt, hooks
 from tutor.commands.config import save as config_save
 from tutor.commands.context import Context
+from tutor.commands.images import (
+    find_images_to_build,
+    find_remote_image_tags,
+    ImageNotFoundError,
+)
 from tutor.commands.params import ConfigLoaderParam
+from tutor.utils import execute as execute_shell
 
 
 class MountParamType(ConfigLoaderParam):
@@ -73,8 +80,9 @@ def mounts_list(context: Context) -> None:
 
 @click.command(name="add")
 @click.argument("mounts", metavar="mount", type=click.Path(), nargs=-1)
+@click.option("-p", "--populate", is_flag=True, help="Populate mount after adding it")
 @click.pass_context
-def mounts_add(context: click.Context, mounts: list[str]) -> None:
+def mounts_add(context: click.Context, mounts: list[str], populate: bool) -> None:
     """
     Add a bind-mounted folder
 
@@ -98,18 +106,24 @@ def mounts_add(context: click.Context, mounts: list[str]) -> None:
     explicit form.
     """
     new_mounts = []
+    implicit_mounts = []
+
     for mount in mounts:
         if not bindmount.parse_explicit_mount(mount):
             # Path is implicit: check that this path is valid
             # (we don't try to validate explicit mounts)
             mount = os.path.abspath(os.path.expanduser(mount))
             if not os.path.exists(mount):
                 raise exceptions.TutorError(f"Path {mount} does not exist on the host")
+            implicit_mounts.append(mount)
         new_mounts.append(mount)
         fmt.echo_info(f"Adding bind-mount: {mount}")
 
     context.invoke(config_save, append_vars=[("MOUNTS", mount) for mount in new_mounts])
 
+    if populate:
+        context.invoke(mounts_populate, mounts=implicit_mounts)
+
 
 @click.command(name="remove")
 @click.argument("mounts", metavar="mount", type=MountParamType(), nargs=-1)
@@ -133,6 +147,91 @@ def mounts_remove(context: click.Context, mounts: list[str]) -> None:
     )
 
 
+@click.command(name="populate", help="TODO document command")
+@click.argument("mounts", metavar="mount", type=str, nargs=-1)
+@click.pass_obj
+def mounts_populate(context, mounts: str) -> None:
+    """
+    TODO document command
+    """
+    container_name = "tutor_mounts_populate_temp"  # TODO: improve name?
+    config = tutor_config.load(context.root)
+    paths_to_copy_by_image: dict[str, tuple[str, str]] = defaultdict(list)
+
+    if not mounts:
+        mounts = bindmount.get_mounts(config)
+
+    for mount in mounts:
+        mount_items: list[tuple[str, str, str]] = bindmount.parse_mount(mount)
+        if not mount_items:
+            raise exceptions.TutorError(f"No mount for {mount}")
+        _service, mount_host_path, _container_path = mount_items[
+            0
+        ]  # [0] is arbitrary, as all host_paths should be equal
+        mount_expanded = os.path.abspath(os.path.expanduser(mount))
+        mount_name = os.path.basename(mount_expanded)
+        for (
+            image,
+            path_on_image,
+            path_in_host_mount,
+        ) in hooks.Filters.COMPOSE_MOUNT_POPULATORS.iterate(mount_name):
+            paths_to_copy_by_image[image].append(
+                (path_on_image, f"{mount_expanded}/{path_in_host_mount}")
+            )
+    for image_name, paths_to_copy in paths_to_copy_by_image.items():
+        image_tag = _get_image_tag(config, image_name)
+        execute_shell("docker", "rm", "-f", container_name)
+        execute_shell("docker", "create", "--name", container_name, image_tag)
+        for path_on_image, path_on_host in paths_to_copy:
+            fmt.echo_info(f"Populating {path_on_host} from {image_name}")
+            execute_shell("rm", "-rf", path_on_host)
+            execute_shell(
+                "docker", "cp", f"{container_name}:{path_on_image}", path_on_host
+            )
+        execute_shell("docker", "rm", "-f", container_name)
+
+
+def _get_image_tag(config: Config, image_name: str) -> str:
+    """
+    Translate from a Tutor/plugin-defined image name to a specific Docker image tag.
+
+    Searches for image_name in IMAGES_PULL then IMAGES_BUILD.
+    Raises ImageNotFoundError if no match.
+    """
+    try:
+        return next(
+            find_remote_image_tags(config, hooks.Filters.IMAGES_PULL, image_name)
+        )
+    except ImageNotFoundError:
+        _name, _path, tag, _args = next(find_images_to_build(config, image_name))
+        return tag
+
+
+@hooks.Filters.COMPOSE_MOUNT_POPULATORS.add()
+def _populate_edx_platform_generated_dirs(
+    populators: list[tuple[str, str, str]], mount_name: str
+) -> list[str]:
+    """
+    TODO write docstring
+    """
+    if mount_name == "edx-platform":
+        populators += [
+            ("openedx-dev", f"/openedx/edx-platform/{generated_dir}", generated_dir)
+            for generated_dir in [
+                "Open_edX.egg-info",
+                "node_modules",
+                "lms/static/css",
+                "lms/static/certificates/css",
+                "cms/static/css",
+                "common/static/bundles",
+                "common/static/common/js/vendor",
+                "common/static/common/css/vendor",
+            ]
+        ]
+    return populators
+
+
 mounts_command.add_command(mounts_list)
 mounts_command.add_command(mounts_add)
 mounts_command.add_command(mounts_remove)
+mounts_command.add_command(mounts_populate)

tutor/hooks/catalog.py

@@ -222,6 +222,17 @@ def your_filter_callback(some_data):
     #:   conditionally add mounts.
     COMPOSE_MOUNTS: Filter[list[tuple[str, str]], [str]] = Filter()
 
+    #: TODO describe
+    #:
+    #: TODO show example
+    #:
+    #: :parameter list[tuple[str, str, str]] populators: each item is a
+    #:   ``(image_name, path_on_image, path_in_host_mount)`` tuple. TODO finish describing.
+    #: :parameter str name: basename of the host-mounted folder. In the example above,
+    #:   this is "edx-platform". When implementing this filter you should check this name to
+    #:   conditionally add populators for this folder.
+    COMPOSE_MOUNT_POPULATORS: Filter[list[tuple[str, str, str], [str]]] = Filter()
+
     #: Declare new default configuration settings that don't necessarily have to be saved in the user
     #: ``config.yml`` file. Default settings may be overridden with ``tutor config save --set=...``, in which
     #: case they will automatically be added to ``config.yml``.