Setup provisioner devenv (#212)

* Setup provisioner devenv * add additional tests to rules_task * Enable running provision deploy against docker
vgijssel · Apr 26, 2023 · ec322dc · ec322dc
1 parent cf19677
commit ec322dc
Show file tree

Hide file tree

Showing 12 changed files with 145 additions and 44 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -25,7 +25,7 @@ common --enable_bzlmod=true
 build --@io_bazel_rules_docker//transitions:enable=false
 
 # Enable builds without the bytes https://github.com/bazelbuild/bazel/issues/6862
-build --remote_download_minimal
+build:remote --remote_download_minimal
 
 # From https://www.buildbuddy.io/docs/cloud
 # Use BuildBuddy

diff --git a/provisioner/BUILD.bazel b/provisioner/BUILD.bazel
@@ -1,4 +1,6 @@
 load("//tools/pyinfra:defs.bzl", "pyinfra_run")
+load("@rules_task//:defs.bzl", "cmd", "task")
+load("//tools/docker:docker.bzl", "docker_load_and_run")
 
 pyinfra_run(
     name = "provision",
@@ -17,3 +19,31 @@ pyinfra_run(
     deploy = "deploy.py",
     inventory = "inventory.py",
 )
+
+docker_load_and_run(
+    name = "dev_image_run",
+    command = "/sbin/init",
+    docker_args = [
+        "--rm",
+        "--name provisioner_dev",
+        "--detach",
+        "--tmpfs /run",
+        "--tmpfs /run/lock",
+        "--tmpfs /tmp",
+        "--privileged",
+        "-v /lib/modules:/lib/modules:ro",
+    ],
+    image = "//tools/ubuntu:ubuntu_snap_base_image",
+)
+
+task(
+    name = "dev",
+    cmds = [
+        "CONTAINER_ID=$($run_dev_image)",
+        {"defer": "docker rm -f $CONTAINER_ID"},
+        "docker logs -f $CONTAINER_ID",
+    ],
+    env = {
+        "run_dev_image": cmd.executable(":dev_image_run"),
+    },
+)
diff --git a/provisioner/deploys/microk8s/tasks/install_microk8s.py b/provisioner/deploys/microk8s/tasks/install_microk8s.py
@@ -7,14 +7,15 @@
 # From https://microk8s.io/docs/getting-started
 @deploy("Install Microk8s")
 def install_microk8s():
-    # From https://microk8s.io/docs/install-raspberry-pi
-    apt.packages(
-        name="Ensure all kernel modules are available",
-        packages=["linux-modules-extra-raspi"],
-        update=True,
-        present=True,
-        _sudo=True,
-    )
+    if not host.data.get("inside_docker"):
+        # From https://microk8s.io/docs/install-raspberry-pi
+        apt.packages(
+            name="Ensure all kernel modules are available",
+            packages=["linux-modules-extra-raspi"],
+            update=True,
+            present=True,
+            _sudo=True,
+        )
 
     # From https://microk8s.io/docs/install-raspberry-pi
     config_file = files.put(
@@ -25,7 +26,7 @@ def install_microk8s():
         _sudo=True,
     )
 
-    if config_file.changed:
+    if config_file.changed and not host.data.get("inside_docker"):
         server.reboot(
             name="Reboot the server and wait to reconnect",
             delay=60,

diff --git a/provisioner/deploys/network/tasks/install_network.py b/provisioner/deploys/network/tasks/install_network.py
@@ -1,9 +1,17 @@
 from pyinfra.api.deploy import deploy
-from pyinfra.operations import files, server
+from pyinfra.operations import files, server, apt
+from pyinfra import host
 
 
 @deploy("Install Network")
 def install_network():
+    apt.packages(
+        name="Install netplan",
+        packages=["netplan.io"],
+        latest=True,
+        _sudo=True,
+    )
+
     config_file = files.put(
         name="Copy netplan config",
         src="provisioner/deploys/network/files/99_config.yaml",
@@ -13,21 +21,30 @@ def install_network():
         _sudo=True,
     )
 
+    apt.packages(
+        name="Install Uncomplicated Firewall (ufw)",
+        packages=["ufw"],
+        latest=True,
+        _sudo=True,
+    )
+
     if config_file.changed:
         server.shell(
             name="Generate netplan",
             commands=["netplan generate"],
             _sudo=True,
         )
 
-        server.shell(
-            name="Apply the netplan configuration",
-            commands=["netplan apply"],
+        if not host.data.get("inside_docker"):
+            server.shell(
+                name="Apply the netplan configuration",
+                commands=["netplan apply"],
+                _sudo=True,
+            )
+
+    if not host.data.get("inside_docker"):
+        server.hostname(
+            name="Set hostname",
+            hostname="provisioner",
             _sudo=True,
         )
-
-    server.hostname(
-        name="Set hostname",
-        hostname="provisioner",
-        _sudo=True,
-    )
diff --git a/provisioner/inventory.py b/provisioner/inventory.py
@@ -1,3 +1,11 @@
-hosts = [
-    ("@ssh/192.168.1.31", {"ssh_user": "ubuntu"}),
-]
+import os
+
+if os.environ.get("SETUP_ENV", "dev") == "prod":
+    hosts = [
+        ("@ssh/192.168.1.31", {"ssh_user": "ubuntu"}),
+    ]
+else:
+    container_id = "provisioner_dev"
+    hosts = [
+        (f"@docker/{container_id}", {"inside_docker": True}),
+    ]
diff --git a/rules/rules_task/defs.bzl b/rules/rules_task/defs.bzl
@@ -70,13 +70,19 @@ def _serialize_env(context, node):
 
     for key, value in node["env"].items():
         env_value = _visit_method(context, value)(context, value)
-        env_string += "export {}={}\n".format(key, env_value)
+
+        # escape single quoted characters for Bash
+        env_value = env_value.replace("'", "\\\'")
+        env_string += "export {}=$'{}'\n".format(key, env_value)
 
     return env_string
 
+def _jinja_rlocation(rlocation):
+    return '{{ rlocation_to_path("%s") }}' % rlocation
+
 def _file_label_to_jinja_path(ctx, label):
     rlocation = ctx.expand_location("$(rlocationpath {})".format(label), ctx.attr.data)
-    rlocation = "{{ rlocation_to_path('%s') }}" % rlocation
+    rlocation = _jinja_rlocation(rlocation)
     return rlocation
 
 def _files_label_to_jinja_path(ctx, label):
@@ -85,7 +91,7 @@ def _files_label_to_jinja_path(ctx, label):
     result = []
 
     for rlocation in rlocations:
-        result.append("{{ rlocation_to_path('%s') }}" % rlocation)
+        result.append(_jinja_rlocation(rlocation))
 
     return " ".join(result)
 
@@ -105,8 +111,7 @@ def _executable_label_to_jinja_path(ctx, label):
     target = target_matches_label[0]
     executable = target[DefaultInfo].files_to_run.executable
     rlocation = to_rlocation_path(ctx, executable)
-    rlocation = "{{ rlocation_to_path('%s') }}" % rlocation
-    return rlocation
+    return _jinja_rlocation(rlocation)
 
 def _python_entry_point(entry_point, args):
     # Translate
@@ -197,7 +202,7 @@ _serializer = {
 
 _data_collector = {
     "visit_root": lambda context, node: _compact(_flatten(_visit_root(context, node))),
-    "visit_env": lambda context, node: _visit_env(context, node),
+    "visit_env": lambda context, node: _compact(_flatten(_visit_env(context, node))),
     "visit_defer": lambda context, node: _visit_defer(context, node),
     "visit_shell": lambda context, node: _visit_shell(context, node),
     "visit_string": lambda context, node: None,

diff --git a/rules/rules_task/runner.py b/rules/rules_task/runner.py
@@ -83,6 +83,9 @@ def main() -> None:
         bash_cmd += cmd + "\n"
 
     bash_cmd = jinja_render_string(bash_cmd)
+
+    # print(bash_cmd)
+
     cmd_env = os.environ.copy()
     cmd_env["CLI_ARGS"] = cli_args
 

diff --git a/rules/rules_task/tests/BUILD.bazel b/rules/rules_task/tests/BUILD.bazel
@@ -137,11 +137,13 @@ task(
         "echo $FOO",
         "$hello",
         "$python",
+        "$command",
     ],
     env = {
-        "FOO": "BAR",
+        "FOO": "BAR's value",
         "hello": cmd.executable(":hello"),
         "python": cmd.executable(":python"),
+        "command": cmd.shell("echo", "some", "inline", "shell"),
     },
 )
 

diff --git a/rules/rules_task/tests/test.py b/rules/rules_task/tests/test.py
@@ -95,7 +95,10 @@ def test_python_entry_point():
 def test_env():
     result = _run_task("env")
     assert result.returncode == 0
-    assert result.stdout.strip() == b"BAR\nHello, world!\nsomevalue"
+    assert (
+        result.stdout.strip()
+        == b"BAR's value\nHello, world!\nsomevalue\nsome inline shell"
+    )
 
 
 def test_defer():

diff --git a/tools/docker/docker.bzl b/tools/docker/docker.bzl
@@ -8,8 +8,8 @@ def docker_load_and_run(name, image, command, docker_args = []):
     """
     Loads a docker image and runs it.
     """
-    image_label = image
-    image_sha_label = "{}.json.sha256".format(image_label)
+    image_label = "{}.tar".format(image)
+    image_sha_label = "{}.json.sha256".format(image)
 
     task(
         name = name,
@@ -22,25 +22,19 @@ def docker_load_and_run(name, image, command, docker_args = []):
         DOCKER_DIGEST=$(cat $DOCKER_DIGEST_FILE)
         DOCKER_LOAD_FILE=$image_label
 
-        # if CLI_ARGS is set, then add interactive flag
-        if [[ ! -z "$CLI_ARGS" ]]; then
-            DOCKER_INTERACTIVE_ARGS="-it"
-        else
-            DOCKER_INTERACTIVE_ARGS=""
-        fi
 
         if ! docker image inspect $DOCKER_DIGEST > /dev/null 2>&1 ; then
-            $DOCKER_LOAD_FILE
+            docker load --input $DOCKER_LOAD_FILE >&2
         else
-            echo "Image already exists"
+            echo Image already exists >&2
         fi
 
-        docker run --rm $DOCKER_INTERACTIVE_ARGS $docker_args $DOCKER_DIGEST $ARGS
+        docker run $docker_args $DOCKER_DIGEST $ARGS
         """,
         ],
         env = {
             "command": command,
-            "docker_args": "'" + " ".join(docker_args) + "'",
+            "docker_args": " ".join(docker_args),
             "image_label": cmd.file(image_label),
             "image_sha_label": cmd.file(image_sha_label),
         },

diff --git a/tools/pyinfra/defs.bzl b/tools/pyinfra/defs.bzl
@@ -29,7 +29,7 @@ def pyinfra_run(name, deploy, inventory, env = {}, srcs = [], deps = [], args =
             "pyinfra": cmd.executable(python_binary),
             "deploy_file": cmd.file(deploy),
             "inventory_file": cmd.file(inventory),
-            "default_args": "'" + " ".join(args) + "'",
+            "default_args": " ".join(args),
         } | env,
         data = data,
         deps = deps,

diff --git a/tools/ubuntu/BUILD.bazel b/tools/ubuntu/BUILD.bazel
@@ -1,4 +1,5 @@
 load("@io_bazel_rules_docker//container:container.bzl", "container_image")
+load("@io_bazel_rules_docker//docker/util:run.bzl", "container_run_and_commit_layer")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -10,4 +11,41 @@ ubuntu_base_image = select({
 container_image(
     name = "ubuntu_base_image",
     base = ubuntu_base_image,
+    env = {
+        "LANG": "C.UTF-8",
+        "LC_ALL": "C.UTF-8",
+        "container": "docker",
+    },
+)
+
+container_run_and_commit_layer(
+    name = "setup_snap",
+    commands = [
+        "apt-get update",
+        "apt-get install -y snapd squashfuse fuse sudo",
+        "systemctl enable snapd",
+        "useradd -m ubuntu -s /bin/bash",
+        "adduser ubuntu sudo",
+        "echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers",
+    ],
+    env = {
+        "DEBIAN_FRONTEND": "noninteractive",
+    },
+    exec_properties = {
+        "workload-isolation-type": "firecracker",
+        "init-dockerd": "true",
+        "recycle-runner": "true",
+    },
+    image = ":ubuntu_base_image.tar",
+)
+
+container_image(
+    name = "ubuntu_snap_base_image",
+    base = ":ubuntu_base_image.tar",
+    env = {
+        "PATH": "/snap/bin:$$PATH",
+    },
+    layers = [
+        ":setup_snap",
+    ],
 )