From ec322dc54c2bd6edf95b26040cd4908e918e79cd Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Wed, 26 Apr 2023 13:53:37 +0200 Subject: [PATCH] Setup provisioner devenv (#212) * Setup provisioner devenv * add additional tests to rules_task * Enable running provision deploy against docker --- .bazelrc | 2 +- provisioner/BUILD.bazel | 30 +++++++++++++++ .../microk8s/tasks/install_microk8s.py | 19 +++++----- .../deploys/network/tasks/install_network.py | 37 +++++++++++++----- provisioner/inventory.py | 14 +++++-- rules/rules_task/defs.bzl | 17 ++++++--- rules/rules_task/runner.py | 3 ++ rules/rules_task/tests/BUILD.bazel | 4 +- rules/rules_task/tests/test.py | 5 ++- tools/docker/docker.bzl | 18 +++------ tools/pyinfra/defs.bzl | 2 +- tools/ubuntu/BUILD.bazel | 38 +++++++++++++++++++ 12 files changed, 145 insertions(+), 44 deletions(-) diff --git a/.bazelrc b/.bazelrc index 9f951b373..f89b357f1 100644 --- a/.bazelrc +++ b/.bazelrc @@ -25,7 +25,7 @@ common --enable_bzlmod=true build --@io_bazel_rules_docker//transitions:enable=false # Enable builds without the bytes https://github.com/bazelbuild/bazel/issues/6862 -build --remote_download_minimal +build:remote --remote_download_minimal # From https://www.buildbuddy.io/docs/cloud # Use BuildBuddy diff --git a/provisioner/BUILD.bazel b/provisioner/BUILD.bazel index 7d2aa04a4..9e4723af2 100644 --- a/provisioner/BUILD.bazel +++ b/provisioner/BUILD.bazel @@ -1,4 +1,6 @@ load("//tools/pyinfra:defs.bzl", "pyinfra_run") +load("@rules_task//:defs.bzl", "cmd", "task") +load("//tools/docker:docker.bzl", "docker_load_and_run") pyinfra_run( name = "provision", @@ -17,3 +19,31 @@ pyinfra_run( deploy = "deploy.py", inventory = "inventory.py", ) + +docker_load_and_run( + name = "dev_image_run", + command = "/sbin/init", + docker_args = [ + "--rm", + "--name provisioner_dev", + "--detach", + "--tmpfs /run", + "--tmpfs /run/lock", + "--tmpfs /tmp", + "--privileged", + "-v /lib/modules:/lib/modules:ro", + ], + image = "//tools/ubuntu:ubuntu_snap_base_image", +) + +task( + name = "dev", + cmds = [ + "CONTAINER_ID=$($run_dev_image)", + {"defer": "docker rm -f $CONTAINER_ID"}, + "docker logs -f $CONTAINER_ID", + ], + env = { + "run_dev_image": cmd.executable(":dev_image_run"), + }, +) diff --git a/provisioner/deploys/microk8s/tasks/install_microk8s.py b/provisioner/deploys/microk8s/tasks/install_microk8s.py index 252b0908b..76acd2298 100644 --- a/provisioner/deploys/microk8s/tasks/install_microk8s.py +++ b/provisioner/deploys/microk8s/tasks/install_microk8s.py @@ -7,14 +7,15 @@ # From https://microk8s.io/docs/getting-started @deploy("Install Microk8s") def install_microk8s(): - # From https://microk8s.io/docs/install-raspberry-pi - apt.packages( - name="Ensure all kernel modules are available", - packages=["linux-modules-extra-raspi"], - update=True, - present=True, - _sudo=True, - ) + if not host.data.get("inside_docker"): + # From https://microk8s.io/docs/install-raspberry-pi + apt.packages( + name="Ensure all kernel modules are available", + packages=["linux-modules-extra-raspi"], + update=True, + present=True, + _sudo=True, + ) # From https://microk8s.io/docs/install-raspberry-pi config_file = files.put( @@ -25,7 +26,7 @@ def install_microk8s(): _sudo=True, ) - if config_file.changed: + if config_file.changed and not host.data.get("inside_docker"): server.reboot( name="Reboot the server and wait to reconnect", delay=60, diff --git a/provisioner/deploys/network/tasks/install_network.py b/provisioner/deploys/network/tasks/install_network.py index a8aca939d..c7126a172 100644 --- a/provisioner/deploys/network/tasks/install_network.py +++ b/provisioner/deploys/network/tasks/install_network.py @@ -1,9 +1,17 @@ from pyinfra.api.deploy import deploy -from pyinfra.operations import files, server +from pyinfra.operations import files, server, apt +from pyinfra import host @deploy("Install Network") def install_network(): + apt.packages( + name="Install netplan", + packages=["netplan.io"], + latest=True, + _sudo=True, + ) + config_file = files.put( name="Copy netplan config", src="provisioner/deploys/network/files/99_config.yaml", @@ -13,6 +21,13 @@ def install_network(): _sudo=True, ) + apt.packages( + name="Install Uncomplicated Firewall (ufw)", + packages=["ufw"], + latest=True, + _sudo=True, + ) + if config_file.changed: server.shell( name="Generate netplan", @@ -20,14 +35,16 @@ def install_network(): _sudo=True, ) - server.shell( - name="Apply the netplan configuration", - commands=["netplan apply"], + if not host.data.get("inside_docker"): + server.shell( + name="Apply the netplan configuration", + commands=["netplan apply"], + _sudo=True, + ) + + if not host.data.get("inside_docker"): + server.hostname( + name="Set hostname", + hostname="provisioner", _sudo=True, ) - - server.hostname( - name="Set hostname", - hostname="provisioner", - _sudo=True, - ) diff --git a/provisioner/inventory.py b/provisioner/inventory.py index eb4513c3f..8787af781 100644 --- a/provisioner/inventory.py +++ b/provisioner/inventory.py @@ -1,3 +1,11 @@ -hosts = [ - ("@ssh/192.168.1.31", {"ssh_user": "ubuntu"}), -] +import os + +if os.environ.get("SETUP_ENV", "dev") == "prod": + hosts = [ + ("@ssh/192.168.1.31", {"ssh_user": "ubuntu"}), + ] +else: + container_id = "provisioner_dev" + hosts = [ + (f"@docker/{container_id}", {"inside_docker": True}), + ] diff --git a/rules/rules_task/defs.bzl b/rules/rules_task/defs.bzl index 57d69d0b0..842fbe593 100644 --- a/rules/rules_task/defs.bzl +++ b/rules/rules_task/defs.bzl @@ -70,13 +70,19 @@ def _serialize_env(context, node): for key, value in node["env"].items(): env_value = _visit_method(context, value)(context, value) - env_string += "export {}={}\n".format(key, env_value) + + # escape single quoted characters for Bash + env_value = env_value.replace("'", "\\\'") + env_string += "export {}=$'{}'\n".format(key, env_value) return env_string +def _jinja_rlocation(rlocation): + return '{{ rlocation_to_path("%s") }}' % rlocation + def _file_label_to_jinja_path(ctx, label): rlocation = ctx.expand_location("$(rlocationpath {})".format(label), ctx.attr.data) - rlocation = "{{ rlocation_to_path('%s') }}" % rlocation + rlocation = _jinja_rlocation(rlocation) return rlocation def _files_label_to_jinja_path(ctx, label): @@ -85,7 +91,7 @@ def _files_label_to_jinja_path(ctx, label): result = [] for rlocation in rlocations: - result.append("{{ rlocation_to_path('%s') }}" % rlocation) + result.append(_jinja_rlocation(rlocation)) return " ".join(result) @@ -105,8 +111,7 @@ def _executable_label_to_jinja_path(ctx, label): target = target_matches_label[0] executable = target[DefaultInfo].files_to_run.executable rlocation = to_rlocation_path(ctx, executable) - rlocation = "{{ rlocation_to_path('%s') }}" % rlocation - return rlocation + return _jinja_rlocation(rlocation) def _python_entry_point(entry_point, args): # Translate @@ -197,7 +202,7 @@ _serializer = { _data_collector = { "visit_root": lambda context, node: _compact(_flatten(_visit_root(context, node))), - "visit_env": lambda context, node: _visit_env(context, node), + "visit_env": lambda context, node: _compact(_flatten(_visit_env(context, node))), "visit_defer": lambda context, node: _visit_defer(context, node), "visit_shell": lambda context, node: _visit_shell(context, node), "visit_string": lambda context, node: None, diff --git a/rules/rules_task/runner.py b/rules/rules_task/runner.py index ca35d56ae..3f0907c5b 100644 --- a/rules/rules_task/runner.py +++ b/rules/rules_task/runner.py @@ -83,6 +83,9 @@ def main() -> None: bash_cmd += cmd + "\n" bash_cmd = jinja_render_string(bash_cmd) + + # print(bash_cmd) + cmd_env = os.environ.copy() cmd_env["CLI_ARGS"] = cli_args diff --git a/rules/rules_task/tests/BUILD.bazel b/rules/rules_task/tests/BUILD.bazel index 5ed0c3e01..ba9e71ce3 100644 --- a/rules/rules_task/tests/BUILD.bazel +++ b/rules/rules_task/tests/BUILD.bazel @@ -137,11 +137,13 @@ task( "echo $FOO", "$hello", "$python", + "$command", ], env = { - "FOO": "BAR", + "FOO": "BAR's value", "hello": cmd.executable(":hello"), "python": cmd.executable(":python"), + "command": cmd.shell("echo", "some", "inline", "shell"), }, ) diff --git a/rules/rules_task/tests/test.py b/rules/rules_task/tests/test.py index 367c07aa5..fe3ef5fa9 100644 --- a/rules/rules_task/tests/test.py +++ b/rules/rules_task/tests/test.py @@ -95,7 +95,10 @@ def test_python_entry_point(): def test_env(): result = _run_task("env") assert result.returncode == 0 - assert result.stdout.strip() == b"BAR\nHello, world!\nsomevalue" + assert ( + result.stdout.strip() + == b"BAR's value\nHello, world!\nsomevalue\nsome inline shell" + ) def test_defer(): diff --git a/tools/docker/docker.bzl b/tools/docker/docker.bzl index beca13ab3..1bca786ae 100644 --- a/tools/docker/docker.bzl +++ b/tools/docker/docker.bzl @@ -8,8 +8,8 @@ def docker_load_and_run(name, image, command, docker_args = []): """ Loads a docker image and runs it. """ - image_label = image - image_sha_label = "{}.json.sha256".format(image_label) + image_label = "{}.tar".format(image) + image_sha_label = "{}.json.sha256".format(image) task( name = name, @@ -22,25 +22,19 @@ def docker_load_and_run(name, image, command, docker_args = []): DOCKER_DIGEST=$(cat $DOCKER_DIGEST_FILE) DOCKER_LOAD_FILE=$image_label - # if CLI_ARGS is set, then add interactive flag - if [[ ! -z "$CLI_ARGS" ]]; then - DOCKER_INTERACTIVE_ARGS="-it" - else - DOCKER_INTERACTIVE_ARGS="" - fi if ! docker image inspect $DOCKER_DIGEST > /dev/null 2>&1 ; then - $DOCKER_LOAD_FILE + docker load --input $DOCKER_LOAD_FILE >&2 else - echo "Image already exists" + echo Image already exists >&2 fi - docker run --rm $DOCKER_INTERACTIVE_ARGS $docker_args $DOCKER_DIGEST $ARGS + docker run $docker_args $DOCKER_DIGEST $ARGS """, ], env = { "command": command, - "docker_args": "'" + " ".join(docker_args) + "'", + "docker_args": " ".join(docker_args), "image_label": cmd.file(image_label), "image_sha_label": cmd.file(image_sha_label), }, diff --git a/tools/pyinfra/defs.bzl b/tools/pyinfra/defs.bzl index ed2092e25..bda21985e 100644 --- a/tools/pyinfra/defs.bzl +++ b/tools/pyinfra/defs.bzl @@ -29,7 +29,7 @@ def pyinfra_run(name, deploy, inventory, env = {}, srcs = [], deps = [], args = "pyinfra": cmd.executable(python_binary), "deploy_file": cmd.file(deploy), "inventory_file": cmd.file(inventory), - "default_args": "'" + " ".join(args) + "'", + "default_args": " ".join(args), } | env, data = data, deps = deps, diff --git a/tools/ubuntu/BUILD.bazel b/tools/ubuntu/BUILD.bazel index 3c70ce0c1..1197ddce8 100644 --- a/tools/ubuntu/BUILD.bazel +++ b/tools/ubuntu/BUILD.bazel @@ -1,4 +1,5 @@ load("@io_bazel_rules_docker//container:container.bzl", "container_image") +load("@io_bazel_rules_docker//docker/util:run.bzl", "container_run_and_commit_layer") package(default_visibility = ["//visibility:public"]) @@ -10,4 +11,41 @@ ubuntu_base_image = select({ container_image( name = "ubuntu_base_image", base = ubuntu_base_image, + env = { + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "container": "docker", + }, +) + +container_run_and_commit_layer( + name = "setup_snap", + commands = [ + "apt-get update", + "apt-get install -y snapd squashfuse fuse sudo", + "systemctl enable snapd", + "useradd -m ubuntu -s /bin/bash", + "adduser ubuntu sudo", + "echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers", + ], + env = { + "DEBIAN_FRONTEND": "noninteractive", + }, + exec_properties = { + "workload-isolation-type": "firecracker", + "init-dockerd": "true", + "recycle-runner": "true", + }, + image = ":ubuntu_base_image.tar", +) + +container_image( + name = "ubuntu_snap_base_image", + base = ":ubuntu_base_image.tar", + env = { + "PATH": "/snap/bin:$$PATH", + }, + layers = [ + ":setup_snap", + ], )