diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod index fb5732563..6f415d9cb 100644 --- a/WORKSPACE.bzlmod +++ b/WORKSPACE.bzlmod @@ -187,3 +187,20 @@ http_archive( sha256 = "6d357e2e2a54a713e79fc0ce9068470fb202ac4fb0e27bafef0fe5a377ada10f", url = "https://cdn.teleport.dev/teleport-v13.0.0-linux-amd64-bin.tar.gz", ) + +# ------------------------------------ 1password client ------------------------------------ # + +# From https://app-updates.agilebits.com/product_history/CLI2 +http_archive( + name = "onepassword_arm64", + build_file = "//tools/onepassword:BUILD.repositories.bazel.tpl", + sha256 = "b93a8e0dc42c0979bb13047ac4412bd73092be57bb84ad223eeca295151159fa", + url = "https://cache.agilebits.com/dist/1P/op2/pkg/v2.18.0/op_linux_arm64_v2.18.0.zip", +) + +http_archive( + name = "onepassword_amd64", + build_file = "//tools/onepassword:BUILD.repositories.bazel.tpl", + sha256 = "2baf610b476727f24c62cc843419f55b157e1a05521a698c1c8b4ed676a766aa", + url = "https://cache.agilebits.com/dist/1P/op2/pkg/v2.18.0/op_linux_amd64_v2.18.0.zip", +) diff --git a/provisioner/BUILD.bazel b/provisioner/BUILD.bazel index a40186b93..a6fe943ea 100644 --- a/provisioner/BUILD.bazel +++ b/provisioner/BUILD.bazel @@ -25,7 +25,8 @@ pyinfra_run( data = [ "deploys/microk8s/files/cmdline.txt", "deploys/monitoring/files/json_file_output.conf", - "deploys/monitoring/files/telegraf.conf", + "deploys/monitoring/files/logzio_output.conf.j2", + "deploys/monitoring/files/telegraf.conf.j2", "deploys/network/files/99_config.yaml", "deploys/teleport/files/teleport.yaml.j2", "deploys/teleport/files/teleport_health_check.conf", @@ -33,6 +34,7 @@ pyinfra_run( deploy = "deploy.py", env = { "TELEPORT_TSH_BINARY": cmd.executable("//tools/teleport:tsh"), + "OP_BINARY": cmd.executable("//tools/onepassword:op"), }, inventory = "inventory.py", ) @@ -153,5 +155,6 @@ task_test( "workload-isolation-type": "firecracker", "init-dockerd": "true", "recycle-runner": "true", + "include-secrets": "true", }, ) diff --git a/provisioner/deploys/microk8s/tasks/install_microk8s.py b/provisioner/deploys/microk8s/tasks/install_microk8s.py index 1b6b593d2..755ef9302 100644 --- a/provisioner/deploys/microk8s/tasks/install_microk8s.py +++ b/provisioner/deploys/microk8s/tasks/install_microk8s.py @@ -16,6 +16,7 @@ def install_microk8s(): packages=["linux-modules-extra-raspi"], update=True, present=True, + cache_time=24 * 60 * 60, _sudo=True, ) diff --git a/provisioner/deploys/monitoring/files/logzio_output.conf.j2 b/provisioner/deploys/monitoring/files/logzio_output.conf.j2 new file mode 100644 index 000000000..2f1d25905 --- /dev/null +++ b/provisioner/deploys/monitoring/files/logzio_output.conf.j2 @@ -0,0 +1,8 @@ +[[outputs.http]] + url = "https://listener.logz.io:8053" + data_format = "prometheusremotewrite" + [outputs.http.headers] + Content-Type = "application/x-protobuf" + Content-Encoding = "snappy" + X-Prometheus-Remote-Write-Version = "0.1.0" + Authorization = "Bearer {{ logzio_metrics_token }}" \ No newline at end of file diff --git a/provisioner/deploys/monitoring/files/telegraf.conf b/provisioner/deploys/monitoring/files/telegraf.conf.j2 similarity index 99% rename from provisioner/deploys/monitoring/files/telegraf.conf rename to provisioner/deploys/monitoring/files/telegraf.conf.j2 index 0407ac7e9..16ee281df 100644 --- a/provisioner/deploys/monitoring/files/telegraf.conf +++ b/provisioner/deploys/monitoring/files/telegraf.conf.j2 @@ -20,6 +20,7 @@ # rack = "1a" ## Environment variables can be used as tags, and throughout the config file # user = "$USER" + env = "{{ setup_env }}" # Configuration for telegraf agent diff --git a/provisioner/deploys/monitoring/tasks/install_monitoring.py b/provisioner/deploys/monitoring/tasks/install_monitoring.py index ca6d60f49..79262489c 100644 --- a/provisioner/deploys/monitoring/tasks/install_monitoring.py +++ b/provisioner/deploys/monitoring/tasks/install_monitoring.py @@ -1,6 +1,7 @@ from pyinfra.api.deploy import deploy from pyinfra.operations import files, server, apt, systemd from pyinfra import host +from provisioner.utils import one_password_item @deploy("Install Monitoring") @@ -11,15 +12,30 @@ def install_monitoring(): _sudo=True, ) - files.put( + files.template( name="Copy telegraf config", - src="provisioner/deploys/monitoring/files/telegraf.conf", + src="provisioner/deploys/monitoring/files/telegraf.conf.j2", dest="/etc/telegraf/telegraf.conf", create_remote_dir=True, _sudo=True, user="root", group="root", mode="0644", # rw r r for root + setup_env=host.data.setup_env, + ) + + logzio_metrics_token = one_password_item("logzio_metrics_token")["password"] + + files.template( + name="Copy telegraf logz.io output config", + src="provisioner/deploys/monitoring/files/logzio_output.conf.j2", + dest="/etc/telegraf/telegraf.d/logzio_output.conf", + create_remote_dir=True, + _sudo=True, + user="root", + group="root", + mode="0644", + logzio_metrics_token=logzio_metrics_token, ) files.put( diff --git a/provisioner/group_data/dev.py b/provisioner/group_data/dev.py index 723c0e96a..ef4f712e8 100644 --- a/provisioner/group_data/dev.py +++ b/provisioner/group_data/dev.py @@ -1,5 +1,7 @@ +setup_env = "dev" inside_docker = True legacy_ip_tables = True teleport_public_addr = "127.0.0.1:10443" teleport_acme_email = "" teleport_acme_enabled = "no" +onepassword_vault_id = "vgijssel-dev" diff --git a/provisioner/group_data/prod.py b/provisioner/group_data/prod.py index a6a4e2973..678fac9be 100644 --- a/provisioner/group_data/prod.py +++ b/provisioner/group_data/prod.py @@ -1,6 +1,7 @@ +setup_env = "prod" inside_docker = False legacy_ip_tables = False - teleport_public_addr = "tele.vgijssel.nl:443" teleport_acme_email = "haves_borzoi_0o@icloud.com" teleport_acme_enabled = "yes" +onepassword_vault_id = "vgijssel-prod" diff --git a/provisioner/group_data/test.py b/provisioner/group_data/test.py index 723c0e96a..200abec59 100644 --- a/provisioner/group_data/test.py +++ b/provisioner/group_data/test.py @@ -1,5 +1,7 @@ +setup_env = "test" inside_docker = True legacy_ip_tables = True teleport_public_addr = "127.0.0.1:10443" teleport_acme_email = "" teleport_acme_enabled = "no" +onepassword_vault_id = "vgijssel-dev" diff --git a/provisioner/inventory.py b/provisioner/inventory.py index 15afb579d..24977db7f 100644 --- a/provisioner/inventory.py +++ b/provisioner/inventory.py @@ -1,9 +1,9 @@ import os import pkg_resources +from pathlib import Path import pyinfra.api.connectors import provisioner.connectors.teleport - from pyinfra.api.connectors import get_all_connectors import pyinfra.api.inventory @@ -21,6 +21,24 @@ def patched_get_all_connectors(): setup_env = os.environ.get("SETUP_ENV", "dev") + +def _get_onepassword_service_account_token(env_key, tmp_file): + if env_key in os.environ: + return os.environ[env_key] + + file = os.path.join( + os.environ.get("BUILD_WORKSPACE_DIRECTORY", ""), + "tmp", + tmp_file, + ) + + if os.path.exists(file): + return Path(file).read_text() + + else: + raise ValueError(f"Either set env variable '{env_key}' or create file '{file}'") + + if setup_env == "prod": if ( "TELEPORT_IDENTITY" in os.environ @@ -41,6 +59,10 @@ def patched_get_all_connectors(): "teleport_proxy": "tele.vgijssel.nl", "teleport_user": teleport_user, "teleport_identity": teleport_identity, + "onepassword_service_account_token": _get_onepassword_service_account_token( + "ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_PROD", + "1password-service-account-token-prod", + ), }, ), ] @@ -48,11 +70,27 @@ def patched_get_all_connectors(): elif setup_env == "test": container_id = os.environ["CONTAINER_ID"] test = [ - (f"@docker/{container_id}"), + ( + f"@docker/{container_id}", + { + "onepassword_service_account_token": _get_onepassword_service_account_token( + "ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_DEV", + "1password-service-account-token-dev", + ), + }, + ), ] else: container_id = "provisioner_dev" dev = [ - (f"@docker/{container_id}"), + ( + f"@docker/{container_id}", + { + "onepassword_service_account_token": _get_onepassword_service_account_token( + "ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_DEV", + "1password-service-account-token-dev", + ), + }, + ), ] diff --git a/provisioner/utils.py b/provisioner/utils.py index f94457c91..ebe14ac79 100644 --- a/provisioner/utils.py +++ b/provisioner/utils.py @@ -1,6 +1,9 @@ from pyinfra.operations import python from time import sleep from pyinfra import host +from pyinfra import local +import json +import os # Inspired by https://github.com/Fizzadar/pyinfra/blob/2.x/pyinfra/operations/server.py#LL51C12-L51C52 @@ -32,3 +35,39 @@ def wait_for_reconnect(name): name=name, function=_wait_for_reconnect, ) + + +def one_password_item( + item_title, + onepassword_vault_id=None, + onepassword_service_account_token=None, +): + if onepassword_vault_id is None: + onepassword_vault_id = host.data.onepassword_vault_id + + if onepassword_service_account_token is None: + onepassword_service_account_token = host.data.onepassword_service_account_token + + command = "{op_binary} item get '{item_title}' --vault='{onepassword_vault_id}' --format=json".format( + op_binary=os.environ["OP_BINARY"], + item_title=item_title, + onepassword_vault_id=onepassword_vault_id, + onepassword_service_account_token=onepassword_service_account_token, + ) + + try: + # NOTE: this environ hackery is so that if the command fail the secret is not printed to stdout/stderr. + os.environ["OP_SERVICE_ACCOUNT_TOKEN"] = onepassword_service_account_token + json_string = local.shell(command, print_input=False) + finally: + del os.environ["OP_SERVICE_ACCOUNT_TOKEN"] + print("An exception occurred") + + raw_data = json.loads(json_string) + + data = {} + + for field in raw_data["fields"]: + data[field["id"]] = field.get("value", None) + + return data diff --git a/tools/onepassword/BUILD.bazel b/tools/onepassword/BUILD.bazel new file mode 100644 index 000000000..3d77e1be3 --- /dev/null +++ b/tools/onepassword/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_task//:defs.bzl", "cmd", "task") + +package(default_visibility = ["//visibility:public"]) + +op_file = select({ + "@platforms//cpu:aarch64": ["@onepassword_arm64//:op"], + "@platforms//cpu:x86_64": ["@onepassword_amd64//:op"], +}) + +sh_binary( + name = "op_binary", + srcs = op_file, +) + +task( + name = "op", + cmds = [ + cmd.shell( + cmd.executable(":op_binary"), + "$CLI_ARGS", + ), + ], +) diff --git a/tools/onepassword/BUILD.repositories.bazel.tpl b/tools/onepassword/BUILD.repositories.bazel.tpl new file mode 100644 index 000000000..11c809fa3 --- /dev/null +++ b/tools/onepassword/BUILD.repositories.bazel.tpl @@ -0,0 +1,5 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "op" +]) \ No newline at end of file diff --git a/tools/ubuntu/BUILD.bazel b/tools/ubuntu/BUILD.bazel index 1197ddce8..4f3ce7fe3 100644 --- a/tools/ubuntu/BUILD.bazel +++ b/tools/ubuntu/BUILD.bazel @@ -22,7 +22,7 @@ container_run_and_commit_layer( name = "setup_snap", commands = [ "apt-get update", - "apt-get install -y snapd squashfuse fuse sudo", + "apt-get install -y snapd squashfuse fuse sudo lsb-release", "systemctl enable snapd", "useradd -m ubuntu -s /bin/bash", "adduser ubuntu sudo",