Setup open telemetry collector

WORKSPACE.bzlmod

@@ -227,3 +227,21 @@ http_archive(
     sha256 = "2baf610b476727f24c62cc843419f55b157e1a05521a698c1c8b4ed676a766aa",
     url = "https://cache.agilebits.com/dist/1P/op2/pkg/v2.18.0/op_linux_amd64_v2.18.0.zip",
 )
+
+# ------------------------------------ regctl ------------------------------------ #
+
+http_file(
+    name = "regctl_arm64",
+    downloaded_file_path = "regctl",
+    executable = True,
+    sha256 = "3973aeff35a7cd28a0a094e3d453fbc7aa230df9cdcffad9d1c44c85ec3192c9",
+    url = "https://github.com/regclient/regclient/releases/download/v0.4.8/regctl-linux-arm64",
+)
+
+http_file(
+    name = "regctl_amd64",
+    downloaded_file_path = "regctl",
+    executable = True,
+    sha256 = "89031da0802724505a73571c7bfa55bc57872147245d0d18bbc6123d7d4245b5",
+    url = "https://github.com/regclient/regclient/releases/download/v0.4.8/regctl-linux-amd64",
+)

provisioner/BUILD.bazel

@@ -30,6 +30,7 @@ pyinfra_run(
         "deploys/monitoring/files/docker-compose.yml.j2",
         "deploys/monitoring/files/newrelic-infra.yml.j2",
         "deploys/monitoring/files/nri-prometheus-config.yaml",
+        "deploys/monitoring/files/otel-collector-config.yaml",
         "deploys/network/files/99_config.yaml",
         "deploys/teleport/files/teleport.yaml.j2",
     ],

provisioner/deploys/monitoring/files/docker-compose.yml.j2

@@ -2,25 +2,56 @@ name: monitoring
 version: '3.8'
 services:
   # https://promhippie.github.io/github_exporter/#getting-started
+
   github_exporter:
     container_name: github_exporter
     image: promhippie/github-exporter:latest@sha256:d2c4dc75d44666ebf1e3433f497fe18e2be6f4bfeeca770790944d745dc1c8ed
     restart: always
     environment:
       - GITHUB_EXPORTER_TOKEN={{ github_exporter_token }}
       - GITHUB_EXPORTER_LOG_PRETTY=true
-      - GITHUB_EXPORTER_ORG=mvgijssel
       - GITHUB_EXPORTER_REPO=mvgijssel/setup
       - GITHUB_EXPORTER_COLLECTOR_WORKFLOWS=true
       - GITHUB_EXPORTER_COLLECTOR_RUNNERS=false
       - GITHUB_EXPORTER_COLLECTOR_BILLING=false
       - GITHUB_EXPORTER_COLLECTOR_ADMIN=false
+      - SETUP_ENV={{ setup_env }}
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5' # Maximum of 0.5 CPU cores
+          memory: '100M' # Maximum of 512 megabytes of memory
 
   nri-prometheus:
     container_name: nri-prometheus
     image: newrelic/nri-prometheus:2.18.1@sha256:cd1b512186551293177fa7c48d85d876f2125c5da5978e03c7a4833b8082bc79
     restart: always
     environment:
       - LICENSE_KEY={{ new_relic_license_key }}
+      - SETUP_ENV={{ setup_env }}
     volumes:
       - "/opt/monitoring/nri-prometheus-config.yaml:/config.yaml"
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5' # Maximum of 0.5 CPU cores
+          memory: '100M' # Maximum of 512 megabytes of memory
+
+  otel-collector:
+    container_name: otel-collector
+    image: otel/opentelemetry-collector-contrib:0.79.0@sha256:a1930dca21b6baa78b1f361f5ced863bb86e30299cc9c3e6be78487af7b720a5
+    restart: always
+    command: [ --config=/etc/otel-collector-config.yaml ]
+    volumes:
+      - "/opt/monitoring/otel-collector-config.yaml:/etc/otel-collector-config.yaml"
+    ports:
+      - "127.0.0.1:13133:13133" # health_check extension
+      - "127.0.0.1:4317:4317" # OTLP gRPC receiver
+    environment:
+      - LICENSE_KEY={{ new_relic_license_key }}
+      - SETUP_ENV={{ setup_env }}
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5' # Maximum of 0.5 CPU cores
+          memory: '100M' # Maximum of 512 megabytes of memory

provisioner/deploys/monitoring/files/otel-collector-config.yaml

@@ -0,0 +1,51 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+processors:
+  batch:
+    timeout: 10s
+
+  transform:
+    error_mode: ignore
+    trace_statements:
+      - context: resource
+        statements:
+          - replace_match(attributes["service.name"], "teleport", "teleport-${SETUP_ENV}")
+          - set(attributes["env"], "${SETUP_ENV}")
+
+exporters:
+  otlp:
+    endpoint: https://otlp.eu01.nr-data.net:443
+    headers:
+      api-key: ${LICENSE_KEY}
+
+  # logging:
+  #   verbosity: detailed
+  #   sampling_initial: 5
+  #   sampling_thereafter: 200
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+
+service:
+  extensions: [health_check]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [transform, batch]
+      # exporters: [logging, otlp]
+      exporters: [otlp]
+    metrics:
+      receivers: [otlp]
+      processors: [transform, batch]
+      # exporters: [logging, otlp]
+      exporters: [otlp]
+    logs:
+      receivers: [otlp]
+      processors: [transform, batch]
+      # exporters: [logging, otlp]
+      exporters: [otlp]

provisioner/deploys/monitoring/tasks/install_monitoring.py

@@ -30,6 +30,7 @@ def install_monitoring():
         group="root",
         github_exporter_token=github_exporter_token,
         new_relic_license_key=new_relic_license_key,
+        setup_env=host.data.setup_env,
     )
 
     nri_prometheus_config = files.put(
@@ -42,7 +43,21 @@ def install_monitoring():
         mode="0644",
     )
 
-    if docker_compose.changed or nri_prometheus_config.changed:
+    otel_collector_config = files.put(
+        name="Copy Oopen Telemetry config",
+        src="provisioner/deploys/monitoring/files/otel-collector-config.yaml",
+        dest="/opt/monitoring/otel-collector-config.yaml",
+        _sudo=True,
+        user="root",
+        group="root",
+        mode="0644",
+    )
+
+    if (
+        docker_compose.changed
+        or nri_prometheus_config.changed
+        or otel_collector_config.changed
+    ):
         server.shell(
             name="Start the monitoring service",
             commands=[

provisioner/deploys/teleport/files/teleport.yaml.j2

@@ -28,3 +28,7 @@ proxy_service:
   acme:
     enabled: "{{ teleport_acme_enabled }}"
     email: "{{ teleport_acme_email }}"
+tracing_service:
+  enabled: yes
+  exporter_url: grpc://127.0.0.1:4317
+  sampling_rate_per_million: 1000000 # 1000000 will sample **all** spans generated by Teleport

provisioner/test_provisioner.py

@@ -108,6 +108,32 @@ def test_nri_prometheus_config(host):
         assert config.contains("http://github_exporter:9504/metrics")
 
 
+def test_otel_collector_service(host):
+    otel_collector = host.docker("otel-collector")
+    assert otel_collector.is_running
+
+
+def test_otel_collector_config(host):
+    with host.sudo():
+        config = host.file("/opt/monitoring/otel-collector-config.yaml")
+        assert config.exists
+        assert config.contains("https://otlp.eu01.nr-data.net:443")
+
+
+def test_otel_collector_port_is_open(host):
+    assert host.socket("tcp://127.0.0.1:4317").is_listening
+    assert not host.socket("tcp://0.0.0.0:4317").is_listening
+
+    assert host.socket("tcp://127.0.0.1:13133").is_listening
+    assert not host.socket("tcp://0.0.0.0:13133").is_listening
+
+
+def test_otel_collector_health(host):
+    assert '"status":"Server available"' in host.check_output(
+        "curl --fail -L http://localhost:13133"
+    )
+
+
 def test_microk8s_installed(host):
     assert "microk8s" in host.check_output("snap list")
 

tools/regctl/BUILD.bazel

@@ -0,0 +1,23 @@
+load("@rules_task//:defs.bzl", "cmd", "task")
+
+package(default_visibility = ["//visibility:public"])
+
+regctl_file = select({
+    "@platforms//cpu:aarch64": ["@regctl_arm64//file:file"],
+    "@platforms//cpu:x86_64": ["@regctl_amd64//file:file"],
+})
+
+sh_binary(
+    name = "regctl_binary",
+    srcs = regctl_file,
+)
+
+task(
+    name = "regctl",
+    cmds = [
+        cmd.shell(
+            cmd.executable(":regctl_binary"),
+            "$CLI_ARGS",
+        ),
+    ],
+)