From 767c1140f9bac79921557645b3ec7fe939c6e6e7 Mon Sep 17 00:00:00 2001 From: ssupecial <56429615+ssupecial@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:22:29 +0900 Subject: [PATCH] feat(infra): setup for metric and modify repo structure and docker profile options (#20) * feat(infra): add prometheus container setup with docker compose - modify docker-compose.yml to include prometheus service - add prometheus.yml for prometheus configuration - setup prometheus container for metric monitoring service * feat(infra): add grafana agent container setup with docker compose - modify docker-compose.yml to include grafana agent service - add config.river for grafana agent configuration - setup grafana agent container for monitoring service * feat(infra): add caddyfile path for grafana agent and prometheus * fix(infra): modify grafana agent url of Caddyfile for collecting metric data * feat(infra): add step for running containers related to monitoring metric * feat(infra): modify config folder structure * feat(infra): modify docker compose profile to log, trace, metric --- "\bconfig/agent/config.river" | 20 ++++++++ .../grafana/grafana.ini" | 0 .../loki/loki-config-aws.yml" | 0 .../loki/loki-config-stage.yml" | 0 .../loki/loki-config-test.yml" | 0 "\bconfig/prometheus/prometheus.yml" | 12 +++++ .github/workflows/update-stage.yml | 12 ++--- Caddyfile | 17 +++++++ docker-compose.yml | 47 +++++++++++++++---- 9 files changed, 92 insertions(+), 16 deletions(-) create mode 100644 "\bconfig/agent/config.river" rename grafana/grafana.ini => "\bconfig/grafana/grafana.ini" (100%) rename grafana/loki-config-aws.yml => "\bconfig/loki/loki-config-aws.yml" (100%) rename grafana/loki-config-stage.yml => "\bconfig/loki/loki-config-stage.yml" (100%) rename grafana/loki-config-test.yml => "\bconfig/loki/loki-config-test.yml" (100%) create mode 100644 "\bconfig/prometheus/prometheus.yml" diff --git "a/\bconfig/agent/config.river" "b/\bconfig/agent/config.river" new file mode 100644 index 0000000..0d8fe0e --- /dev/null +++ "b/\bconfig/agent/config.river" @@ -0,0 +1,20 @@ +otelcol.receiver.otlp "default" { + http { + endpoint = "0.0.0.0:4318" + metrics_url_path = "/v1/metrics" + } + + output { + metrics = [otelcol.exporter.prometheus.default.input] + } +} + +otelcol.exporter.prometheus "default" { + forward_to = [prometheus.remote_write.local.receiver] +} + +prometheus.remote_write "local" { + endpoint { + url = "http://prometheus:9090/api/v1/write" + } +} \ No newline at end of file diff --git a/grafana/grafana.ini "b/\bconfig/grafana/grafana.ini" similarity index 100% rename from grafana/grafana.ini rename to "\bconfig/grafana/grafana.ini" diff --git a/grafana/loki-config-aws.yml "b/\bconfig/loki/loki-config-aws.yml" similarity index 100% rename from grafana/loki-config-aws.yml rename to "\bconfig/loki/loki-config-aws.yml" diff --git a/grafana/loki-config-stage.yml "b/\bconfig/loki/loki-config-stage.yml" similarity index 100% rename from grafana/loki-config-stage.yml rename to "\bconfig/loki/loki-config-stage.yml" diff --git a/grafana/loki-config-test.yml "b/\bconfig/loki/loki-config-test.yml" similarity index 100% rename from grafana/loki-config-test.yml rename to "\bconfig/loki/loki-config-test.yml" diff --git "a/\bconfig/prometheus/prometheus.yml" "b/\bconfig/prometheus/prometheus.yml" new file mode 100644 index 0000000..ce1b0dd --- /dev/null +++ "b/\bconfig/prometheus/prometheus.yml" @@ -0,0 +1,12 @@ +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# scrape_configs: +# - job_name: 'grafana-agent' +# scrape_interval: 10s +# static_configs: +# # - targets: ['localhost:9464'] +# # - targets: ['agent:12345'] +# - targets: ['localhost:12345'] \ No newline at end of file diff --git a/.github/workflows/update-stage.yml b/.github/workflows/update-stage.yml index be2c8c3..96585d6 100644 --- a/.github/workflows/update-stage.yml +++ b/.github/workflows/update-stage.yml @@ -32,17 +32,17 @@ jobs: echo EOF } >> "$GITHUB_OUTPUT" - - name: Run Docker Compose Of Loki + - name: Run Docker Compose for Log (Loki, Minio, Grafana) run: | - docker compose --profile loki up -d --no-recreate + docker compose --profile log up -d --no-recreate - - name: Run Docker Compose Of Minio + - name: Run Docker Compose Of Trace (Agent, Tempo, Grafana) run: | - docker compose --profile minio up -d --no-recreate + docker compose --profile trace up -d --no-recreate - - name: Initialize grafana container + - name: Run Docker Compose for Metric (Agent, Prometheus, Grafana) run: | - docker compose --profile grafana-console up -d --no-recreate + docker compose --profile metric up -d --no-recreate - name: when caddy container down, caddy up if: steps.check-caddy-container.outputs.stdout == '' diff --git a/Caddyfile b/Caddyfile index 96b5b61..f633771 100644 --- a/Caddyfile +++ b/Caddyfile @@ -12,6 +12,23 @@ grafana.codedang.com { reverse_proxy 127.0.0.1:3199 } + handle /agent { + handle /metric { + uri strip_prefix /agent/metric + reverse_proxy 127.0.0.1:4318/v1/metrics + } + + handle { + uri strip_prefix /agent + reverse_proxy 127.0.0.1:12345 + } + } + + handle /prometheus { + uri strip_prefix /prometheus + reverse_proxy 127.0.0.1:9090 + } + handle { reverse_proxy 127.0.0.1:3000 } diff --git a/docker-compose.yml b/docker-compose.yml index 8d51ccb..574a174 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: network_mode: host minio: - profiles: ["minio"] + profiles: ["log"] container_name: minio image: minio/minio:latest volumes: @@ -32,7 +32,7 @@ services: shm_size: "1gb" createbuckets: - profiles: ["minio"] + profiles: ["log"] image: minio/mc volumes: - minio_data_volume:/data @@ -53,7 +53,7 @@ services: " loki-test: - profiles: ["loki"] + profiles: ["log"] image: grafana/loki:2.9.0 container_name: loki-test environment: @@ -62,13 +62,13 @@ services: ports: - "3199:3100" volumes: - - "$PWD/grafana/loki-config-test.yml:/etc/loki/loki-config.yml" + - "$PWD/config/loki/loki-config-test.yml:/etc/loki/loki-config.yml" command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true networks: - loki loki-stage: - profiles: ["loki"] + profiles: ["log"] image: grafana/loki:2.9.0 container_name: loki-stage environment: @@ -77,13 +77,13 @@ services: ports: - "3101:3100" volumes: - - "$PWD/grafana/loki-config-stage.yml:/etc/loki/loki-config.yml" + - "$PWD/config/loki/loki-config-stage.yml:/etc/loki/loki-config.yml" command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true networks: - loki loki-aws: - profiles: ["loki"] + profiles: ["log"] image: grafana/loki:2.9.0 container_name: loki-aws environment: @@ -92,13 +92,40 @@ services: ports: - "3100:3100" volumes: - - "$PWD/grafana/loki-config-aws.yml:/etc/loki/loki-config.yml" + - "$PWD/config/loki/loki-config-aws.yml:/etc/loki/loki-config.yml" command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true networks: - loki + prometheus: + profiles: ["metric"] + image: prom/prometheus:latest + container_name: prometheus + ports: + - "9090:9090" + volumes: + - "$PWD/config/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml" + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.enable-remote-write-receiver' + + + agent: + profiles: ["metric", "trace"] + image: grafana/agent:latest + container_name: agent + environment: + - AGENT_MODE=flow + ports: + - "12345:12345" + - "4318:4318" # OTLP over HTTP receiver + volumes: + - $PWD/config/agent/config.river:/etc/agent/config.river + command: run --server.http.listen-addr=0.0.0.0:12345 /etc/agent/config.river + grafana: - profiles: ["grafana-console"] + profiles: ["log", "metric", "trace"] image: grafana/grafana:latest env_file: - .env @@ -112,7 +139,7 @@ services: - "3000:3000" volumes: - "grafana_storage:/var/lib/grafana" - - "$PWD/grafana/grafana.ini:/etc/grafana/grafana.ini" + - "$PWD/config/grafana/grafana.ini:/etc/grafana/grafana.ini" networks: - loki