feat(infra): setup for metric and modify repo structure and docker pr…

…ofile options (#20) * feat(infra): add prometheus container setup with docker compose - modify docker-compose.yml to include prometheus service - add prometheus.yml for prometheus configuration - setup prometheus container for metric monitoring service * feat(infra): add grafana agent container setup with docker compose - modify docker-compose.yml to include grafana agent service - add config.river for grafana agent configuration - setup grafana agent container for monitoring service * feat(infra): add caddyfile path for grafana agent and prometheus * fix(infra): modify grafana agent url of Caddyfile for collecting metric data * feat(infra): add step for running containers related to monitoring metric * feat(infra): modify config folder structure * feat(infra): modify docker compose profile to log, trace, metric
skkuding · Feb 22, 2024 · 767c114 · 767c114
1 parent 9e64ba5
commit 767c114
Show file tree

Hide file tree

Showing 9 changed files with 92 additions and 16 deletions.
diff --git a/config/agent/config.river b/config/agent/config.river
@@ -0,0 +1,20 @@
+otelcol.receiver.otlp "default" {
+    http {
+        endpoint = "0.0.0.0:4318"
+        metrics_url_path = "/v1/metrics"
+    }
+
+    output {
+        metrics  = [otelcol.exporter.prometheus.default.input]
+    }
+}
+
+otelcol.exporter.prometheus "default" {
+    forward_to = [prometheus.remote_write.local.receiver]
+}
+
+prometheus.remote_write "local" {
+    endpoint {
+        url = "http://prometheus:9090/api/v1/write"
+    }
+}
diff --git a/grafana/grafana.ini → config/grafana/grafana.ini b/grafana/grafana.ini → config/grafana/grafana.ini
diff --git a/grafana/loki-config-aws.yml → config/loki/loki-config-aws.yml b/grafana/loki-config-aws.yml → config/loki/loki-config-aws.yml
diff --git a/grafana/loki-config-stage.yml → config/loki/loki-config-stage.yml b/grafana/loki-config-stage.yml → config/loki/loki-config-stage.yml
diff --git a/grafana/loki-config-test.yml → config/loki/loki-config-test.yml b/grafana/loki-config-test.yml → config/loki/loki-config-test.yml
diff --git a/config/prometheus/prometheus.yml b/config/prometheus/prometheus.yml
@@ -0,0 +1,12 @@
+global:
+  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# scrape_configs:
+#  - job_name: 'grafana-agent'
+#    scrape_interval: 10s
+#    static_configs:
+#     #  - targets: ['localhost:9464']
+#     # - targets: ['agent:12345']
+#     - targets: ['localhost:12345']
diff --git a/.github/workflows/update-stage.yml b/.github/workflows/update-stage.yml
@@ -32,17 +32,17 @@ jobs:
             echo EOF
           } >> "$GITHUB_OUTPUT"
 
-      - name: Run Docker Compose Of Loki
+      - name: Run Docker Compose for Log (Loki, Minio, Grafana)
         run: |
-          docker compose --profile loki up -d --no-recreate
+          docker compose --profile log up -d --no-recreate
 
-      - name: Run Docker Compose Of Minio
+      - name: Run Docker Compose Of Trace (Agent, Tempo, Grafana)
         run: |
-          docker compose --profile minio up -d --no-recreate
+          docker compose --profile trace up -d --no-recreate
 
-      - name: Initialize grafana container
+      - name: Run Docker Compose for Metric (Agent, Prometheus, Grafana)
         run: |
-          docker compose --profile grafana-console up -d --no-recreate
+          docker compose --profile metric up -d --no-recreate
 
       - name: when caddy container down, caddy up
         if: steps.check-caddy-container.outputs.stdout == ''

diff --git a/Caddyfile b/Caddyfile
@@ -12,6 +12,23 @@ grafana.codedang.com {
 		reverse_proxy 127.0.0.1:3199
 	}
 
+	handle /agent {
+		handle /metric {
+			uri strip_prefix /agent/metric
+			reverse_proxy 127.0.0.1:4318/v1/metrics
+		}
+
+		handle {
+			uri strip_prefix /agent
+			reverse_proxy 127.0.0.1:12345
+		}
+	}
+
+	handle /prometheus {
+		uri strip_prefix /prometheus
+		reverse_proxy 127.0.0.1:9090
+	}
+
 	handle {
 		reverse_proxy 127.0.0.1:3000
 	}

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -15,7 +15,7 @@ services:
     network_mode: host
 
   minio:
-    profiles: ["minio"]
+    profiles: ["log"]
     container_name: minio
     image: minio/minio:latest
     volumes:
@@ -32,7 +32,7 @@ services:
     shm_size: "1gb"
 
   createbuckets:
-    profiles: ["minio"]
+    profiles: ["log"]
     image: minio/mc
     volumes:
       - minio_data_volume:/data
@@ -53,7 +53,7 @@ services:
       "
 
   loki-test:
-    profiles: ["loki"]
+    profiles: ["log"]
     image: grafana/loki:2.9.0
     container_name: loki-test
     environment:
@@ -62,13 +62,13 @@ services:
     ports:
       - "3199:3100"
     volumes:
-      - "$PWD/grafana/loki-config-test.yml:/etc/loki/loki-config.yml"
+      - "$PWD/config/loki/loki-config-test.yml:/etc/loki/loki-config.yml"
     command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true
     networks:
       - loki
 
   loki-stage:
-    profiles: ["loki"]
+    profiles: ["log"]
     image: grafana/loki:2.9.0
     container_name: loki-stage
     environment:
@@ -77,13 +77,13 @@ services:
     ports:
       - "3101:3100"
     volumes:
-      - "$PWD/grafana/loki-config-stage.yml:/etc/loki/loki-config.yml"
+      - "$PWD/config/loki/loki-config-stage.yml:/etc/loki/loki-config.yml"
     command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true
     networks:
       - loki
 
   loki-aws:
-    profiles: ["loki"]
+    profiles: ["log"]
     image: grafana/loki:2.9.0
     container_name: loki-aws
     environment:
@@ -92,13 +92,40 @@ services:
     ports:
       - "3100:3100"
     volumes:
-      - "$PWD/grafana/loki-config-aws.yml:/etc/loki/loki-config.yml"
+      - "$PWD/config/loki/loki-config-aws.yml:/etc/loki/loki-config.yml"
     command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true
     networks:
       - loki
 
+  prometheus:
+    profiles: ["metric"]
+    image: prom/prometheus:latest
+    container_name: prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - "$PWD/config/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml"
+    command: 
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.enable-remote-write-receiver'
+
+
+  agent:
+    profiles: ["metric", "trace"]
+    image: grafana/agent:latest
+    container_name: agent
+    environment:
+      - AGENT_MODE=flow
+    ports:
+      - "12345:12345"
+      - "4318:4318"   # OTLP over HTTP receiver
+    volumes: 
+      - $PWD/config/agent/config.river:/etc/agent/config.river
+    command: run --server.http.listen-addr=0.0.0.0:12345 /etc/agent/config.river
+
   grafana:
-    profiles: ["grafana-console"]
+    profiles: ["log", "metric", "trace"]
     image: grafana/grafana:latest
     env_file:
       - .env
@@ -112,7 +139,7 @@ services:
       - "3000:3000"
     volumes:
       - "grafana_storage:/var/lib/grafana"
-      - "$PWD/grafana/grafana.ini:/etc/grafana/grafana.ini"
+      - "$PWD/config/grafana/grafana.ini:/etc/grafana/grafana.ini"
     networks:
       - loki