docker-compose(traces): metrics_generator enabled

Signed-off-by: Weifeng Wang <[email protected]>

Microservices mode(traces): add metrics_generator

Signed-off-by: Weifeng Wang <[email protected]>

[grafana-agent] Jaeger for Metrics Logs Traces

Signed-off-by: Weifeng Wang <[email protected]>

docker-compose/common/compose-include/load-mimir-rule.yaml

@@ -3,7 +3,7 @@ services:
     depends_on:
       gateway:
         condition: service_started
-    image: grafana/mimirtool:2.10.0
+    image: grafana/mimirtool:2.11.0
     volumes:
       - ../../../monitoring-mixins/mimir-mixin/deploy/rules.yaml:/mimir-mixin/rules.yaml
       - ../../../monitoring-mixins/mimir-mixin/deploy/alerts.yaml:/mimir-mixin/alerts.yaml

docker-compose/common/compose-include/load-tempo-rule.yaml

@@ -0,0 +1,15 @@
+services:
+  load-tempo-rules:
+    depends_on:
+      gateway:
+        condition: service_started
+    image: grafana/mimirtool:2.11.0
+    volumes:
+      - ../../../monitoring-mixins/tempo-mixin/deploy/rules.yaml:/tempo-mixin/rules.yaml
+      - ../../../monitoring-mixins/tempo-mixin/deploy/alerts.yaml:/tempo-mixin/alerts.yaml
+    environment:
+      - MIMIR_ADDRESS=http://gateway:8080
+      - MIMIR_TENANT_ID=anonymous
+    command: >-
+      rules load /tempo-mixin/rules.yaml /tempo-mixin/alerts.yaml
+  

docker-compose/common/config/agent-flow/metrics.river

@@ -45,6 +45,12 @@ discovery.relabel "containers" {
 		target_label  = "pod"
 	}
 
+	rule {
+		source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+		regex         = "(.*)"
+		target_label  = "service_name"
+	}
+
 	rule {
 		source_labels = ["__meta_docker_container_name"]
 		regex         = "/(.*)"

docker-compose/common/config/agent-flow/monolithic-mode-all.river

@@ -46,6 +46,12 @@ discovery.relabel "containers" {
 		target_label  = "pod"
 	}
 
+	rule {
+		source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+		regex         = "(.*)"
+		target_label  = "service_name"
+	}
+
 	rule {
 		source_labels = ["__meta_docker_container_name"]
 		regex         = "/(.*)"
@@ -150,6 +156,36 @@ loki.process "containers" {
 	}
 }
 
+/********************************************
+ * Jaeger for Metrics Logs Traces
+ ********************************************/
+
+otelcol.receiver.jaeger "containers" {
+	protocols {
+		grpc {
+			endpoint = "0.0.0.0:14250"
+		}
+
+		thrift_http {
+			endpoint = "0.0.0.0:14268"
+		}
+
+		thrift_binary {
+			endpoint = "0.0.0.0:6832"
+		}
+
+		thrift_compact {
+			endpoint = "0.0.0.0:6831"
+		}
+	}
+
+	output {
+		metrics = [otelcol.processor.batch.containers.input]
+		logs    = [otelcol.processor.batch.containers.input]
+		traces  = [otelcol.processor.batch.containers.input]
+	}
+}
+
 /********************************************
  * Otelcol for Metrics Logs Traces
  ********************************************/
@@ -166,7 +202,10 @@ otelcol.receiver.otlp "containers" {
 	output {
 		metrics = [otelcol.processor.batch.containers.input]
 		logs    = [otelcol.processor.batch.containers.input]
-		traces  = [otelcol.processor.batch.containers.input]
+		traces  = [
+			otelcol.processor.batch.containers.input,
+			otelcol.connector.spanlogs.autologging.input,
+		]
 	}
 }
 
@@ -178,6 +217,51 @@ otelcol.processor.batch "containers" {
 	}
 }
 
+// The OpenTelemetry spanlog connector processes incoming trace spans and extracts data from them ready
+// for logging.
+otelcol.connector.spanlogs "autologging" {
+	// We only want to output a line for each root span (ie. every single trace), and not for every
+	// process or span (outputting a line for every span would be extremely verbose).
+	spans     = false
+	roots     = true
+	processes = false
+	// We want to ensure that the following three span attributes are included in the log line, if present.
+	span_attributes = ["http.method", "http.target", "http.status_code"]
+
+	// Overrides the default key in the log line to be `traceId`, which is then used by Grafana to
+	// identify the trace ID for correlation with the Tempo datasource.
+	overrides {
+		trace_id_key = "traceId"
+	}
+	// Send to the OpenTelemetry Loki exporter.
+	output {
+		logs = [otelcol.exporter.loki.autologging.input]
+	}
+}
+
+// Simply forwards the incoming OpenTelemetry log format out as a Loki log.
+// We need this stage to ensure we can then process the logline as a Loki object.
+otelcol.exporter.loki "autologging" {
+	forward_to = [loki.process.autologging.receiver]
+}
+
+// The Loki processor allows us to accept a correctly formatted Loki log and mutate it into
+// a set of fields for output.
+loki.process "autologging" {
+	// The JSON stage simply extracts the `body` (the actual logline) from the Loki log, ignoring
+	// all other fields.
+	stage.json {
+		expressions = {"body" = ""}
+	}
+	// The output stage takes the body (the main logline) and uses this as the source for the output
+	// logline. In this case, it essentially turns it into logfmt.
+	stage.output {
+		source = "body"
+	}
+
+	forward_to = [module.file.docker_compose.exports.logs_receiver]
+}
+
 otelcol.processor.memory_limiter "containers" {
 	check_interval = "1s"
 	limit          = "256MiB"

docker-compose/common/config/agent-flow/traces.river

@@ -14,12 +14,160 @@ module.file "docker_compose" {
 	filename = env("AGENT_CONFIG_FOLDER") + "/modules/docker_compose.river"
 
 	arguments {
-		traces_endpoint = "gateway:4317"
+		metrics_endpoint = "http://gateway:8080"
+		traces_endpoint  = "gateway:4317"
 	}
 }
 
+discovery.docker "containers" {
+	host = "unix:///var/run/docker.sock"
+
+	filter {
+		name   = "status"
+		values = ["running"]
+	}
+}
+
+discovery.relabel "containers" {
+	targets = discovery.docker.containers.targets
+
+	// filter by service name
+	rule {
+		source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+		action        = "keep"
+		regex         = "(agent|mimir|grafana|loki|loki-.*|tempo|pyroscope)"
+	}
+
+	rule {
+		source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+		regex         = "(.*)"
+		replacement   = "monitoring-system/$1"
+		target_label  = "job"
+	}
+
+	rule {
+		source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+		regex         = "(.*)"
+		target_label  = "pod"
+	}
+
+	rule {
+		source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+		regex         = "(.*)"
+		target_label  = "service_name"
+	}
+
+	rule {
+		source_labels = ["__meta_docker_container_name"]
+		regex         = "/(.*)"
+		target_label  = "container"
+	}
+}
+
+/********************************************
+ * Metrics
+ ********************************************/
+
+prometheus.exporter.unix "containers" {
+	set_collectors     = ["cpu"]
+	disable_collectors = ["diskstats", "mdadm", "textfile", "hwmon"]
+}
+
+prometheus.scrape "integrations" {
+	targets = concat(
+		prometheus.exporter.unix.containers.targets,
+	)
+	scrape_interval = "15s"
+
+	enable_protobuf_negotiation = true
+	scrape_classic_histograms   = true
+
+	clustering {
+		enabled = true
+	}
+
+	forward_to = [prometheus.relabel.integrations.receiver]
+}
+
+prometheus.scrape "containers" {
+	targets         = discovery.relabel.containers.output
+	scrape_interval = "15s"
+
+	enable_protobuf_negotiation = true
+	scrape_classic_histograms   = true
+
+	clustering {
+		enabled = true
+	}
+
+	forward_to = [module.file.docker_compose.exports.metrics_receiver]
+}
+
+prometheus.scrape "minio" {
+	targets = [{"__address__" = "minio:9000", "job" = "minio-job"}]
+
+	scrape_interval = "15s"
+
+	enable_protobuf_negotiation = true
+	scrape_classic_histograms   = true
+
+	clustering {
+		enabled = true
+	}
+	metrics_path = "/minio/v2/metrics/cluster"
+
+	forward_to = [prometheus.relabel.integrations.receiver]
+}
+
+prometheus.relabel "integrations" {
+	rule {
+		source_labels = ["job"]
+		regex         = "(integrations|monitoring-system)/(.*)"
+		target_label  = "pod"
+		replacement   = "${2}"
+	}
+
+	rule {
+		source_labels = ["job"]
+		regex         = "(integrations|monitoring-system)/(.*)"
+		target_label  = "container"
+		replacement   = "${2}"
+	}
+
+	forward_to = [module.file.docker_compose.exports.metrics_receiver]
+}
+
 /********************************************
- * Otelcol for Traces
+ * Jaeger for Traces Metrics
+ ********************************************/
+
+otelcol.receiver.jaeger "containers" {
+	protocols {
+		grpc {
+			endpoint = "0.0.0.0:14250"
+		}
+
+		thrift_http {
+			endpoint = "0.0.0.0:14268"
+		}
+
+		thrift_binary {
+			endpoint = "0.0.0.0:6832"
+		}
+
+		thrift_compact {
+			endpoint = "0.0.0.0:6831"
+		}
+	}
+
+	output {
+		metrics = [otelcol.processor.batch.containers.input]
+		traces  = [otelcol.processor.batch.containers.input]
+	}
+}
+
+/********************************************
+ * Otelcol for Traces Metrics
  ********************************************/
 
 otelcol.receiver.otlp "containers" {
@@ -32,13 +180,15 @@ otelcol.receiver.otlp "containers" {
 	}
 
 	output {
-		traces = [otelcol.processor.batch.containers.input]
+		metrics = [otelcol.processor.batch.containers.input]
+		traces  = [otelcol.processor.batch.containers.input]
 	}
 }
 
 otelcol.processor.batch "containers" {
 	output {
-		traces = [otelcol.processor.memory_limiter.containers.input]
+		metrics = [otelcol.processor.memory_limiter.containers.input]
+		traces  = [otelcol.processor.memory_limiter.containers.input]
 	}
 }
 
@@ -47,6 +197,11 @@ otelcol.processor.memory_limiter "containers" {
 	limit          = "256MiB"
 
 	output {
-		traces = [module.file.docker_compose.exports.traces_receiver]
+		metrics = [otelcol.exporter.prometheus.containers.input]
+		traces  = [module.file.docker_compose.exports.traces_receiver]
 	}
 }
+
+otelcol.exporter.prometheus "containers" {
+	forward_to = [module.file.docker_compose.exports.metrics_receiver]
+}

docker-compose/common/config/loki/datasources.yaml

@@ -40,7 +40,7 @@ datasources:
 
 # Pyroscope for profiles
 - name: Profiles
-  type: phlare
+  type: grafana-pyroscope-datasource
   access: proxy
   orgId: 1
   uid: profiles

docker-compose/common/config/mimir/datasources.yaml

@@ -40,7 +40,7 @@ datasources:
 
 # Pyroscope for profiles
 - name: Profiles
-  type: phlare
+  type: grafana-pyroscope-datasource
   access: proxy
   orgId: 1
   uid: profiles

docker-compose/common/config/pyroscope/monolithic-mode-profiles.yaml

@@ -1,6 +1,10 @@
 analytics:
   reporting_enabled: false
 
+# https://grafana.com/docs/pyroscope/latest/configure-server/configure-disk-storage/#configure-pyroscope-disk-storage
+pyroscopedb:
+  max_block_duration: 5m
+
 storage:
   backend: s3
   s3: