[logs] add grafana-agent for logs

Signed-off-by: Weifeng Wang <[email protected]>

[logs] add grafana-agent for logs

Makefile

@@ -60,6 +60,7 @@ clean:  ## clean cluster
 .PHONY: manifests
 manifests: $(KUSTOMIZE)  ## Generates k8s manifests
 	$(KUSTOMIZE) build kubernetes/microservices-mode/metrics > kubernetes/microservices-mode/metrics/k8s-all-in-one.yaml
+	$(KUSTOMIZE) build kubernetes/microservices-mode/logs > kubernetes/microservices-mode/logs/k8s-all-in-one.yaml
 	$(KUSTOMIZE) build monitoring-mixins > monitoring-mixins/k8s-all-in-one.yaml
 
 ##@ General

docker-compose/monolithic-mode/README.md

@@ -2,8 +2,8 @@
 
 The monolithic mode runs all required components in a single process.
 
-- [Logs [ gent + Loki + Grafana]](./logs)
-- [Traces [ gent + Tempo + Grafana]](./traces)
-- [Metrics [ gent + Mimir + Grafana]](./metrics)
-- [Profiles [ gent + Pyroscope + Grafana]](./profiles)
-- [All In One [ gent + Loki + Tempo + Mimir + Pyroscope + Grafana]](./all-in-one)
+- [Logs [ agent + Loki + Grafana]](./logs)
+- [Traces [ agent + Tempo + Grafana]](./traces)
+- [Metrics [ agent + Mimir + Grafana]](./metrics)
+- [Profiles [ agent + Pyroscope + Grafana]](./profiles)
+- [All In One [ agent + Loki + Tempo + Mimir + Pyroscope + Grafana]](./all-in-one)

kubernetes/microservices-mode/logs/grafana-agent/Makefile

@@ -0,0 +1,14 @@
+include ../../../../.bingo/Variables.mk
+
+all: update build
+
+.PHONY: update
+update: $(KUSTOMIZE)
+	$(KUSTOMIZE) build --enable-helm . > k8s-all-in-one.yaml
+	@cp -rf charts/grafana-agent/values.yaml .
+	@rm -rf charts/
+
+.PHONY: build
+build: $(KUSTOMIZE)
+	@$(KUSTOMIZE) build --enable-helm . > k8s-all-in-one.yaml
+	@rm -rf charts/

kubernetes/microservices-mode/logs/grafana-agent/configs/logs.river

@@ -0,0 +1,29 @@
+/*
+The following example shows using the default all logs processing module, for
+a single tenant and specifying the destination url/credentials via environment
+variables.
+*/
+logging {
+  level  = coalesce(env("AGENT_LOG_LEVEL"), "info")
+  format = "logfmt"
+}
+
+module.file "lgtmp" {
+	filename = env("AGENT_CONFIG_FOLDER") + "/lgtmp.river"
+
+	arguments {
+    cluster       = coalesce(env("CLUSTER"), "k3d-k3s-codelab")
+    logs_endpoint = coalesce(env("LOGS_ENDPOINT"), "http://nginx.monitoring-system:3100")
+	}
+}
+
+module.git "event_logs" {
+  repository = "https://github.com/grafana/agent-modules.git"
+  revision   = "main"
+  path       = "modules/kubernetes/logs/events.river"
+
+  arguments {
+    forward_to = [module.file.lgtmp.exports.logs_receiver]
+    git_pull_freq = "0s"
+  }
+}

kubernetes/microservices-mode/logs/grafana-agent/configs/modules/grafana-cloud.river

@@ -0,0 +1,97 @@
+
+/********************************************
+ * ARGUMENTS
+ ********************************************/
+argument "stack_name" { }
+
+argument "token" { }
+
+/********************************************
+ * EXPORTS
+ ********************************************/
+
+export "metrics_receiver" {
+	value = prometheus.remote_write.default.receiver
+}
+
+export "logs_receiver" {
+	value = loki.write.default.receiver
+}
+
+export "traces_receiver" {
+	value = otelcol.exporter.otlp.default.input
+}
+
+export "profiles_receiver" {
+	value = pyroscope.write.default.receiver
+}
+
+export "stack_information" {
+	value = json_decode(remote.http.config_file.content)
+}
+
+/********************************************
+ * External information
+ ********************************************/
+
+remote.http "config_file" {
+	url = "https://grafana.com/api/instances/" + argument.stack_name.value
+
+	client {
+		bearer_token = argument.token.value
+	}
+	poll_frequency = "24h"
+}
+
+/********************************************
+ * Endpoints
+ ********************************************/
+
+// Metrics
+prometheus.remote_write "default" {
+	endpoint {
+		url = json_decode(remote.http.config_file.content)["hmInstancePromUrl"] + "/api/prom/push"
+
+		basic_auth {
+			username = json_decode(remote.http.config_file.content)["hmInstancePromId"]
+			password = argument.token.value
+		}
+	}
+}
+
+// Logs
+loki.write "default" {
+	endpoint {
+		url = json_decode(remote.http.config_file.content)["hlInstanceUrl"] + "/loki/api/v1/push"
+
+		basic_auth {
+			username = json_decode(remote.http.config_file.content)["hlInstanceId"]
+			password = argument.token.value
+		}
+	}
+}
+
+// Traces
+otelcol.auth.basic "default" {
+	username = json_decode(remote.http.config_file.content)["htInstanceId"]
+	password = argument.token.value
+}
+
+otelcol.exporter.otlp "default" {
+	client {
+		endpoint = json_decode(remote.http.config_file.content)["htInstanceUrl"] + ":443"
+		auth     = otelcol.auth.basic.default.handler
+	}
+}
+
+// Profiles
+pyroscope.write "default" {
+	endpoint {
+		url = json_decode(remote.http.config_file.content)["hpInstanceUrl"]
+
+		basic_auth {
+			username = json_decode(remote.http.config_file.content)["hpInstanceId"]
+			password = argument.token.value
+		}
+	}
+}

kubernetes/microservices-mode/logs/grafana-agent/configs/modules/lgtmp.river

@@ -0,0 +1,103 @@
+/********************************************
+ * ARGUMENTS
+ ********************************************/
+argument "cluster" {
+	optional = true
+	default  = "monitoring-system"
+}
+
+argument "metrics_endpoint" {
+	optional = true
+	default  = "http://mimir:8080"
+	//comment = "Where to send collected metrics."
+}
+
+argument "logs_endpoint" {
+	optional = true
+	default  = "http://loki:3100"
+	//comment = "Where to send collected logs."
+}
+
+argument "traces_endpoint" {
+	optional = true
+	default  = "tempo:4317"
+	//comment = "Where to send collected traces."
+}
+
+argument "profiles_endpoint" {
+	optional = true
+	default  = "http://pyroscope:4040"
+	//comment  = "Where to send collected profiles."
+}
+
+/********************************************
+ * EXPORTS
+ ********************************************/
+
+export "metrics_receiver" {
+	value = prometheus.remote_write.mimir.receiver
+}
+
+export "logs_receiver" {
+	value = loki.write.loki.receiver
+}
+
+export "traces_receiver" {
+	value = otelcol.exporter.otlp.tempo.input
+}
+
+export "profiles_receiver" {
+	value = pyroscope.write.pyroscope.receiver
+}
+
+/********************************************
+ * Endpoints
+ ********************************************/
+
+// Metrics
+prometheus.remote_write "mimir" {
+	endpoint {
+		url = argument.metrics_endpoint.value + "/api/v1/push"
+	}
+
+	external_labels = {
+		"scraped_by" = "grafana-agent",
+		"cluster" 	 = argument.cluster.value,
+	}
+}
+
+// Logs
+loki.write "loki" {
+	endpoint {
+		url = argument.logs_endpoint.value + "/loki/api/v1/push"
+	}
+
+	external_labels = {
+		"scraped_by" = "grafana-agent",
+		"cluster" 	 = argument.cluster.value,
+	}
+}
+
+// Traces
+otelcol.exporter.otlp "tempo" {
+	client {
+		endpoint = argument.traces_endpoint.value
+
+		tls {
+			insecure             = true
+			insecure_skip_verify = true
+		}
+	}
+}
+
+// Profiles
+pyroscope.write "pyroscope" {
+	endpoint {
+		url = argument.profiles_endpoint.value
+	}
+
+	external_labels = {
+		"scraped_by" = "grafana-agent",
+		"cluster" 	 = argument.cluster.value,
+	}
+}