From 4d382caa8d0f44b14dc6815b39ce8cf5bd8fe057 Mon Sep 17 00:00:00 2001 From: Weifeng Wang Date: Wed, 6 Mar 2024 23:30:59 +0800 Subject: [PATCH] imporve tracesToProfiles and tracesToLogsV2 Signed-off-by: Weifeng Wang imporve tracesToProfiles and tracesToLogsV2 Signed-off-by: Weifeng Wang imporve tracesToProfiles and tracesToLogsV2 Signed-off-by: Weifeng Wang imporve tracesToProfiles and tracesToLogsV2 Signed-off-by: Weifeng Wang --- .../common/grafana-agent/configs/config.river | 76 ---------- .../configs/modules/traces.river | 130 ++++++++++++++++-- .../manifests/k8s-all-in-one.yaml | 76 +++++----- .../common/grafana-agent/values-k3d-k3s.yaml | 1 + .../profiles/configs/config.river | 33 +++++ .../profiles/k8s-all-in-one.yaml | 21 +++ .../profiles/kustomization.yaml | 7 + .../traces/configs/config.river | 35 +---- .../traces/configs/tempo-distributed.yaml | 1 + .../traces/k8s-all-in-one.yaml | 43 +++--- .../traces/kustomization.yaml | 12 +- .../all-in-one/configs/config.river | 50 +------ .../grafana-datasources-all-in-one.yaml | 6 +- .../all-in-one/k8s-all-in-one.yaml | 46 +++---- .../all-in-one/kustomization.yaml | 20 +-- .../monolithic-mode/logs/k8s-all-in-one.yaml | 1 + .../logs/loki/k8s-all-in-one.yaml | 1 + .../logs/loki/values-k3d-k3s.yaml | 1 + .../metrics/k8s-all-in-one.yaml | 1 + .../metrics/mimir/k8s-all-in-one.yaml | 1 + .../metrics/mimir/mimir-dep.yaml | 1 + .../profiles/configs/config.river | 15 +- .../profiles/k8s-all-in-one.yaml | 7 +- .../profiles/pyroscope/k8s-all-in-one.yaml | 1 + .../profiles/pyroscope/values-k3d-k3s.yaml | 11 ++ .../traces/configs/config.river | 35 +---- .../traces/k8s-all-in-one.yaml | 35 ++--- .../monolithic-mode/traces/kustomization.yaml | 12 +- .../traces/tempo/k8s-all-in-one.yaml | 1 + .../traces/tempo/values-k3d-k3s.yaml | 1 + 30 files changed, 322 insertions(+), 359 deletions(-) create mode 100644 kubernetes/microservices-mode/profiles/configs/config.river diff --git a/kubernetes/common/grafana-agent/configs/config.river b/kubernetes/common/grafana-agent/configs/config.river index f99e7c55..859ecfc8 100644 --- a/kubernetes/common/grafana-agent/configs/config.river +++ b/kubernetes/common/grafana-agent/configs/config.river @@ -7,79 +7,3 @@ logging { level = coalesce(env("AGENT_LOG_LEVEL"), "info") format = "logfmt" } - -module.file "lgtmp" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/lgtmp.river" - - arguments { - cluster = coalesce(env("CLUSTER"), "k3d-k3s-codelab") - logs_endpoint = coalesce(env("LOGS_ENDPOINT"), "http://nginx.gateway.svc:3100") - metrics_endpoint = coalesce(env("METRICS_ENDPOINT"), "http://nginx.gateway.svc:8080") - profiles_endpoint = coalesce(env("PROFILES_ENDPOINT"), "http://nginx.gateway.svc:4040") - traces_endpoint = coalesce(env("TRACES_ENDPOINT"), "nginx.gateway.svc:4317") - } -} - -/******************************************** - * Metrics - ********************************************/ -module.file "metrics_primary" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/metrics.river" - - arguments { - forward_to = [module.file.lgtmp.exports.metrics_receiver] - clustering = true - } -} - -/******************************************** - * Logs - ********************************************/ -module.git "logs_primary" { - repository = "https://github.com/grafana/agent-modules.git" - revision = "main" - path = "modules/kubernetes/logs/all.river" - - arguments { - forward_to = [module.file.lgtmp.exports.logs_receiver] - } -} - -/******************************************** - * Traces - ********************************************/ -module.file "traces_primary" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/traces.river" - - arguments { - metrics_forward_to = [module.file.lgtmp.exports.metrics_receiver] - logs_forward_to = [module.file.lgtmp.exports.logs_receiver] - traces_forward_to = [module.file.lgtmp.exports.traces_receiver] - cluster = coalesce(env("CLUSTER"), "k3d-k3s-codelab") - } -} - -/******************************************** - * Profiles - ********************************************/ -module.file "profiles_primary" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/profiles.river" - - arguments { - forward_to = [module.file.lgtmp.exports.profiles_receiver] - clustering = true - } -} - -/******************************************** - * Agent Integrations - ********************************************/ -module.file "agent_integrations" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/integrations.river" - - arguments { - name = "agent-integrations" - namespace = "monitoring-system" - forward_to = [module.file.lgtmp.exports.metrics_receiver] - } -} diff --git a/kubernetes/common/grafana-agent/configs/modules/traces.river b/kubernetes/common/grafana-agent/configs/modules/traces.river index f5bca557..7bc99800 100644 --- a/kubernetes/common/grafana-agent/configs/modules/traces.river +++ b/kubernetes/common/grafana-agent/configs/modules/traces.river @@ -1,6 +1,10 @@ /* Module: traces */ + +/******************************************** + * ARGUMENTS + ********************************************/ argument "traces_forward_to" { optional = false } @@ -28,6 +32,13 @@ argument "otlp_grpc_endpoint" { default = "0.0.0.0:4317" } +/******************************************** + * EXPORTS + ********************************************/ +export "agent_traces_input" { + value = otelcol.processor.batch.default.input +} + /******************************************** * Jaeger for Metrics Logs Traces ********************************************/ @@ -52,9 +63,9 @@ otelcol.receiver.jaeger "default" { } output { - metrics = [otelcol.processor.batch.default.input] - logs = [otelcol.processor.batch.default.input] - traces = [otelcol.processor.batch.default.input] + metrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input] + logs = [otelcol.processor.resourcedetection.default.input] + traces = [otelcol.processor.resourcedetection.default.input] } } @@ -72,15 +83,114 @@ otelcol.receiver.otlp "default" { } output { - metrics = [otelcol.processor.batch.default.input] - logs = [otelcol.processor.batch.default.input] + metrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input] + logs = [otelcol.processor.resourcedetection.default.input] traces = [ - otelcol.processor.batch.default.input, + otelcol.processor.resourcedetection.default.input, otelcol.connector.spanlogs.autologging.input, ] } } +otelcol.processor.resourcedetection "default" { + detectors = ["env"] + + output { + logs = [otelcol.processor.k8sattributes.default.input] + metrics = [otelcol.processor.k8sattributes.default.input] + traces = [otelcol.processor.k8sattributes.default.input] + } +} + +otelcol.processor.transform "add_metric_datapoint_attributes" { + error_mode = "ignore" + + metric_statements { + context = "datapoint" + statements = [ + `set(attributes["deployment.environment"], resource.attributes["deployment.environment"])`, + `set(attributes["service.version"], resource.attributes["service.version"])`, + ] + } + + output { + metrics = [otelcol.processor.k8sattributes.default.input] + } +} + +otelcol.processor.k8sattributes "default" { + extract { + metadata = [ + "k8s.namespace.name", + "k8s.pod.name", + "k8s.deployment.name", + "k8s.statefulset.name", + "k8s.daemonset.name", + "k8s.cronjob.name", + "k8s.job.name", + "k8s.node.name", + "k8s.pod.uid", + "k8s.pod.start_time", + ] + } + + pod_association { + source { + from = "connection" + } + } + + output { + metrics = [otelcol.processor.transform.add_resource_attributes.input] + logs = [otelcol.processor.transform.add_resource_attributes.input] + traces = [otelcol.processor.transform.add_resource_attributes.input] + } +} + +otelcol.processor.transform "add_resource_attributes" { + error_mode = "ignore" + + metric_statements { + context = "resource" + statements = [ + `set(attributes["k8s.cluster.name"], "k3d-k3s-codelab") where attributes["k8s.cluster.name"] == nil`, + ] + } + + log_statements { + context = "resource" + statements = [ + `set(attributes["pod"], attributes["k8s.pod.name"])`, + `set(attributes["namespace"], attributes["k8s.namespace.name"])`, + `set(attributes["loki.resource.labels"], "pod, namespace, cluster, job")`, + `set(attributes["k8s.cluster.name"], "k3d-k3s-codelab") where attributes["k8s.cluster.name"] == nil`, + ] + } + + trace_statements { + context = "resource" + statements = [ + `set(attributes["k8s.cluster.name"], "k3d-k3s-codelab") where attributes["k8s.cluster.name"] == nil`, + ] + } + + output { + metrics = [otelcol.processor.filter.default.input] + logs = [otelcol.processor.filter.default.input] + traces = [otelcol.processor.filter.default.input] + } +} + +otelcol.processor.filter "default" { + error_mode = "ignore" + + output { + metrics = [otelcol.processor.batch.default.input] + logs = [otelcol.processor.batch.default.input] + traces = [otelcol.processor.batch.default.input] + } +} + otelcol.processor.batch "default" { send_batch_size = 16384 send_batch_max_size = 0 @@ -98,14 +208,6 @@ otelcol.processor.memory_limiter "default" { limit_percentage = 50 spike_limit_percentage = 30 - output { - metrics = [otelcol.processor.k8sattributes.default.input] - logs = [otelcol.processor.k8sattributes.default.input] - traces = [otelcol.processor.k8sattributes.default.input] - } -} - -otelcol.processor.k8sattributes "default" { output { metrics = [otelcol.exporter.prometheus.tracesmetrics.input] logs = [otelcol.exporter.loki.traceslogs.input] diff --git a/kubernetes/common/grafana-agent/manifests/k8s-all-in-one.yaml b/kubernetes/common/grafana-agent/manifests/k8s-all-in-one.yaml index 27c8d2e8..b454c810 100644 --- a/kubernetes/common/grafana-agent/manifests/k8s-all-in-one.yaml +++ b/kubernetes/common/grafana-agent/manifests/k8s-all-in-one.yaml @@ -136,33 +136,7 @@ data: config.river: "/*\nThe following example shows using the default all logs processing module, for\na single tenant and specifying the destination url/credentials via environment\nvariables.\n*/\nlogging {\n\tlevel = coalesce(env(\"AGENT_LOG_LEVEL\"), - \"info\")\n\tformat = \"logfmt\"\n}\n\nmodule.file \"lgtmp\" {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), - \"/etc/agent-modules\") + \"/lgtmp.river\"\n\n\targuments {\n\t\tcluster = - coalesce(env(\"CLUSTER\"), \"k3d-k3s-codelab\")\n\t\tlogs_endpoint = coalesce(env(\"LOGS_ENDPOINT\"), - \"http://nginx.gateway.svc:3100\")\n\t\tmetrics_endpoint = coalesce(env(\"METRICS_ENDPOINT\"), - \"http://nginx.gateway.svc:8080\")\n\t\tprofiles_endpoint = coalesce(env(\"PROFILES_ENDPOINT\"), - \"http://nginx.gateway.svc:4040\")\n\t\ttraces_endpoint = coalesce(env(\"TRACES_ENDPOINT\"), - \"nginx.gateway.svc:4317\")\n\t}\n}\n\n/********************************************\n - * Metrics\n ********************************************/\nmodule.file \"metrics_primary\" - {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\") - + \"/metrics.river\"\n\n\targuments {\n\t\tforward_to = [module.file.lgtmp.exports.metrics_receiver]\n\t\tclustering - = true\n\t}\n}\n\n/********************************************\n * Logs\n ********************************************/\nmodule.git - \"logs_primary\" {\n\trepository = \"https://github.com/grafana/agent-modules.git\"\n\trevision - \ = \"main\"\n\tpath = \"modules/kubernetes/logs/all.river\"\n\n\targuments - {\n\t\tforward_to = [module.file.lgtmp.exports.logs_receiver]\n\t}\n}\n\n/********************************************\n - * Traces\n ********************************************/\nmodule.file \"traces_primary\" - {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\") - + \"/traces.river\"\n\n\targuments {\n\t\tmetrics_forward_to = [module.file.lgtmp.exports.metrics_receiver]\n\t\tlogs_forward_to - \ = [module.file.lgtmp.exports.logs_receiver]\n\t\ttraces_forward_to = [module.file.lgtmp.exports.traces_receiver]\n\t\tcluster - \ = coalesce(env(\"CLUSTER\"), \"k3d-k3s-codelab\")\n\t}\n}\n\n/********************************************\n - * Profiles\n ********************************************/\nmodule.file \"profiles_primary\" - {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\") - + \"/profiles.river\"\n\n\targuments {\n\t\tforward_to = [module.file.lgtmp.exports.profiles_receiver]\n\t\tclustering - = true\n\t}\n}\n\n/********************************************\n * Agent Integrations\n - ********************************************/\nmodule.file \"agent_integrations\" - {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\") - + \"/integrations.river\"\n\n\targuments {\n\t\tname = \"agent-integrations\"\n\t\tnamespace - \ = \"monitoring-system\"\n\t\tforward_to = [module.file.lgtmp.exports.metrics_receiver]\n\t}\n}\n" + \"info\")\n\tformat = \"logfmt\"\n}\n" kind: ConfigMap metadata: name: agent-config @@ -561,31 +535,56 @@ data: = false\n\t\t}\n\n\t\tprofile.process_cpu {\n\t\t\tenabled = false\n\t\t}\n\n\t\tprofile.goroutine {\n\t\t\tenabled = false\n\t\t}\n\n\t\tprofile.block {\n\t\t\tenabled = false\n\t\t}\n\n\t\tprofile.mutex {\n\t\t\tenabled = false\n\t\t}\n\n\t\tprofile.fgprof {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" - traces.river: "/*\nModule: traces\n*/\nargument \"traces_forward_to\" {\n\toptional - = false\n}\n\nargument \"logs_forward_to\" {\n\toptional = false\n}\n\nargument + traces.river: "/*\nModule: traces\n*/\n\n/********************************************\n + * ARGUMENTS\n ********************************************/\nargument \"traces_forward_to\" + {\n\toptional = false\n}\n\nargument \"logs_forward_to\" {\n\toptional = false\n}\n\nargument \"metrics_forward_to\" {\n\toptional = false\n}\n\nargument \"cluster\" {\n\toptional = true\n\tdefault = \"k3d-k3s-codelab\"\n}\n\nargument \"otlp_http_endpoint\" {\n\toptional = true\n\tdefault = \"0.0.0.0:4318\"\n}\n\nargument \"otlp_grpc_endpoint\" {\n\toptional = true\n\tdefault = \"0.0.0.0:4317\"\n}\n\n/********************************************\n + * EXPORTS\n ********************************************/\nexport \"agent_traces_input\" + {\n\tvalue = otelcol.processor.batch.default.input\n}\n\n/********************************************\n * Jaeger for Metrics Logs Traces\n ********************************************/\n\notelcol.receiver.jaeger \"default\" {\n\tprotocols {\n\t\tgrpc {\n\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t}\n\n\t\tthrift_http {\n\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t}\n\n\t\tthrift_binary {\n\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t}\n\n\t\tthrift_compact {\n\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t}\n\t}\n\n\toutput - {\n\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\tlogs = [otelcol.processor.batch.default.input]\n\t\ttraces - \ = [otelcol.processor.batch.default.input]\n\t}\n}\n\n/********************************************\n + {\n\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\ttraces = [otelcol.processor.resourcedetection.default.input]\n\t}\n}\n\n/********************************************\n * Otelcol for Metrics Logs Traces\n ********************************************/\n// https://grafana.com/docs/agent/latest/flow/reference/components/otelcol.receiver.otlp/\notelcol.receiver.otlp \"default\" {\n\tgrpc {\n\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t}\n\n\thttp {\n\t\tendpoint = argument.otlp_http_endpoint.value\n\t}\n\n\toutput {\n\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\tlogs = [otelcol.processor.batch.default.input]\n\t\ttraces - \ = [\n\t\t\totelcol.processor.batch.default.input,\n\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t]\n\t}\n}\n\notelcol.processor.batch + = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\ttraces = [\n\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t]\n\t}\n}\n\notelcol.processor.resourcedetection + \"default\" {\n\tdetectors = [\"env\"]\n\n\toutput {\n\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\tmetrics + = [otelcol.processor.k8sattributes.default.input]\n\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t}\n}\n\notelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\terror_mode = \"ignore\"\n\n\tmetric_statements + {\n\t\tcontext = \"datapoint\"\n\t\tstatements = [\n\t\t\t`set(attributes[\"deployment.environment\"], + resource.attributes[\"deployment.environment\"])`,\n\t\t\t`set(attributes[\"service.version\"], + resource.attributes[\"service.version\"])`,\n\t\t]\n\t}\n\n\toutput {\n\t\tmetrics + = [otelcol.processor.k8sattributes.default.input]\n\t}\n}\n\notelcol.processor.k8sattributes + \"default\" {\n\textract {\n\t\tmetadata = [\n\t\t\t\"k8s.namespace.name\",\n\t\t\t\"k8s.pod.name\",\n\t\t\t\"k8s.deployment.name\",\n\t\t\t\"k8s.statefulset.name\",\n\t\t\t\"k8s.daemonset.name\",\n\t\t\t\"k8s.cronjob.name\",\n\t\t\t\"k8s.job.name\",\n\t\t\t\"k8s.node.name\",\n\t\t\t\"k8s.pod.uid\",\n\t\t\t\"k8s.pod.start_time\",\n\t\t]\n\t}\n\n\tpod_association + {\n\t\tsource {\n\t\t\tfrom = \"connection\"\n\t\t}\n\t}\n\n\toutput {\n\t\tmetrics + = [otelcol.processor.transform.add_resource_attributes.input]\n\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\ttraces + \ = [otelcol.processor.transform.add_resource_attributes.input]\n\t}\n}\n\notelcol.processor.transform + \"add_resource_attributes\" {\n\terror_mode = \"ignore\"\n\n\tmetric_statements + {\n\t\tcontext = \"resource\"\n\t\tstatements = [\n\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t]\n\t}\n\n\tlog_statements + {\n\t\tcontext = \"resource\"\n\t\tstatements = [\n\t\t\t`set(attributes[\"pod\"], + attributes[\"k8s.pod.name\"])`,\n\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t`set(attributes[\"loki.resource.labels\"], + \"pod, namespace, cluster, job\")`,\n\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t]\n\t}\n\n\ttrace_statements + {\n\t\tcontext = \"resource\"\n\t\tstatements = [\n\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t]\n\t}\n\n\toutput + {\n\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\tlogs = [otelcol.processor.filter.default.input]\n\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t}\n}\n\notelcol.processor.filter + \"default\" {\n\terror_mode = \"ignore\"\n\n\toutput {\n\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\ttraces = [otelcol.processor.batch.default.input]\n\t}\n}\n\notelcol.processor.batch \"default\" {\n\tsend_batch_size = 16384\n\tsend_batch_max_size = 0\n\ttimeout \ = \"2s\"\n\n\toutput {\n\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t}\n}\n\notelcol.processor.memory_limiter \"default\" {\n\tcheck_interval = \"1s\"\n\tlimit_percentage = 50\n\tspike_limit_percentage - = 30\n\n\toutput {\n\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\tlogs - \ = [otelcol.processor.k8sattributes.default.input]\n\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t}\n}\n\notelcol.processor.k8sattributes - \"default\" {\n\toutput {\n\t\tmetrics = [otelcol.exporter.prometheus.tracesmetrics.input]\n\t\tlogs + = 30\n\n\toutput {\n\t\tmetrics = [otelcol.exporter.prometheus.tracesmetrics.input]\n\t\tlogs \ = [otelcol.exporter.loki.traceslogs.input]\n\t\ttraces = argument.traces_forward_to.value\n\t}\n}\n\notelcol.exporter.prometheus \"tracesmetrics\" {\n\tforward_to = argument.metrics_forward_to.value\n}\n\notelcol.exporter.loki \"traceslogs\" {\n\tforward_to = [loki.process.traceslogs.receiver]\n}\n\n// The @@ -614,7 +613,7 @@ data: {\n\t\tvalue = \"anonymous\"\n\t}\n\n\tforward_to = argument.logs_forward_to.value\n}\n" kind: ConfigMap metadata: - name: agent-modules-k2kfd9mm27 + name: agent-modules-bcccgk262t namespace: monitoring-system --- apiVersion: v1 @@ -754,6 +753,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: http-metrics profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: grafana-agent labels: app.kubernetes.io/instance: grafana-agent app.kubernetes.io/name: grafana-agent @@ -838,7 +838,7 @@ spec: path: /var/log name: varlog - configMap: - name: agent-modules-k2kfd9mm27 + name: agent-modules-bcccgk262t name: agent-modules --- apiVersion: monitoring.coreos.com/v1 diff --git a/kubernetes/common/grafana-agent/values-k3d-k3s.yaml b/kubernetes/common/grafana-agent/values-k3d-k3s.yaml index cfc6389b..c1a8093a 100644 --- a/kubernetes/common/grafana-agent/values-k3d-k3s.yaml +++ b/kubernetes/common/grafana-agent/values-k3d-k3s.yaml @@ -59,6 +59,7 @@ controller: logs.agent.grafana.com/scrape: "true" logs.agent.grafana.com/scrub-level: "debug" # Boolean whether or not the level should be dropped from the log message (as it is a label). # logs.agent.grafana.com/tenant: ".*" # Allow a pod to override the tenant for its logs. + pyroscope.io/service_name: grafana-agent profiles.grafana.com/memory.scrape: "true" profiles.grafana.com/memory.port_name: "http-metrics" profiles.grafana.com/cpu.scrape: "true" diff --git a/kubernetes/microservices-mode/profiles/configs/config.river b/kubernetes/microservices-mode/profiles/configs/config.river new file mode 100644 index 00000000..96eb0c76 --- /dev/null +++ b/kubernetes/microservices-mode/profiles/configs/config.river @@ -0,0 +1,33 @@ +/* +The following example shows using the default all logs processing module, for +a single tenant and specifying the destination url/credentials via environment +variables. +*/ +logging { + level = coalesce(env("AGENT_LOG_LEVEL"), "info") + format = "logfmt" +} + +module.file "lgtmp" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/lgtmp.river" + + arguments { + cluster = coalesce(env("CLUSTER"), "k3d-k3s-codelab") + logs_endpoint = coalesce(env("LOGS_ENDPOINT"), "http://nginx.gateway.svc:3100") + metrics_endpoint = coalesce(env("METRICS_ENDPOINT"), "http://nginx.gateway.svc:8080") + profiles_endpoint = coalesce(env("PROFILES_ENDPOINT"), "http://nginx.gateway.svc:4040") + traces_endpoint = coalesce(env("TRACES_ENDPOINT"), "nginx.gateway.svc:4317") + } +} + +/******************************************** + * Profiles + ********************************************/ +module.file "profiles_primary" { + filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/profiles.river" + + arguments { + forward_to = [module.file.lgtmp.exports.profiles_receiver] + clustering = true + } +} \ No newline at end of file diff --git a/kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml b/kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml index 79a96aff..3a912193 100644 --- a/kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml +++ b/kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml @@ -100,6 +100,27 @@ metadata: namespace: gateway --- apiVersion: v1 +data: + config.river: "/*\nThe following example shows using the default all logs processing + module, for\na single tenant and specifying the destination url/credentials via + environment\nvariables.\n*/\nlogging {\n\tlevel = coalesce(env(\"AGENT_LOG_LEVEL\"), + \"info\")\n\tformat = \"logfmt\"\n}\n\nmodule.file \"lgtmp\" {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), + \"/etc/agent-modules\") + \"/lgtmp.river\"\n\n\targuments {\n\t\tcluster = + coalesce(env(\"CLUSTER\"), \"k3d-k3s-codelab\")\n\t\tlogs_endpoint = coalesce(env(\"LOGS_ENDPOINT\"), + \"http://nginx.gateway.svc:3100\")\n\t\tmetrics_endpoint = coalesce(env(\"METRICS_ENDPOINT\"), + \"http://nginx.gateway.svc:8080\")\n\t\tprofiles_endpoint = coalesce(env(\"PROFILES_ENDPOINT\"), + \"http://nginx.gateway.svc:4040\")\n\t\ttraces_endpoint = coalesce(env(\"TRACES_ENDPOINT\"), + \"nginx.gateway.svc:4317\")\n\t}\n}\n\n/********************************************\n + * Profiles\n ********************************************/\nmodule.file \"profiles_primary\" + {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\") + + \"/profiles.river\"\n\n\targuments {\n\t\tforward_to = [module.file.lgtmp.exports.profiles_receiver]\n\t\tclustering + = true\n\t}\n}" +kind: ConfigMap +metadata: + name: agent-config + namespace: monitoring-system +--- +apiVersion: v1 data: datasources.yaml: | apiVersion: 1 diff --git a/kubernetes/microservices-mode/profiles/kustomization.yaml b/kubernetes/microservices-mode/profiles/kustomization.yaml index aa468368..49274202 100644 --- a/kubernetes/microservices-mode/profiles/kustomization.yaml +++ b/kubernetes/microservices-mode/profiles/kustomization.yaml @@ -28,6 +28,13 @@ configMapGenerator: files: - datasources.yaml=configs/grafana-datasources-pyroscope.yaml +- name: agent-config + namespace: monitoring-system + options: + disableNameSuffixHash: true + files: + - configs/config.river + - name: pyroscope-config namespace: profiles-system options: diff --git a/kubernetes/microservices-mode/traces/configs/config.river b/kubernetes/microservices-mode/traces/configs/config.river index 04833a08..b66810e9 100644 --- a/kubernetes/microservices-mode/traces/configs/config.river +++ b/kubernetes/microservices-mode/traces/configs/config.river @@ -36,40 +36,7 @@ module.file "traces_primary" { tracing { sampling_fraction = 0.8 - write_to = [otelcol.processor.batch.containers.input] -} - -otelcol.processor.batch "containers" { - send_batch_size = 16384 - send_batch_max_size = 0 - timeout = "2s" - - output { - metrics = [otelcol.processor.memory_limiter.containers.input] - traces = [otelcol.processor.memory_limiter.containers.input] - } -} - -otelcol.processor.memory_limiter "containers" { - check_interval = "1s" - limit_percentage = 50 - spike_limit_percentage = 30 - - output { - metrics = [otelcol.processor.k8sattributes.containers.input] - traces = [otelcol.processor.k8sattributes.containers.input] - } -} - -otelcol.processor.k8sattributes "containers" { - output { - metrics = [otelcol.exporter.prometheus.containers.input] - traces = [module.file.lgtmp.exports.traces_receiver] - } -} - -otelcol.exporter.prometheus "containers" { - forward_to = [module.file.lgtmp.exports.metrics_receiver] + write_to = [module.file.traces_primary.exports.agent_traces_input] } /******************************************** diff --git a/kubernetes/microservices-mode/traces/configs/tempo-distributed.yaml b/kubernetes/microservices-mode/traces/configs/tempo-distributed.yaml index 6e59a5c7..f4a018f1 100644 --- a/kubernetes/microservices-mode/traces/configs/tempo-distributed.yaml +++ b/kubernetes/microservices-mode/traces/configs/tempo-distributed.yaml @@ -39,6 +39,7 @@ ingester: tokens_file_path: /var/tempo/tokens.json memberlist: + abort_if_cluster_join_fails: false join_members: - dns+tempo-distributed-gossip-ring:7946 diff --git a/kubernetes/microservices-mode/traces/k8s-all-in-one.yaml b/kubernetes/microservices-mode/traces/k8s-all-in-one.yaml index 4c63bd72..a019f75b 100644 --- a/kubernetes/microservices-mode/traces/k8s-all-in-one.yaml +++ b/kubernetes/microservices-mode/traces/k8s-all-in-one.yaml @@ -949,16 +949,7 @@ data: + \"/traces.river\"\n\n\targuments {\n\t\tmetrics_forward_to = [module.file.lgtmp.exports.metrics_receiver]\n\t\tlogs_forward_to \ = [module.file.lgtmp.exports.logs_receiver]\n\t\ttraces_forward_to = [module.file.lgtmp.exports.traces_receiver]\n\t\tcluster \ = coalesce(env(\"CLUSTER\"), \"k3d-k3s-codelab\")\n\t}\n}\n\ntracing - {\n\tsampling_fraction = 0.8\n\twrite_to = [otelcol.processor.batch.containers.input]\n}\n\notelcol.processor.batch - \"containers\" {\n\tsend_batch_size = 16384\n\tsend_batch_max_size = 0\n\ttimeout - \ = \"2s\"\n\n\toutput {\n\t\tmetrics = [otelcol.processor.memory_limiter.containers.input]\n\t\ttraces - \ = [otelcol.processor.memory_limiter.containers.input]\n\t}\n}\n\notelcol.processor.memory_limiter - \"containers\" {\n\tcheck_interval = \"1s\"\n\tlimit_percentage = - 50\n\tspike_limit_percentage = 30\n\n\toutput {\n\t\tmetrics = [otelcol.processor.k8sattributes.containers.input]\n\t\ttraces - \ = [otelcol.processor.k8sattributes.containers.input]\n\t}\n}\n\notelcol.processor.k8sattributes - \"containers\" {\n\toutput {\n\t\tmetrics = [otelcol.exporter.prometheus.containers.input]\n\t\ttraces - \ = [module.file.lgtmp.exports.traces_receiver]\n\t}\n}\n\notelcol.exporter.prometheus - \"containers\" {\n\tforward_to = [module.file.lgtmp.exports.metrics_receiver]\n}\n\n/********************************************\n + {\n\tsampling_fraction = 0.8\n\twrite_to = [module.file.traces_primary.exports.agent_traces_input]\n}\n\n/********************************************\n * Metrics\n ********************************************/\nmodule.file \"metrics_primary\" {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\") + \"/metrics.river\"\n\n\targuments {\n\t\tforward_to = [module.file.lgtmp.exports.metrics_receiver]\n\t\tclustering @@ -43482,6 +43473,7 @@ data: tokens_file_path: /var/tempo/tokens.json memberlist: + abort_if_cluster_join_fails: false join_members: - dns+tempo-distributed-gossip-ring:7946 @@ -43574,8 +43566,10 @@ data: Bjb3JyZWxhdGlvbnMgbWV0cmljc1N1bW1hcnkgdHJhY2VUb01ldHJpY3MgdHJhY2VUb1By b2ZpbGVz GF_LOG_LEVEL: d2Fybg== - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES: c2VydmljZS5uYW1lc3BhY2U6bW9uaXRvcmluZy1zeXN0ZW0= - GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbTo0MzE3 + GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES: YXBwOmdyYWZhbmE= + GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS: | + Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbDo0Mz + E3 NAMESPACE: bW9uaXRvcmluZy1zeXN0ZW0= kind: Secret metadata: @@ -43585,29 +43579,29 @@ type: Opaque --- apiVersion: v1 data: - JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbQ== + JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbA== JAEGER_AGENT_PORT: NjgzMQ== JAEGER_SAMPLER_PARAM: MQ== JAEGER_SAMPLER_TYPE: Y29uc3Q= - JAEGER_SERVICE_NAME: bWltaXI= + JAEGER_TAGS: YXBwPW1pbWly MIMIR_S3_SECRET_ACCESS_KEY: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= kind: Secret metadata: - name: mimir-env-kkhtd4d9mb + name: mimir-env-hb789m659c namespace: monitoring-system type: Opaque --- apiVersion: v1 data: - JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbQ== + JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbA== JAEGER_AGENT_PORT: NjgzMQ== JAEGER_SAMPLER_PARAM: MQ== JAEGER_SAMPLER_TYPE: Y29uc3Q= - JAEGER_SERVICE_NAME: dGVtcG8= + JAEGER_TAGS: YXBwPXRlbXBv TEMPO_S3_SECRET_KEY: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= kind: Secret metadata: - name: tempo-distributed-env-457c2t4d7c + name: tempo-distributed-env-gk54k88t7g namespace: tracing-system type: Opaque --- @@ -44020,6 +44014,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: http-metrics profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: mimir labels: app.kubernetes.io/component: mimir app.kubernetes.io/instance: mimir-monolithic-mode @@ -44039,7 +44034,7 @@ spec: fieldPath: status.podIP envFrom: - secretRef: - name: mimir-env-kkhtd4d9mb + name: mimir-env-hb789m659c image: docker.io/grafana/mimir:2.11.0 imagePullPolicy: IfNotPresent name: mimir @@ -44130,7 +44125,7 @@ spec: - -config.expand-env=true envFrom: - secretRef: - name: tempo-distributed-env-457c2t4d7c + name: tempo-distributed-env-gk54k88t7g image: docker.io/grafana/tempo:2.4.0 imagePullPolicy: IfNotPresent name: compactor @@ -44246,7 +44241,7 @@ spec: - -config.expand-env=true envFrom: - secretRef: - name: tempo-distributed-env-457c2t4d7c + name: tempo-distributed-env-gk54k88t7g image: docker.io/grafana/tempo:2.4.0 imagePullPolicy: IfNotPresent name: distributor @@ -44517,7 +44512,7 @@ spec: - -config.expand-env=true envFrom: - secretRef: - name: tempo-distributed-env-457c2t4d7c + name: tempo-distributed-env-gk54k88t7g image: docker.io/grafana/tempo:2.4.0 imagePullPolicy: IfNotPresent name: querier @@ -44651,7 +44646,7 @@ spec: - -config.expand-env=true envFrom: - secretRef: - name: tempo-distributed-env-457c2t4d7c + name: tempo-distributed-env-gk54k88t7g image: docker.io/grafana/tempo:2.4.0 imagePullPolicy: IfNotPresent name: query-frontend @@ -44777,7 +44772,7 @@ spec: - -config.expand-env=true envFrom: - secretRef: - name: tempo-distributed-env-457c2t4d7c + name: tempo-distributed-env-gk54k88t7g image: docker.io/grafana/tempo:2.4.0 imagePullPolicy: IfNotPresent name: ingester diff --git a/kubernetes/microservices-mode/traces/kustomization.yaml b/kubernetes/microservices-mode/traces/kustomization.yaml index 7612c81c..5368f3b9 100644 --- a/kubernetes/microservices-mode/traces/kustomization.yaml +++ b/kubernetes/microservices-mode/traces/kustomization.yaml @@ -22,17 +22,17 @@ secretGenerator: literals: - NAMESPACE=monitoring-system - GF_LOG_LEVEL=warn - - GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS=grafana-agent.monitoring-system:4317 - - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES=service.namespace:monitoring-system + - GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS=grafana-agent.monitoring-system.svc.cluster.local:4317 + - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES=app:grafana - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor tracesEmbeddedFlameGraph traceqlSearch correlations metricsSummary traceToMetrics traceToProfiles - name: tempo-distributed-env namespace: tracing-system literals: - TEMPO_S3_SECRET_KEY=VD538OYxSEiGD4I9mmFfqFMCGq1vIiGm - - JAEGER_SERVICE_NAME=tempo - - JAEGER_AGENT_HOST=grafana-agent.monitoring-system + - JAEGER_AGENT_HOST=grafana-agent.monitoring-system.svc.cluster.local - JAEGER_AGENT_PORT=6831 + - JAEGER_TAGS=app=tempo - JAEGER_SAMPLER_TYPE=const - JAEGER_SAMPLER_PARAM=1 @@ -40,9 +40,9 @@ secretGenerator: namespace: monitoring-system literals: - MIMIR_S3_SECRET_ACCESS_KEY=VD538OYxSEiGD4I9mmFfqFMCGq1vIiGm - - JAEGER_SERVICE_NAME=mimir - - JAEGER_AGENT_HOST=grafana-agent.monitoring-system + - JAEGER_AGENT_HOST=grafana-agent.monitoring-system.svc.cluster.local - JAEGER_AGENT_PORT=6831 + - JAEGER_TAGS=app=mimir - JAEGER_SAMPLER_TYPE=const - JAEGER_SAMPLER_PARAM=1 diff --git a/kubernetes/monolithic-mode/all-in-one/configs/config.river b/kubernetes/monolithic-mode/all-in-one/configs/config.river index a24ccc6a..78c4abe0 100644 --- a/kubernetes/monolithic-mode/all-in-one/configs/config.river +++ b/kubernetes/monolithic-mode/all-in-one/configs/config.river @@ -61,55 +61,7 @@ module.file "traces_primary" { tracing { sampling_fraction = 0.8 - write_to = [otelcol.processor.batch.containers.input] -} - -otelcol.processor.batch "containers" { - send_batch_size = 16384 - send_batch_max_size = 0 - timeout = "2s" - - output { - metrics = [otelcol.processor.memory_limiter.containers.input] - logs = [otelcol.processor.memory_limiter.containers.input] - traces = [otelcol.processor.memory_limiter.containers.input] - } -} - -otelcol.processor.memory_limiter "containers" { - check_interval = "1s" - limit_percentage = 50 - spike_limit_percentage = 30 - - output { - metrics = [otelcol.processor.k8sattributes.containers.input] - logs = [otelcol.processor.k8sattributes.containers.input] - traces = [otelcol.processor.k8sattributes.containers.input] - } -} - -otelcol.processor.k8sattributes "containers" { - output { - metrics = [otelcol.exporter.prometheus.containers.input] - logs = [otelcol.exporter.loki.containers.input] - traces = [module.file.lgtmp.exports.traces_receiver] - } -} - -otelcol.exporter.prometheus "containers" { - forward_to = [module.file.lgtmp.exports.metrics_receiver] -} - -otelcol.exporter.loki "containers" { - forward_to = [loki.process.containers.receiver] -} - -loki.process "containers" { - stage.tenant { - value = "anonymous" - } - - forward_to = [module.file.lgtmp.exports.logs_receiver] + write_to = [module.file.traces_primary.exports.agent_traces_input] } /******************************************** diff --git a/kubernetes/monolithic-mode/all-in-one/configs/grafana-datasources-all-in-one.yaml b/kubernetes/monolithic-mode/all-in-one/configs/grafana-datasources-all-in-one.yaml index c8e1701e..ac1358c2 100644 --- a/kubernetes/monolithic-mode/all-in-one/configs/grafana-datasources-all-in-one.yaml +++ b/kubernetes/monolithic-mode/all-in-one/configs/grafana-datasources-all-in-one.yaml @@ -75,7 +75,7 @@ datasources: datasourceUid: metrics spanStartTimeShift: '-30m' spanEndTimeShift: '30m' - tags: [{ key: 'service.name', value: 'service' }, { key: 'span_name' }, { key: 'http_method' }] + tags: [{ key: 'service.name', value: 'service' }] queries: - name: '(R) Rate' query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[$$__rate_interval]))' @@ -87,14 +87,14 @@ datasources: datasourceUid: logs spanStartTimeShift: '-30m' spanEndTimeShift: '30m' - tags: [{ key: 'service.name', value: 'app' }, { key: 'namespace' }, { key: 'cluster' }] + tags: [{ key: 'app', value: 'app' }] filterByTraceID: false filterBySpanID: false tracesToProfiles: customQuery: false datasourceUid: "profiles" profileTypeId: "process_cpu:cpu:nanoseconds:cpu:nanoseconds" - tags: [{ key: 'service.name', value: 'service_name' }] + tags: [{ key: 'app', value: 'service_name' }] # Pyroscope for profiles - name: Profiles diff --git a/kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml index d77ad346..dcc0abbd 100644 --- a/kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml @@ -1318,19 +1318,7 @@ data: + \"/traces.river\"\n\n\targuments {\n\t\tmetrics_forward_to = [module.file.lgtmp.exports.metrics_receiver]\n\t\tlogs_forward_to \ = [module.file.lgtmp.exports.logs_receiver]\n\t\ttraces_forward_to = [module.file.lgtmp.exports.traces_receiver]\n\t\tcluster \ = coalesce(env(\"CLUSTER\"), \"k3d-k3s-codelab\")\n\t}\n}\n\ntracing - {\n\tsampling_fraction = 0.8\n\twrite_to = [otelcol.processor.batch.containers.input]\n}\n\notelcol.processor.batch - \"containers\" {\n\tsend_batch_size = 16384\n\tsend_batch_max_size = 0\n\ttimeout - \ = \"2s\"\n\n\toutput {\n\t\tmetrics = [otelcol.processor.memory_limiter.containers.input]\n\t\tlogs - \ = [otelcol.processor.memory_limiter.containers.input]\n\t\ttraces = [otelcol.processor.memory_limiter.containers.input]\n\t}\n}\n\notelcol.processor.memory_limiter - \"containers\" {\n\tcheck_interval = \"1s\"\n\tlimit_percentage = - 50\n\tspike_limit_percentage = 30\n\n\toutput {\n\t\tmetrics = [otelcol.processor.k8sattributes.containers.input]\n\t\tlogs - \ = [otelcol.processor.k8sattributes.containers.input]\n\t\ttraces = [otelcol.processor.k8sattributes.containers.input]\n\t}\n}\n\notelcol.processor.k8sattributes - \"containers\" {\n\toutput {\n\t\tmetrics = [otelcol.exporter.prometheus.containers.input]\n\t\tlogs - \ = [otelcol.exporter.loki.containers.input]\n\t\ttraces = [module.file.lgtmp.exports.traces_receiver]\n\t}\n}\n\notelcol.exporter.prometheus - \"containers\" {\n\tforward_to = [module.file.lgtmp.exports.metrics_receiver]\n}\n\notelcol.exporter.loki - \"containers\" {\n\tforward_to = [loki.process.containers.receiver]\n}\n\nloki.process - \"containers\" {\n\tstage.tenant {\n\t\tvalue = \"anonymous\"\n\t}\n\n\tforward_to - = [module.file.lgtmp.exports.logs_receiver]\n}\n\n/********************************************\n + {\n\tsampling_fraction = 0.8\n\twrite_to = [module.file.traces_primary.exports.agent_traces_input]\n}\n\n/********************************************\n * Profiles\n ********************************************/\nmodule.file \"profiles_primary\" {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\") + \"/profiles.river\"\n\n\targuments {\n\t\tforward_to = [module.file.lgtmp.exports.profiles_receiver]\n\t\tclustering @@ -4558,7 +4546,7 @@ data: datasourceUid: metrics spanStartTimeShift: '-30m' spanEndTimeShift: '30m' - tags: [{ key: 'service.name', value: 'service' }, { key: 'span_name' }, { key: 'http_method' }] + tags: [{ key: 'service.name', value: 'service' }] queries: - name: '(R) Rate' query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[$$__rate_interval]))' @@ -4570,14 +4558,14 @@ data: datasourceUid: logs spanStartTimeShift: '-30m' spanEndTimeShift: '30m' - tags: [{ key: 'service.name', value: 'app' }, { key: 'namespace' }, { key: 'cluster' }] + tags: [{ key: 'app', value: 'app' }] filterByTraceID: false filterBySpanID: false tracesToProfiles: customQuery: false datasourceUid: "profiles" profileTypeId: "process_cpu:cpu:nanoseconds:cpu:nanoseconds" - tags: [{ key: 'service.name', value: 'service_name' }] + tags: [{ key: 'app', value: 'service_name' }] # Pyroscope for profiles - name: Profiles @@ -62889,11 +62877,11 @@ metadata: --- apiVersion: v1 data: - JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbQ== + JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbA== JAEGER_AGENT_PORT: NjgzMQ== JAEGER_SAMPLER_PARAM: MQ== JAEGER_SAMPLER_TYPE: Y29uc3Q= - JAEGER_SERVICE_NAME: bG9raQ== + JAEGER_TAGS: YXBwPWxva2k= LOKI_S3_SECRET_ACCESS_KEY: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= kind: Secret metadata: @@ -62908,8 +62896,10 @@ data: Bjb3JyZWxhdGlvbnMgbWV0cmljc1N1bW1hcnkgdHJhY2VUb01ldHJpY3MgdHJhY2VUb1By b2ZpbGVz GF_LOG_LEVEL: d2Fybg== - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES: c2VydmljZS5uYW1lc3BhY2U6bW9uaXRvcmluZy1zeXN0ZW0= - GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbTo0MzE3 + GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES: YXBwOmdyYWZhbmE= + GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS: | + Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbDo0Mz + E3 kind: Secret metadata: name: grafana-env @@ -62918,11 +62908,11 @@ type: Opaque --- apiVersion: v1 data: - JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbQ== + JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbA== JAEGER_AGENT_PORT: NjgzMQ== JAEGER_SAMPLER_PARAM: MQ== JAEGER_SAMPLER_TYPE: Y29uc3Q= - JAEGER_SERVICE_NAME: bWltaXI= + JAEGER_TAGS: YXBwPW1pbWly MIMIR_S3_SECRET_ACCESS_KEY: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= kind: Secret metadata: @@ -62932,11 +62922,11 @@ type: Opaque --- apiVersion: v1 data: - JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbQ== + JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbA== JAEGER_AGENT_PORT: NjgzMQ== JAEGER_SAMPLER_PARAM: MQ== JAEGER_SAMPLER_TYPE: Y29uc3Q= - JAEGER_SERVICE_NAME: cHlyb3Njb3Bl + JAEGER_TAGS: YXBwPXB5cm9zY29wZQ== PYROSCOPE_STORAGE_S3_SECRET_ACCESS_KEY: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= kind: Secret metadata: @@ -62946,11 +62936,11 @@ type: Opaque --- apiVersion: v1 data: - JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbQ== + JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbA== JAEGER_AGENT_PORT: NjgzMQ== JAEGER_SAMPLER_PARAM: MQ== JAEGER_SAMPLER_TYPE: Y29uc3Q= - JAEGER_SERVICE_NAME: dGVtcG8= + JAEGER_TAGS: YXBwPXRlbXBv TEMPO_S3_SECRET_KEY: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= kind: Secret metadata: @@ -63250,6 +63240,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: http-metrics profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: mimir labels: app.kubernetes.io/component: mimir app.kubernetes.io/instance: mimir-monolithic-mode @@ -63343,6 +63334,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: http-metrics profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: loki labels: app.kubernetes.io/component: single-binary app.kubernetes.io/instance: loki @@ -63468,6 +63460,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: http2 profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: pyroscope labels: app.kubernetes.io/component: all app.kubernetes.io/instance: pyroscope @@ -63559,6 +63552,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: prom-metrics profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: tempo labels: app.kubernetes.io/instance: tempo app.kubernetes.io/name: tempo diff --git a/kubernetes/monolithic-mode/all-in-one/kustomization.yaml b/kubernetes/monolithic-mode/all-in-one/kustomization.yaml index cbefe446..09871a75 100644 --- a/kubernetes/monolithic-mode/all-in-one/kustomization.yaml +++ b/kubernetes/monolithic-mode/all-in-one/kustomization.yaml @@ -27,17 +27,17 @@ secretGenerator: namespace: monitoring-system literals: - GF_LOG_LEVEL=warn - - GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS=grafana-agent.monitoring-system:4317 - - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES=service.namespace:monitoring-system + - GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS=grafana-agent.monitoring-system.svc.cluster.local:4317 + - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES=app:grafana - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor tracesEmbeddedFlameGraph traceqlSearch correlations metricsSummary traceToMetrics traceToProfiles - name: loki-env namespace: logging-system literals: - LOKI_S3_SECRET_ACCESS_KEY=VD538OYxSEiGD4I9mmFfqFMCGq1vIiGm - - JAEGER_SERVICE_NAME=loki - - JAEGER_AGENT_HOST=grafana-agent.monitoring-system + - JAEGER_AGENT_HOST=grafana-agent.monitoring-system.svc.cluster.local - JAEGER_AGENT_PORT=6831 + - JAEGER_TAGS=app=loki - JAEGER_SAMPLER_TYPE=const - JAEGER_SAMPLER_PARAM=1 @@ -45,9 +45,9 @@ secretGenerator: namespace: tracing-system literals: - TEMPO_S3_SECRET_KEY=VD538OYxSEiGD4I9mmFfqFMCGq1vIiGm - - JAEGER_SERVICE_NAME=tempo - - JAEGER_AGENT_HOST=grafana-agent.monitoring-system + - JAEGER_AGENT_HOST=grafana-agent.monitoring-system.svc.cluster.local - JAEGER_AGENT_PORT=6831 + - JAEGER_TAGS=app=tempo - JAEGER_SAMPLER_TYPE=const - JAEGER_SAMPLER_PARAM=1 @@ -55,9 +55,9 @@ secretGenerator: namespace: monitoring-system literals: - MIMIR_S3_SECRET_ACCESS_KEY=VD538OYxSEiGD4I9mmFfqFMCGq1vIiGm - - JAEGER_SERVICE_NAME=mimir - - JAEGER_AGENT_HOST=grafana-agent.monitoring-system + - JAEGER_AGENT_HOST=grafana-agent.monitoring-system.svc.cluster.local - JAEGER_AGENT_PORT=6831 + - JAEGER_TAGS=app=mimir - JAEGER_SAMPLER_TYPE=const - JAEGER_SAMPLER_PARAM=1 @@ -65,9 +65,9 @@ secretGenerator: namespace: profiles-system literals: - PYROSCOPE_STORAGE_S3_SECRET_ACCESS_KEY=VD538OYxSEiGD4I9mmFfqFMCGq1vIiGm - - JAEGER_SERVICE_NAME=pyroscope - - JAEGER_AGENT_HOST=grafana-agent.monitoring-system + - JAEGER_AGENT_HOST=grafana-agent.monitoring-system.svc.cluster.local - JAEGER_AGENT_PORT=6831 + - JAEGER_TAGS=app=pyroscope - JAEGER_SAMPLER_TYPE=const - JAEGER_SAMPLER_PARAM=1 diff --git a/kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml index 2e9342db..49710400 100644 --- a/kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml @@ -334,6 +334,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: http-metrics profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: loki labels: app.kubernetes.io/component: single-binary app.kubernetes.io/instance: loki diff --git a/kubernetes/monolithic-mode/logs/loki/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/logs/loki/k8s-all-in-one.yaml index 51d6e890..dfedc576 100644 --- a/kubernetes/monolithic-mode/logs/loki/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/logs/loki/k8s-all-in-one.yaml @@ -253,6 +253,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: http-metrics profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: loki labels: app.kubernetes.io/component: single-binary app.kubernetes.io/instance: loki diff --git a/kubernetes/monolithic-mode/logs/loki/values-k3d-k3s.yaml b/kubernetes/monolithic-mode/logs/loki/values-k3d-k3s.yaml index b22d351d..fbfc9f6f 100644 --- a/kubernetes/monolithic-mode/logs/loki/values-k3d-k3s.yaml +++ b/kubernetes/monolithic-mode/logs/loki/values-k3d-k3s.yaml @@ -5,6 +5,7 @@ loki: # https://github.com/grafana/agent-modules/tree/main/modules/kubernetes logs.agent.grafana.com/scrape: "true" logs.agent.grafana.com/scrub-level: "info" + pyroscope.io/service_name: loki profiles.grafana.com/cpu.scrape: "true" profiles.grafana.com/cpu.port_name: http-metrics profiles.grafana.com/goroutine.scrape: "true" diff --git a/kubernetes/monolithic-mode/metrics/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/metrics/k8s-all-in-one.yaml index f26ec656..4520ed61 100644 --- a/kubernetes/monolithic-mode/metrics/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/metrics/k8s-all-in-one.yaml @@ -43364,6 +43364,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: http-metrics profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: mimir labels: app.kubernetes.io/component: mimir app.kubernetes.io/instance: mimir-monolithic-mode diff --git a/kubernetes/monolithic-mode/metrics/mimir/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/metrics/mimir/k8s-all-in-one.yaml index 383841ce..31e4e380 100644 --- a/kubernetes/monolithic-mode/metrics/mimir/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/metrics/mimir/k8s-all-in-one.yaml @@ -78,6 +78,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: http-metrics profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: mimir labels: app.kubernetes.io/component: mimir app.kubernetes.io/instance: mimir-monolithic-mode diff --git a/kubernetes/monolithic-mode/metrics/mimir/mimir-dep.yaml b/kubernetes/monolithic-mode/metrics/mimir/mimir-dep.yaml index 163dfa3c..89f1916d 100644 --- a/kubernetes/monolithic-mode/metrics/mimir/mimir-dep.yaml +++ b/kubernetes/monolithic-mode/metrics/mimir/mimir-dep.yaml @@ -15,6 +15,7 @@ spec: annotations: logs.agent.grafana.com/scrape: "true" logs.agent.grafana.com/scrub-level: info + pyroscope.io/service_name: mimir profiles.grafana.com/cpu.port_name: http-metrics profiles.grafana.com/cpu.scrape: "true" profiles.grafana.com/goroutine.port_name: http-metrics diff --git a/kubernetes/monolithic-mode/profiles/configs/config.river b/kubernetes/monolithic-mode/profiles/configs/config.river index 0a1dc5e1..96eb0c76 100644 --- a/kubernetes/monolithic-mode/profiles/configs/config.river +++ b/kubernetes/monolithic-mode/profiles/configs/config.river @@ -30,17 +30,4 @@ module.file "profiles_primary" { forward_to = [module.file.lgtmp.exports.profiles_receiver] clustering = true } -} - -/******************************************** - * Agent Integrations - ********************************************/ -module.file "agent_integrations" { - filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/integrations.river" - - arguments { - name = "agent-integrations" - namespace = "monitoring-system" - forward_to = [module.file.lgtmp.exports.metrics_receiver] - } -} +} \ No newline at end of file diff --git a/kubernetes/monolithic-mode/profiles/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/profiles/k8s-all-in-one.yaml index ff50c190..370f3de0 100644 --- a/kubernetes/monolithic-mode/profiles/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/profiles/k8s-all-in-one.yaml @@ -76,11 +76,7 @@ data: * Profiles\n ********************************************/\nmodule.file \"profiles_primary\" {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\") + \"/profiles.river\"\n\n\targuments {\n\t\tforward_to = [module.file.lgtmp.exports.profiles_receiver]\n\t\tclustering - = true\n\t}\n}\n\n/********************************************\n * Agent Integrations\n - ********************************************/\nmodule.file \"agent_integrations\" - {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\") - + \"/integrations.river\"\n\n\targuments {\n\t\tname = \"agent-integrations\"\n\t\tnamespace - \ = \"monitoring-system\"\n\t\tforward_to = [module.file.lgtmp.exports.metrics_receiver]\n\t}\n}\n" + = true\n\t}\n}" kind: ConfigMap metadata: name: agent-config @@ -270,6 +266,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: http2 profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: pyroscope labels: app.kubernetes.io/component: all app.kubernetes.io/instance: pyroscope diff --git a/kubernetes/monolithic-mode/profiles/pyroscope/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/profiles/pyroscope/k8s-all-in-one.yaml index 9564cce7..b2c3cf63 100644 --- a/kubernetes/monolithic-mode/profiles/pyroscope/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/profiles/pyroscope/k8s-all-in-one.yaml @@ -196,6 +196,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: http2 profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: pyroscope labels: app.kubernetes.io/component: all app.kubernetes.io/instance: pyroscope diff --git a/kubernetes/monolithic-mode/profiles/pyroscope/values-k3d-k3s.yaml b/kubernetes/monolithic-mode/profiles/pyroscope/values-k3d-k3s.yaml index 153c37d7..858d24c5 100644 --- a/kubernetes/monolithic-mode/profiles/pyroscope/values-k3d-k3s.yaml +++ b/kubernetes/monolithic-mode/profiles/pyroscope/values-k3d-k3s.yaml @@ -11,3 +11,14 @@ pyroscope: extraEnvFrom: - secretRef: name: pyroscope-env + podAnnotations: + # Scrapes itself see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles + profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: pyroscope + profiles.grafana.com/memory.port_name: http2 + profiles.grafana.com/cpu.scrape: "true" + profiles.grafana.com/cpu.port_name: http2 + profiles.grafana.com/goroutine.scrape: "true" + profiles.grafana.com/goroutine.port_name: http2 + # profiles.grafana.com/block.scrape: "true" + # profiles.grafana.com/mutex.scrape: "true" diff --git a/kubernetes/monolithic-mode/traces/configs/config.river b/kubernetes/monolithic-mode/traces/configs/config.river index 04833a08..b66810e9 100644 --- a/kubernetes/monolithic-mode/traces/configs/config.river +++ b/kubernetes/monolithic-mode/traces/configs/config.river @@ -36,40 +36,7 @@ module.file "traces_primary" { tracing { sampling_fraction = 0.8 - write_to = [otelcol.processor.batch.containers.input] -} - -otelcol.processor.batch "containers" { - send_batch_size = 16384 - send_batch_max_size = 0 - timeout = "2s" - - output { - metrics = [otelcol.processor.memory_limiter.containers.input] - traces = [otelcol.processor.memory_limiter.containers.input] - } -} - -otelcol.processor.memory_limiter "containers" { - check_interval = "1s" - limit_percentage = 50 - spike_limit_percentage = 30 - - output { - metrics = [otelcol.processor.k8sattributes.containers.input] - traces = [otelcol.processor.k8sattributes.containers.input] - } -} - -otelcol.processor.k8sattributes "containers" { - output { - metrics = [otelcol.exporter.prometheus.containers.input] - traces = [module.file.lgtmp.exports.traces_receiver] - } -} - -otelcol.exporter.prometheus "containers" { - forward_to = [module.file.lgtmp.exports.metrics_receiver] + write_to = [module.file.traces_primary.exports.agent_traces_input] } /******************************************** diff --git a/kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml index 103277bf..625900a0 100644 --- a/kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml @@ -949,16 +949,7 @@ data: + \"/traces.river\"\n\n\targuments {\n\t\tmetrics_forward_to = [module.file.lgtmp.exports.metrics_receiver]\n\t\tlogs_forward_to \ = [module.file.lgtmp.exports.logs_receiver]\n\t\ttraces_forward_to = [module.file.lgtmp.exports.traces_receiver]\n\t\tcluster \ = coalesce(env(\"CLUSTER\"), \"k3d-k3s-codelab\")\n\t}\n}\n\ntracing - {\n\tsampling_fraction = 0.8\n\twrite_to = [otelcol.processor.batch.containers.input]\n}\n\notelcol.processor.batch - \"containers\" {\n\tsend_batch_size = 16384\n\tsend_batch_max_size = 0\n\ttimeout - \ = \"2s\"\n\n\toutput {\n\t\tmetrics = [otelcol.processor.memory_limiter.containers.input]\n\t\ttraces - \ = [otelcol.processor.memory_limiter.containers.input]\n\t}\n}\n\notelcol.processor.memory_limiter - \"containers\" {\n\tcheck_interval = \"1s\"\n\tlimit_percentage = - 50\n\tspike_limit_percentage = 30\n\n\toutput {\n\t\tmetrics = [otelcol.processor.k8sattributes.containers.input]\n\t\ttraces - \ = [otelcol.processor.k8sattributes.containers.input]\n\t}\n}\n\notelcol.processor.k8sattributes - \"containers\" {\n\toutput {\n\t\tmetrics = [otelcol.exporter.prometheus.containers.input]\n\t\ttraces - \ = [module.file.lgtmp.exports.traces_receiver]\n\t}\n}\n\notelcol.exporter.prometheus - \"containers\" {\n\tforward_to = [module.file.lgtmp.exports.metrics_receiver]\n}\n\n/********************************************\n + {\n\tsampling_fraction = 0.8\n\twrite_to = [module.file.traces_primary.exports.agent_traces_input]\n}\n\n/********************************************\n * Metrics\n ********************************************/\nmodule.file \"metrics_primary\" {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"), \"/etc/agent-modules\") + \"/metrics.river\"\n\n\targuments {\n\t\tforward_to = [module.file.lgtmp.exports.metrics_receiver]\n\t\tclustering @@ -43550,8 +43541,10 @@ data: Bjb3JyZWxhdGlvbnMgbWV0cmljc1N1bW1hcnkgdHJhY2VUb01ldHJpY3MgdHJhY2VUb1By b2ZpbGVz GF_LOG_LEVEL: d2Fybg== - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES: c2VydmljZS5uYW1lc3BhY2U6bW9uaXRvcmluZy1zeXN0ZW0= - GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbTo0MzE3 + GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES: YXBwOmdyYWZhbmE= + GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS: | + Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbDo0Mz + E3 NAMESPACE: bW9uaXRvcmluZy1zeXN0ZW0= kind: Secret metadata: @@ -43561,29 +43554,29 @@ type: Opaque --- apiVersion: v1 data: - JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbQ== + JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbA== JAEGER_AGENT_PORT: NjgzMQ== JAEGER_SAMPLER_PARAM: MQ== JAEGER_SAMPLER_TYPE: Y29uc3Q= - JAEGER_SERVICE_NAME: bWltaXI= + JAEGER_TAGS: YXBwPW1pbWly MIMIR_S3_SECRET_ACCESS_KEY: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= kind: Secret metadata: - name: mimir-env-kkhtd4d9mb + name: mimir-env-hb789m659c namespace: monitoring-system type: Opaque --- apiVersion: v1 data: - JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbQ== + JAEGER_AGENT_HOST: Z3JhZmFuYS1hZ2VudC5tb25pdG9yaW5nLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbA== JAEGER_AGENT_PORT: NjgzMQ== JAEGER_SAMPLER_PARAM: MQ== JAEGER_SAMPLER_TYPE: Y29uc3Q= - JAEGER_SERVICE_NAME: dGVtcG8= + JAEGER_TAGS: YXBwPXRlbXBv TEMPO_S3_SECRET_KEY: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= kind: Secret metadata: - name: tempo-env-457c2t4d7c + name: tempo-env-gk54k88t7g namespace: tracing-system type: Opaque --- @@ -43731,6 +43724,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: http-metrics profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: mimir labels: app.kubernetes.io/component: mimir app.kubernetes.io/instance: mimir-monolithic-mode @@ -43750,7 +43744,7 @@ spec: fieldPath: status.podIP envFrom: - secretRef: - name: mimir-env-kkhtd4d9mb + name: mimir-env-hb789m659c image: docker.io/grafana/mimir:2.11.0 imagePullPolicy: IfNotPresent name: mimir @@ -43816,6 +43810,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: prom-metrics profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: tempo labels: app.kubernetes.io/instance: tempo app.kubernetes.io/name: tempo @@ -43828,7 +43823,7 @@ spec: - -config.expand-env=true envFrom: - secretRef: - name: tempo-env-457c2t4d7c + name: tempo-env-gk54k88t7g image: grafana/tempo:2.4.0 imagePullPolicy: IfNotPresent name: tempo diff --git a/kubernetes/monolithic-mode/traces/kustomization.yaml b/kubernetes/monolithic-mode/traces/kustomization.yaml index c633e2a0..d80ddbf4 100644 --- a/kubernetes/monolithic-mode/traces/kustomization.yaml +++ b/kubernetes/monolithic-mode/traces/kustomization.yaml @@ -23,17 +23,17 @@ secretGenerator: literals: - NAMESPACE=monitoring-system - GF_LOG_LEVEL=warn - - GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS=grafana-agent.monitoring-system:4317 - - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES=service.namespace:monitoring-system + - GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS=grafana-agent.monitoring-system.svc.cluster.local:4317 + - GF_TRACING_OPENTELEMETRY_CUSTOM_ATTRIBUTES=app:grafana - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor tracesEmbeddedFlameGraph traceqlSearch correlations metricsSummary traceToMetrics traceToProfiles - name: tempo-env namespace: tracing-system literals: - TEMPO_S3_SECRET_KEY=VD538OYxSEiGD4I9mmFfqFMCGq1vIiGm - - JAEGER_SERVICE_NAME=tempo - - JAEGER_AGENT_HOST=grafana-agent.monitoring-system + - JAEGER_AGENT_HOST=grafana-agent.monitoring-system.svc.cluster.local - JAEGER_AGENT_PORT=6831 + - JAEGER_TAGS=app=tempo - JAEGER_SAMPLER_TYPE=const - JAEGER_SAMPLER_PARAM=1 @@ -41,9 +41,9 @@ secretGenerator: namespace: monitoring-system literals: - MIMIR_S3_SECRET_ACCESS_KEY=VD538OYxSEiGD4I9mmFfqFMCGq1vIiGm - - JAEGER_SERVICE_NAME=mimir - - JAEGER_AGENT_HOST=grafana-agent.monitoring-system + - JAEGER_AGENT_HOST=grafana-agent.monitoring-system.svc.cluster.local - JAEGER_AGENT_PORT=6831 + - JAEGER_TAGS=app=mimir - JAEGER_SAMPLER_TYPE=const - JAEGER_SAMPLER_PARAM=1 diff --git a/kubernetes/monolithic-mode/traces/tempo/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/traces/tempo/k8s-all-in-one.yaml index 1ae16985..63fdc646 100644 --- a/kubernetes/monolithic-mode/traces/tempo/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/traces/tempo/k8s-all-in-one.yaml @@ -177,6 +177,7 @@ spec: profiles.grafana.com/goroutine.scrape: "true" profiles.grafana.com/memory.port_name: prom-metrics profiles.grafana.com/memory.scrape: "true" + pyroscope.io/service_name: tempo labels: app.kubernetes.io/instance: tempo app.kubernetes.io/name: tempo diff --git a/kubernetes/monolithic-mode/traces/tempo/values-k3d-k3s.yaml b/kubernetes/monolithic-mode/traces/tempo/values-k3d-k3s.yaml index 6241b94e..bb446dd7 100644 --- a/kubernetes/monolithic-mode/traces/tempo/values-k3d-k3s.yaml +++ b/kubernetes/monolithic-mode/traces/tempo/values-k3d-k3s.yaml @@ -2,6 +2,7 @@ podAnnotations: # https://github.com/grafana/agent-modules/tree/main/modules/kubernetes logs.agent.grafana.com/scrape: "true" logs.agent.grafana.com/scrub-level: "info" + pyroscope.io/service_name: tempo profiles.grafana.com/cpu.scrape: "true" profiles.grafana.com/cpu.port_name: prom-metrics profiles.grafana.com/goroutine.scrape: "true"